diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee25567f..ffffe185 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -231,19 +231,26 @@ jobs: if-no-files-found: error retention-days: 14 - - name: Publish BackFiller linux-x64 - # Same Native AOT invocation as the VectorNNTP release script, plus the + - name: Publish production services linux-x64 + # Same Native AOT invocation as tools/Publish-Release.ps1, plus the # x64 platform contract. Publish restores the ILCompiler pack and the # StuDev.AotAnywhere Zig toolchain for this runtime identifier. + # Both services share one flat artifact directory (identical RabbitMQ JSON). run: | - dotnet publish src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj \ - --configuration Release \ - -p:Platform=x64 \ - --runtime linux-x64 \ - --self-contained true \ - -p:PublishAot=true \ - -p:SelfContained=true \ - --output ./.artifacts/linux-x64 + set -euo pipefail + for proj in \ + src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj \ + src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj + do + dotnet publish "$proj" \ + --configuration Release \ + -p:Platform=x64 \ + --runtime linux-x64 \ + --self-contained true \ + -p:PublishAot=true \ + -p:SelfContained=true \ + --output ./.artifacts/linux-x64 + done - name: Add Linux deployment files shell: bash @@ -252,55 +259,71 @@ jobs: dir=./.artifacts/linux-x64 test -f deploy/systemd/vectornntp-backfiller.service test -f deploy/systemd/backfiller.env.example + test -f deploy/systemd/vectornntp-storageservice.service + test -f deploy/systemd/storageservice.env.example test -f README.md cp deploy/systemd/vectornntp-backfiller.service "$dir/vectornntp-backfiller.service" cp deploy/systemd/backfiller.env.example "$dir/backfiller.env.example" + cp deploy/systemd/vectornntp-storageservice.service "$dir/vectornntp-storageservice.service" + cp deploy/systemd/storageservice.env.example "$dir/storageservice.env.example" cp README.md "$dir/README.md" test -f "$dir/VectorNNTPD.BackFillerService.json" + test -f "$dir/VectorNNTPD.StorageService.json" test -f "$dir/VectorNNTPD.RabbitMQ.json" test ! -e "$dir/VectorNNTP.RabbitMQ.json" + test ! -e "$dir/VectorNNTP.StorageServer.json" test ! -e "$dir/appsettings.json" test ! -e "$dir/RabbitMq.json" - - name: Validate linux-x64 Native AOT executable + - name: Validate linux-x64 Native AOT executables shell: bash run: | set -euo pipefail dir=./.artifacts/linux-x64 - exe="$dir/VectorNNTPD.BackFillerService" - test -f "$exe" - test -x "$exe" + for exe_name in VectorNNTPD.BackFillerService VectorNNTPD.StorageService; do + exe="$dir/$exe_name" + test -f "$exe" + test -x "$exe" + file "$exe" + readelf -h "$exe" + objdump -f "$exe" + file -b "$exe" | grep -q "ELF 64-bit LSB" + file -b "$exe" | grep -q "x86-64" + readelf -h "$exe" | grep -q "Class:[[:space:]]*ELF64" + readelf -h "$exe" | grep -q "Data:[[:space:]]*2's complement, little endian" + readelf -h "$exe" | grep -q "Machine:[[:space:]]*Advanced Micro Devices X86-64" + readelf -h "$exe" | grep -E -q "Type:[[:space:]]*(EXEC|DYN)" + objdump -f "$exe" | grep -q "elf64-x86-64" + if ldd "$exe" | grep -E -q 'coreclr|hostfxr'; then + echo "Executable depends on the shared CLR: $exe" + ldd "$exe" + exit 1 + fi + done test -f "$dir/vectornntp-backfiller.service" test -f "$dir/backfiller.env.example" + test -f "$dir/vectornntp-storageservice.service" + test -f "$dir/storageservice.env.example" test -f "$dir/README.md" test -f "$dir/VectorNNTPD.BackFillerService.json" + test -f "$dir/VectorNNTPD.StorageService.json" test -f "$dir/VectorNNTPD.RabbitMQ.json" test ! -e "$dir/VectorNNTP.RabbitMQ.json" + test ! -e "$dir/VectorNNTP.StorageServer.json" test ! -e "$dir/appsettings.json" test ! -e "$dir/RabbitMq.json" - file "$exe" - readelf -h "$exe" - objdump -f "$exe" - file -b "$exe" | grep -q "ELF 64-bit LSB" - file -b "$exe" | grep -q "x86-64" - readelf -h "$exe" | grep -q "Class:[[:space:]]*ELF64" - readelf -h "$exe" | grep -q "Data:[[:space:]]*2's complement, little endian" - readelf -h "$exe" | grep -q "Machine:[[:space:]]*Advanced Micro Devices X86-64" - readelf -h "$exe" | grep -E -q "Type:[[:space:]]*(EXEC|DYN)" - objdump -f "$exe" | grep -q "elf64-x86-64" - for forbidden in VectorNNTPD.BackFillerService.dll libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll; do + for forbidden in \ + VectorNNTPD.BackFillerService.dll \ + VectorNNTPD.StorageService.dll \ + libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll + do if [ -e "$dir/$forbidden" ]; then echo "Native AOT output contains $forbidden." exit 1 fi done - if ldd "$exe" | grep -E -q 'coreclr|hostfxr'; then - echo "Executable depends on the shared CLR." - ldd "$exe" - exit 1 - fi - - name: Smoke-test linux-x64 Native AOT startup + - name: Smoke-test linux-x64 Native AOT startup (BackFiller) shell: bash run: | set -euo pipefail @@ -311,28 +334,59 @@ jobs: export VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1 export VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG="$config" set +e - timeout 30 "$exe" >./.artifacts/linux-x64-smoke-stdout.txt 2>./.artifacts/linux-x64-smoke-stderr.txt + timeout 30 "$exe" >./.artifacts/linux-x64-backfiller-smoke-stdout.txt 2>./.artifacts/linux-x64-backfiller-smoke-stderr.txt code=$? set -e - echo "linux-x64 startup smoke exit=$code" + echo "linux-x64 BackFiller startup smoke exit=$code" echo "----- stdout -----" - cat ./.artifacts/linux-x64-smoke-stdout.txt + cat ./.artifacts/linux-x64-backfiller-smoke-stdout.txt echo "----- stderr -----" - cat ./.artifacts/linux-x64-smoke-stderr.txt + cat ./.artifacts/linux-x64-backfiller-smoke-stderr.txt if [ "$code" -eq 124 ]; then - echo "linux-x64 startup smoke timed out." + echo "linux-x64 BackFiller startup smoke timed out." exit 1 fi if [ "$code" -ne 0 ]; then - echo "linux-x64 startup smoke failed." + echo "linux-x64 BackFiller startup smoke failed." exit 1 fi - grep -q "BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja" ./.artifacts/linux-x64-smoke-stdout.txt + grep -q "BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja" ./.artifacts/linux-x64-backfiller-smoke-stdout.txt + + - name: Smoke-test linux-x64 Native AOT startup (StorageService) + shell: bash + run: | + set -euo pipefail + exe=./.artifacts/linux-x64/VectorNNTPD.StorageService + config="$PWD/tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json" + test -f "$exe" + test -f "$config" + export VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE=1 + export VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG="$config" + set +e + timeout 30 "$exe" >./.artifacts/linux-x64-storageservice-smoke-stdout.txt 2>./.artifacts/linux-x64-storageservice-smoke-stderr.txt + code=$? + set -e + echo "linux-x64 StorageService startup smoke exit=$code" + echo "----- stdout -----" + cat ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + echo "----- stderr -----" + cat ./.artifacts/linux-x64-storageservice-smoke-stderr.txt + if [ "$code" -eq 124 ]; then + echo "linux-x64 StorageService startup smoke timed out." + exit 1 + fi + if [ "$code" -ne 0 ]; then + echo "linux-x64 StorageService startup smoke failed." + exit 1 + fi + grep -q "StorageService startup smoke: configuration serverId=1 fqdn=cache01.usenet.ninja" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + grep -q "Application logging initialized" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + grep -q "VectorNNTPD.StorageService host starting" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt - name: Upload Linux build artifact uses: actions/upload-artifact@v7 with: - name: backfiller-linux-x64-ci-build + name: vectornntpd-linux-x64-ci-build path: .artifacts/linux-x64/ if-no-files-found: error retention-days: 7 @@ -361,58 +415,76 @@ jobs: # VectorNNTPD.slnx is the canonical CI entrypoint for the production and test projects. run: dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore - - name: Publish BackFiller win-x64 + - name: Publish production services win-x64 # Native AOT. PublishAot is true only for this runtime identifier, and the # ILCompiler pack is restored only when that identifier is present. # The solution restore above does not pass a runtime identifier, so this # publish restores itself. --no-restore here would publish a self-contained # managed app and the RequireNativeAotCompiler target would fail. + # Both services share one flat artifact directory (identical RabbitMQ JSON). shell: pwsh run: | - dotnet publish src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj ` - --configuration Release ` - -p:Platform=x64 ` - --runtime win-x64 ` - --self-contained true ` - -p:PublishAot=true ` - -p:SelfContained=true ` - --output ./.artifacts/win-x64 - - - name: Validate win-x64 Native AOT executable + $ErrorActionPreference = 'Stop' + foreach ($proj in @( + 'src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj', + 'src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj' + )) { + dotnet publish $proj ` + --configuration Release ` + -p:Platform=x64 ` + --runtime win-x64 ` + --self-contained true ` + -p:PublishAot=true ` + -p:SelfContained=true ` + --output ./.artifacts/win-x64 + if ($LASTEXITCODE -ne 0) { throw "dotnet publish failed for $proj" } + } + + - name: Validate win-x64 Native AOT executables shell: pwsh run: | $ErrorActionPreference = 'Stop' $dir = './.artifacts/win-x64' - $exe = Join-Path $dir 'VectorNNTPD.BackFillerService.exe' - if (-not (Test-Path -LiteralPath $exe)) { throw "Missing $exe" } - foreach ($required in @('VectorNNTPD.BackFillerService.json', 'VectorNNTPD.RabbitMQ.json')) { + foreach ($exeName in @('VectorNNTPD.BackFillerService.exe', 'VectorNNTPD.StorageService.exe')) { + $exe = Join-Path $dir $exeName + if (-not (Test-Path -LiteralPath $exe)) { throw "Missing $exe" } + $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path $exe)) + if ($bytes.Length -lt 512 -or $bytes[0] -ne 0x4D -or $bytes[1] -ne 0x5A) { throw "Expected a Windows PE executable: $exe" } + $peOffset = [System.BitConverter]::ToInt32($bytes, 0x3C) + if ($bytes[$peOffset] -ne 0x50 -or $bytes[$peOffset + 1] -ne 0x45) { throw "Missing PE signature: $exe" } + $machine = [System.BitConverter]::ToUInt16($bytes, $peOffset + 4) + if ($machine -ne 0x8664) { throw "Expected x64 machine 0x8664 and found 0x$('{0:X4}' -f $machine): $exe" } + $optional = $peOffset + 24 + $magic = [System.BitConverter]::ToUInt16($bytes, $optional) + if ($magic -ne 0x20B) { throw "Expected PE32+ and found magic 0x$('{0:X4}' -f $magic): $exe" } + $clrSize = [System.BitConverter]::ToUInt32($bytes, $optional + 112 + (14 * 8) + 4) + if ($clrSize -ne 0) { throw "Executable still has a CLR runtime header, so it is not Native AOT: $exe" } + } + foreach ($required in @('VectorNNTPD.BackFillerService.json', 'VectorNNTPD.StorageService.json', 'VectorNNTPD.RabbitMQ.json')) { $path = Join-Path $dir $required if (-not (Test-Path -LiteralPath $path)) { throw "Missing $path" } } - foreach ($forbidden in @('VectorNNTPD.BackFillerService.dll', 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', 'vectornntp-backfiller.service', 'backfiller.env.example', 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json')) { + foreach ($forbidden in @( + 'VectorNNTPD.BackFillerService.dll', + 'VectorNNTPD.StorageService.dll', + 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', + 'vectornntp-backfiller.service', 'backfiller.env.example', + 'vectornntp-storageservice.service', 'storageservice.env.example', + 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json', 'VectorNNTP.StorageServer.json', + 'VectorNNTPD.StorageService.Development.json', 'VectorNNTPD.BackFillerService.Development.json' + )) { $path = Join-Path $dir $forbidden if (Test-Path -LiteralPath $path) { throw "Native AOT output contains $forbidden." } } - $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path $exe)) - if ($bytes.Length -lt 512 -or $bytes[0] -ne 0x4D -or $bytes[1] -ne 0x5A) { throw "Expected a Windows PE executable." } - $peOffset = [System.BitConverter]::ToInt32($bytes, 0x3C) - if ($bytes[$peOffset] -ne 0x50 -or $bytes[$peOffset + 1] -ne 0x45) { throw "Missing PE signature." } - $machine = [System.BitConverter]::ToUInt16($bytes, $peOffset + 4) - if ($machine -ne 0x8664) { throw "Expected x64 machine 0x8664 and found 0x$('{0:X4}' -f $machine)." } - $optional = $peOffset + 24 - $magic = [System.BitConverter]::ToUInt16($bytes, $optional) - if ($magic -ne 0x20B) { throw "Expected PE32+ and found magic 0x$('{0:X4}' -f $magic)." } - $clrSize = [System.BitConverter]::ToUInt32($bytes, $optional + 112 + (14 * 8) + 4) - if ($clrSize -ne 0) { throw "Executable still has a CLR runtime header, so it is not Native AOT." } - - - name: Smoke-test win-x64 Native AOT startup + + - name: Smoke-test win-x64 Native AOT startup (BackFiller) shell: pwsh run: | $ErrorActionPreference = 'Stop' $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.BackFillerService.exe' $config = (Resolve-Path './tests/VectorNNTPD.BackFillerService.Tests/Fixtures/startup-smoke.json').Path - $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-smoke-stdout.txt' - $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-smoke-stderr.txt' + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stderr.txt' $process = [System.Diagnostics.Process]::new() $process.StartInfo.FileName = $exe $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe @@ -421,32 +493,76 @@ jobs: $process.StartInfo.RedirectStandardError = $true $process.StartInfo.Environment['VECTORNNTP_BACKFILLER_STARTUP_SMOKE'] = '1' $process.StartInfo.Environment['VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG'] = $config - if (-not $process.Start()) { throw 'Failed to start the win-x64 startup smoke process.' } + if (-not $process.Start()) { throw 'Failed to start the win-x64 BackFiller startup smoke process.' } $stdoutTask = $process.StandardOutput.ReadToEndAsync() $stderrTask = $process.StandardError.ReadToEndAsync() if (-not $process.WaitForExit(30000)) { try { $process.Kill($true) } catch { } - throw 'win-x64 startup smoke timed out.' + throw 'win-x64 BackFiller startup smoke timed out.' } $process.WaitForExit() $stdout = $stdoutTask.GetAwaiter().GetResult() $stderr = $stderrTask.GetAwaiter().GetResult() Set-Content -LiteralPath $stdoutPath -Value $stdout Set-Content -LiteralPath $stderrPath -Value $stderr - Write-Host "win-x64 startup smoke exit=$($process.ExitCode)" + Write-Host "win-x64 BackFiller startup smoke exit=$($process.ExitCode)" Write-Host '----- stdout -----' Write-Host $stdout Write-Host '----- stderr -----' Write-Host $stderr - if ($process.ExitCode -ne 0) { throw 'win-x64 startup smoke failed.' } + if ($process.ExitCode -ne 0) { throw 'win-x64 BackFiller startup smoke failed.' } if ($stdout -notmatch 'BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01\.usenet\.ninja') { - throw 'win-x64 startup smoke did not reach the configuration checkpoint.' + throw 'win-x64 BackFiller startup smoke did not reach the configuration checkpoint.' + } + + - name: Smoke-test win-x64 Native AOT startup (StorageService) + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.StorageService.exe' + $config = (Resolve-Path './tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json').Path + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stderr.txt' + $process = [System.Diagnostics.Process]::new() + $process.StartInfo.FileName = $exe + $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe + $process.StartInfo.UseShellExecute = $false + $process.StartInfo.RedirectStandardOutput = $true + $process.StartInfo.RedirectStandardError = $true + $process.StartInfo.Environment['VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE'] = '1' + $process.StartInfo.Environment['VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG'] = $config + if (-not $process.Start()) { throw 'Failed to start the win-x64 StorageService startup smoke process.' } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(30000)) { + try { $process.Kill($true) } catch { } + throw 'win-x64 StorageService startup smoke timed out.' + } + $process.WaitForExit() + $stdout = $stdoutTask.GetAwaiter().GetResult() + $stderr = $stderrTask.GetAwaiter().GetResult() + Set-Content -LiteralPath $stdoutPath -Value $stdout + Set-Content -LiteralPath $stderrPath -Value $stderr + Write-Host "win-x64 StorageService startup smoke exit=$($process.ExitCode)" + Write-Host '----- stdout -----' + Write-Host $stdout + Write-Host '----- stderr -----' + Write-Host $stderr + if ($process.ExitCode -ne 0) { throw 'win-x64 StorageService startup smoke failed.' } + if ($stdout -notmatch 'StorageService startup smoke: configuration serverId=1 fqdn=cache01\.usenet\.ninja') { + throw 'win-x64 StorageService startup smoke did not reach the configuration checkpoint.' + } + if ($stdout -notmatch 'Application logging initialized') { + throw 'win-x64 StorageService startup smoke did not initialize logging.' + } + if ($stdout -notmatch 'VectorNNTPD\.StorageService host starting') { + throw 'win-x64 StorageService startup smoke did not log host starting.' } - name: Upload Windows build artifact uses: actions/upload-artifact@v7 with: - name: backfiller-win-x64-ci-build + name: vectornntpd-win-x64-ci-build path: .artifacts/win-x64/ if-no-files-found: error retention-days: 7 diff --git a/Directory.Build.targets b/Directory.Build.targets index cfa3624d..849cc76d 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -11,7 +11,7 @@ - + ©2026 Chris Knipe <cknipe@opticnetworks.net> @@ -22,7 +22,7 @@ --> + Condition="'$(DesignTimeBuild)' != 'true' AND ('$(MSBuildProjectName)' == 'VectorNNTPD.BackFillerService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.StorageService')"> <_ApplicationVersionNow>$([System.DateTime]::Now.Ticks) <_ApplicationVersionBuild>$([System.DateTime]::new($([System.Int64]::Parse($(_ApplicationVersionNow)))).Subtract($([System.DateTime]::new(2000,1,1))).Days) @@ -38,21 +38,36 @@ Production publish is self-contained Native AOT for win-x64 and linux-x64 only. A normal build does not set RuntimeIdentifier, so libraries, tests, and tools stay portable. --> - + win-x64;linux-x64 true true + + + $(NoWarn);IL2104;IL3050;IL3000;IL3002 + $(WarningsNotAsErrors);IL2104;IL3050 + false + false + + + Text="VectorNNTPD builds only for Platform x64. Platform '$(Platform)' is not supported." /> + Condition="('$(MSBuildProjectName)' == 'VectorNNTPD.BackFillerService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.StorageService') AND '$(RuntimeIdentifier)' != '' AND '$(RuntimeIdentifier)' != 'win-x64' AND '$(RuntimeIdentifier)' != 'linux-x64'"> diff --git a/README.md b/README.md index 659cbaac..b92585e1 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ -# BackFiller (Standalone) +# VectorNNTPD -Standalone migration of `VectorNNTPD.BackFillerService` into an independent repository. +Standalone VectorNNTPD hosts extracted from VectorNNTP.NNTPD. This repository currently ships BackFiller and StorageService, plus shared `VectorNNTPD.Common`. ## Structure -- `src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj` - standalone BackFiller worker app (net10.0) -- `src/VectorNNTPD.BackFillerService/` - production code organized by responsibility (for example `Acme`, `Articles`, `Configuration`, `Core`, `Hosting`, `Nntp`, `Transport`) -- `tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj` - xUnit test suite, namespace `VectorNNTPD.BackFillerService.Tests` -- `VectorNNTPD.slnx` - solution including app + tests +- `src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj` - BackFiller worker app (net10.0) +- `src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj` - StorageService / VATP cache host (net10.0) +- `src/VectorNNTPD.Common/VectorNNTPD.Common.csproj` - shared library +- `tests/VectorNNTPD.BackFillerService.Tests/` / `tests/VectorNNTPD.StorageService.Tests/` / `tests/VectorNNTPD.Common.Tests/` - xUnit suites +- `VectorNNTPD.slnx` - solution including apps + tests +- `tools/Publish-Release.ps1` - Native AOT release publish for both production hosts - `tools/Invoke-CodeQlAnalysis.ps1` - local traced C# CodeQL analysis ## Build @@ -24,21 +26,27 @@ dotnet test .\tests\VectorNNTPD.BackFillerService.Tests\VectorNNTPD.BackFillerSe ## Publish -Windows x64: +Preferred release path (both hosts into one flat directory): ```powershell -dotnet publish .\src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj -c Release -p:Platform=x64 -r win-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true +.\tools\Publish-Release.ps1 -Platform win-64 +.\tools\Publish-Release.ps1 -Platform linux-64 ``` -Linux x64: +Per-project Native AOT publish still works. Example for StorageService: ```powershell -dotnet publish .\src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj -c Release -p:Platform=x64 -r linux-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true +dotnet publish .\src\VectorNNTPD.StorageService\VectorNNTPD.StorageService.csproj -c Release -p:Platform=x64 -r win-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true ``` -The production project references the `StuDev.AotAnywhere` 1.0.4 MSBuild SDK. On Windows that SDK links `linux-x64` with the Zig toolchain restored by NuGet. A globalization-enabled Linux host still needs `libicu`. +Production hosts reference the `StuDev.AotAnywhere` 1.0.4 MSBuild SDK. On Windows that SDK links `linux-x64` with the Zig toolchain restored by NuGet. A globalization-enabled Linux host still needs `libicu`. -Native AOT publish keeps trim and AOT analysis enabled. CI then runs the published executable with `VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1` and `VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG` pointing at an isolated JSON file. The process must print `BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja` and exit 0. That checkpoint is configuration binding and host construction. It does not open a listener or contact a database, broker, DNS provider, or certificate authority. +Native AOT publish keeps trim and AOT analysis enabled. CI runs each published executable against an isolated JSON fixture: + +- BackFiller: `VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1` / `VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG` → checkpoint `BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja` +- StorageService: `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE=1` / `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG` → checkpoint `StorageService startup smoke: configuration serverId=1 fqdn=cache01.usenet.ninja` + +Those checkpoints prove configuration binding and host construction (and StorageService logging initialization). They do not open a listener or contact a database, broker, DNS provider, or certificate authority. StorageService AOT publish still scopes Serilog.Settings.Configuration IL suppressions (`IL2104`/`IL3050`/`IL3000`/`IL3002`) because MinimumLevel is applied via `ReadFrom.Configuration` while sinks are registered explicitly. BackFiller references `VectorNNTPD.Common` for `IApplicationService`, NntpDB, ACME certificate management, the Cloudflare DNS API client, authoritative DNS queries used for DNS-01, RabbitMQ connectivity, and article parsing and validation. It keeps its own `ApplicationServiceManager` and does not use Common's `ApplicationLifecycle`. BackFiller still owns certificate watchdog health, the decision to publish listener-bound A/AAAA records only after the cache listener binds, Article Work topology, the RabbitMQ hosted adapter, and the retained article image that dot-stuffs body lines after Common's parser accepts the article. @@ -48,9 +56,11 @@ BackFiller references `VectorNNTPD.Common` for `IApplicationService`, NntpDB, AC ## Linux service -`tools/Publish-Release.ps1 -Platform linux-64` writes `release/linux-64/`. The Linux CI job writes `.artifacts/linux-x64/` and uploads that directory. Both contain `VectorNNTPD.BackFillerService`, `VectorNNTPD.BackFillerService.json`, `VectorNNTPD.RabbitMQ.json`, `README.md`, `vectornntp-backfiller.service`, and `backfiller.env.example`. +`tools/Publish-Release.ps1 -Platform linux-64` writes `release/linux-64/`. The Linux CI job writes `.artifacts/linux-x64/` and uploads that directory. Both contain the flat Native AOT outputs for `VectorNNTPD.BackFillerService` and `VectorNNTPD.StorageService`, their primary JSON files, shared `VectorNNTPD.RabbitMQ.json`, `README.md`, and the BackFiller and StorageService systemd units plus environment-file examples. + +Install BackFiller at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. -Install that directory at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. +Install StorageService at `/opt/vectornntp/storageservice` using `VectorNNTPD.StorageService`, `VectorNNTPD.StorageService.json`, `VectorNNTPD.RabbitMQ.json`, `vectornntp-storageservice.service`, and `storageservice.env.example` from the same release directory. That unit’s `ReadWritePaths` also include `db/` and `spool/` for the article index and segment-pool roots. Create the account and install the release as root-owned files. This step copies the placeholder environment file and reloads systemd. It does not start the service. diff --git a/VectorNNTPD.sln.DotSettings b/VectorNNTPD.sln.DotSettings index 4bbb1760..88332f6c 100644 --- a/VectorNNTPD.sln.DotSettings +++ b/VectorNNTPD.sln.DotSettings @@ -1,4 +1,5 @@  True + True True True \ No newline at end of file diff --git a/VectorNNTPD.slnx b/VectorNNTPD.slnx index 5ab1b747..ae1f5980 100644 --- a/VectorNNTPD.slnx +++ b/VectorNNTPD.slnx @@ -9,6 +9,9 @@ + + + @@ -17,5 +20,9 @@ + + + + diff --git a/deploy/systemd/storageservice.env.example b/deploy/systemd/storageservice.env.example new file mode 100644 index 00000000..5779a1e3 --- /dev/null +++ b/deploy/systemd/storageservice.env.example @@ -0,0 +1,16 @@ +# systemd EnvironmentFile for vectornntp-storageservice.service. +# One KEY=VALUE per line. A line whose first non-space character is # is a comment. +# systemd parses this file itself. source does not reproduce that parse. +# An unquoted backslash escapes the next character. \\ is one backslash. +# Confirm a value that contains a backslash, a quote, or leading or trailing whitespace on the target host. +# Copy to /etc/vectornntp/storageservice.env as root:root mode 0600, then replace the placeholder with sudoedit. +# Do not commit the installed file. +ConnectionStrings__NntpDB=Server=REPLACE_WITH_MYSQL_HOST;User ID=REPLACE_WITH_MYSQL_USER;Password=REPLACE_WITH_MYSQL_PASSWORD;Database=REPLACE_WITH_DATABASE_NAME +# These shared names are not required in /etc/vectornntp/storageservice.env. +# StorageService does not take them from this file. nntpsharedconfig supplies them after the connection string works. +# VECTOR__RABBITMQ__USERNAME +# VECTOR__RABBITMQ__PASSWORD +# VECTOR__CLOUDFLAREAPIKEY +# VECTOR__CLOUDFLAREZONEID +# VECTOR__ACMEACCOUNT +# VECTOR__ACMECERTIFICATEPASSWORD diff --git a/deploy/systemd/vectornntp-storageservice.service b/deploy/systemd/vectornntp-storageservice.service new file mode 100644 index 00000000..30fe9d21 --- /dev/null +++ b/deploy/systemd/vectornntp-storageservice.service @@ -0,0 +1,77 @@ +[Unit] +Description=VectorNNTP StorageService +# Install the flat linux-x64 release directory contents needed for this service +# at this path. The shared release also contains BackFiller binaries and units. +# This unit uses only VectorNNTPD.StorageService, VectorNNTPD.StorageService.json, +# VectorNNTPD.RabbitMQ.json, README.md, vectornntp-storageservice.service, and +# storageservice.env.example. +# Copy the unit to /etc/systemd/system/. Create user and group vectornntp, +# create logs/, certs/, db/, and spool/ owned by that account, and install +# /etc/vectornntp/storageservice.env before enable. +Documentation=file:///opt/vectornntp/storageservice/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +# Dedicated non-root service account (create before enabling the unit). +# Same account as BackFiller is acceptable when both share /opt/vectornntp. +User=vectornntp +Group=vectornntp + +# The executable directory is the configuration root. The process loads +# VectorNNTPD.StorageService.json and VectorNNTPD.RabbitMQ.json from that directory, +# not appsettings.json, RabbitMq.json, or the process working directory. +# Relative logs (logs/), ACME state (certs/), article index (db/), and spool +# roots also resolve there. +WorkingDirectory=/opt/vectornntp/storageservice +ExecStart=/opt/vectornntp/storageservice/VectorNNTPD.StorageService + +# Host GracefulShutdownTimeout follows StorageServer:GracefulShutdownTimeout. +# The shipped value is 00:00:30. TimeoutStopSec must stay above that setting +# or systemd SIGKILLs an in-progress drain. +TimeoutStartSec=180 +TimeoutStopSec=60 + +Restart=on-failure +RestartSec=5 + +# systemd reads this file as root before exec. The leading '-' suppresses errors +# reading the file and still starts the process. Current systemd ignores a missing, +# unreadable, or invalid file. That does not insert ConnectionStrings__NntpDB. +# Startup validation still fails when the variable is absent. The file is KEY=VALUE, +# not a shell script. An unquoted backslash is an escape. The connection string has +# no VECTOR__ prefix. RabbitMQ, ACME, and Cloudflare secrets come from +# nntpsharedconfig; this unit does not set them. +EnvironmentFile=-/etc/vectornntp/storageservice.env +# DOTNET_ENVIRONMENT selects the optional VectorNNTPD.StorageService.Production.json overlay. +Environment=DOTNET_ENVIRONMENT=Production + +# Watchdog heartbeats run only when EnableWatchdog is true (the shipped default) +# and this unit sets WatchdogSec, which supplies WATCHDOG_USEC. +# Leave WatchdogSec unset until readiness has been checked on the target host. +# Example (enable only after validation): +# WatchdogSec=30 +# Restart=on-watchdog + +# File logging is the shipped destination (StorageServer:LogDir=logs). +# The shipped VectorNNTPD.StorageService.json also enables the Console sink, so +# Serilog Information+ lines reach stdout and the journal under this unit. +# Hardening. StorageService binds TLS and opens outbound RabbitMQ, MySQL, and Cloudflare. +# ProtectSystem=strict mounts the filesystem read-only. ReadWritePaths re-opens only +# logs/, certs/, db/, and spool/. It does not grant Unix ownership; those directories +# must already exist and be writable by the vectornntp account. The executable and +# JSON stay read-only. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/vectornntp/storageservice/logs /opt/vectornntp/storageservice/certs /opt/vectornntp/storageservice/db /opt/vectornntp/storageservice/spool + +KillSignal=SIGTERM +SendSIGKILL=yes + +[Install] +WantedBy=multi-user.target diff --git a/docs/common/configuration.md b/docs/common/configuration.md index d6b47f71..6af693be 100644 --- a/docs/common/configuration.md +++ b/docs/common/configuration.md @@ -18,7 +18,7 @@ NNTPD, BackFiller, and StorageServer publish the ACME account email, ACME certif Top-level `RabbitMQ` section. The Common sample is `src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json`. `ApplicationJsonConfiguration.AddSharedRabbitMqJsonFile` loads only the filename the caller passes. BackFiller passes `VectorNNTPD.RabbitMQ.json`. The file is inserted after the entry-assembly JSON files and before environment variables and command-line arguments. Options types and connection validation live in `VectorNNTPD.Common.Messaging.RabbitMq`. Applications bind the section and call `AddRabbitMqInfrastructure()`. Nested `Management` is validated only when `BaseUrl` is set. -NNTPD and StorageServer are not projects in this repository, so their build output is not part of the contract here. BackFiller ships connectivity in `src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json`. Its primary file, `VectorNNTPD.BackFillerService.json`, still must not repeat a top-level `RabbitMQ` section. `BackFiller:Logging:RabbitMQ` is the log sink, not broker connectivity. +NNTPD is not a project in this repository, so its build output is not part of the contract here. BackFiller and StorageService ship connectivity as `VectorNNTPD.RabbitMQ.json` beside each host project. Primary host files (`VectorNNTPD.BackFillerService.json`, `VectorNNTPD.StorageService.json`) must not repeat a top-level `RabbitMQ` section. `BackFiller:Logging:RabbitMQ` is the log sink, not broker connectivity. `RabbitMqService` owns one process-wide connection, verifies that it is open, and replaces it on connectivity loss or when a broker block lasts longer than `ConnectionBlockedTimeoutSeconds`, while incrementing a monotonic generation. Client automatic recovery is disabled. The AMQP client name comes from `IRabbitMqConnectionNameProvider`. Callers obtain the current connection with `TryGetCurrent` and do not dispose it. diff --git a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj index d19de2d7..e648b0b5 100644 --- a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj +++ b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj @@ -16,8 +16,8 @@ - - + +