From 2774567e92934f0951fba1504c5b53ae7f7b3a48 Mon Sep 17 00:00:00 2001 From: "savage@savage.za.org" Date: Thu, 22 Oct 2026 18:41:39 +0200 Subject: [PATCH 01/10] Integrate StorageService into VectorNNTPD solution and release pipeline --- .github/workflows/build.yml | 266 ++- Directory.Build.targets | 25 +- README.md | 38 +- VectorNNTPD.sln.DotSettings | 1 + VectorNNTPD.slnx | 7 + deploy/systemd/storageservice.env.example | 16 + .../systemd/vectornntp-storageservice.service | 77 + docs/common/configuration.md | 2 +- .../VectorNNTPD.Common.csproj | 4 +- src/VectorNNTPD.StorageService/.editorconfig | 3 + .../Acme/AcmeCertificateApplicationService.cs | 53 + .../Configuration/SegmentPoolsOptions.cs | 63 + .../Configuration/SpoolPersistenceOptions.cs | 96 + ...orageServerAcmeCloudflareOptionsAdapter.cs | 59 + .../Configuration/StorageServerIdentity.cs | 73 + .../Configuration/StorageServerOptions.cs | 268 +++ .../StorageServerOptionsValidator.cs | 446 +++++ .../StorageServerRuntimeOptions.cs | 37 + .../StorageServerRuntimeOptionsFactory.cs | 106 ++ .../StorageServerRuntimeOptionsSource.cs | 34 + .../Hosting/StorageServerHostShutdown.cs | 94 + .../Hosting/StorageServerHostedService.cs | 158 ++ ...torageServerServiceCollectionExtensions.cs | 318 ++++ .../Hosting/StorageServerStartupSmoke.cs | 116 ++ .../Hosting/Systemd/ApplicationHealth.cs | 26 + .../Hosting/Systemd/IApplicationHealth.cs | 21 + .../Hosting/Systemd/ISystemdNotifyBridge.cs | 25 + .../Hosting/Systemd/ISystemdRuntime.cs | 35 + .../Systemd/SystemdLifecycleNotifier.cs | 195 ++ .../Hosting/Systemd/SystemdNotifyBridge.cs | 76 + .../Hosting/Systemd/SystemdRuntime.cs | 121 ++ .../Systemd/SystemdWatchdogInterval.cs | 44 + .../Hosting/Systemd/SystemdWatchdogService.cs | 166 ++ .../Ingress/AcceptedArticle.cs | 23 + .../Ingress/AcceptedArticlePathProcessor.cs | 217 +++ .../Ingress/AcceptedArticleQueue.cs | 548 ++++++ .../Ingress/AcceptedArticleWorker.cs | 513 +++++ .../AcceptedArticleWorkerLogMessages.cs | 37 + .../Ingress/IAcceptedArticleProcessor.cs | 35 + .../Ingress/IAcceptedArticleQueue.cs | 109 ++ .../Ingress/QueueReservation.cs | 14 + .../Listener/IStorageArticleOpenBoundary.cs | 78 + .../Listener/IStorageVatpTransport.cs | 322 ++++ .../Listener/StorageVatpListenerService.cs | 569 ++++++ .../Listener/StorageVatpLogMessages.cs | 141 ++ .../Listener/StorageVatpSession.Store.cs | 631 ++++++ .../Listener/StorageVatpSession.cs | 1172 ++++++++++++ .../Listener/StoreAssemblyAdmission.cs | 50 + .../Listener/ThreadCpuTimes.cs | 58 + .../Listener/VatpStoreOutcome.cs | 20 + .../Logging/HostingLogMessages.cs | 93 + .../Logging/StorageServerConfiguredSinks.cs | 421 ++++ .../Logging/StorageServerFileLogging.cs | 129 ++ .../Logging/StorageServerLogCategories.cs | 12 + .../Logging/StorageServerLoggingExtensions.cs | 149 ++ .../Logging/SystemdLogMessages.cs | 135 ++ ...ageServerSharedConfigurationLogMessages.cs | 43 + .../StorageServerSharedConfigurationReader.cs | 98 + ...StorageServerSharedConfigurationService.cs | 237 +++ src/VectorNNTPD.StorageService/Program.cs | 83 + .../Properties/launchSettings.json | 12 + .../ArticleIndex/ArticleAccessCoalescer.cs | 175 ++ .../ArticleIndex/ArticleIndexCounters.cs | 269 +++ .../ArticleIndex/ArticleIndexDatabase.cs | 892 +++++++++ .../ArticleIndexLegacyMigration.cs | 207 ++ .../ArticleIndex/ArticleIndexLogMessages.cs | 141 ++ .../ArticleIndex/ArticleIndexModels.cs | 193 ++ .../ArticleIndex/ArticleIndexService.cs | 691 +++++++ .../ArticleIndex/ArticleIndexSettings.cs | 153 ++ .../ArticleIndex/ArticleIndexWriter.cs | 900 +++++++++ .../ArticleIngestionReservations.cs | 107 ++ .../ArticleIndex/ArticlePayloadProbe.cs | 49 + .../ArticleIndex/ArticlePresenceIndex.cs | 420 ++++ .../ArticleIndex/ArticleSpoolRegistry.cs | 248 +++ .../Storage/ArticleIndex/IArticleIndex.cs | 117 ++ .../SegmentPools/BlockDeviceCounters.cs | 44 + .../SegmentPools/BlockDeviceMonitorService.cs | 415 ++++ .../Storage/SegmentPools/BlockDeviceRates.cs | 258 +++ .../SegmentPools/FileSegmentPoolWriteProbe.cs | 96 + .../SegmentPools/IBlockDeviceCounterSource.cs | 16 + .../Storage/SegmentPools/ILinuxSysfs.cs | 34 + .../SegmentPools/ISegmentPoolInspector.cs | 21 + .../SegmentPools/ISegmentPoolWriteProbe.cs | 17 + .../SegmentPools/IWindowsStorageQuery.cs | 41 + .../SegmentPools/LinuxBlockDeviceIdentity.cs | 256 +++ .../Storage/SegmentPools/LinuxDiskstats.cs | 137 ++ .../Storage/SegmentPools/LinuxMountTable.cs | 146 ++ .../Storage/SegmentPools/LinuxSysfsReader.cs | 100 + .../PlatformBlockDeviceCounterSource.cs | 95 + .../PlatformSegmentPoolInspector.cs | 264 +++ .../SegmentPools/SegmentPoolCatalog.cs | 27 + .../SegmentPools/SegmentPoolInspection.cs | 22 + .../SegmentPools/SegmentPoolLogMessages.cs | 54 + .../Storage/SegmentPools/SegmentPoolPaths.cs | 53 + .../SegmentPoolValidationService.cs | 125 ++ .../SegmentPools/SpoolObservationSnapshot.cs | 31 + .../WindowsDiskPerformanceReader.cs | 166 ++ .../SegmentPools/WindowsStorageNative.cs | 490 +++++ .../SpoolPersistence/FileSpoolArticleFiles.cs | 121 ++ .../FilesystemSpoolFreeSpaceProbe.cs | 94 + .../SpoolPersistence/ISpoolArticleFiles.cs | 54 + .../SpoolArticleIdentityGate.cs | 156 ++ .../SpoolArticlePersistenceProcessor.cs | 443 +++++ .../SpoolPersistence/SpoolArticlePublisher.cs | 227 +++ .../SpoolPersistence/SpoolCandidate.cs | 37 + .../SpoolPersistence/SpoolCandidateBuilder.cs | 142 ++ .../Storage/SpoolPersistence/SpoolHealth.cs | 108 ++ .../SpoolPersistenceCounters.cs | 76 + .../SpoolPersistenceLogMessages.cs | 70 + .../Storage/SpoolPersistence/SpoolPressure.cs | 21 + .../SpoolPersistence/SpoolSelectionPolicy.cs | 64 + .../Storage/SpoolPersistence/SpoolSelector.cs | 240 +++ .../SpoolPersistence/SpoolWriteGate.cs | 100 + .../FileSpoolShardDirectorySystem.cs | 98 + .../SpoolShards/ISpoolShardDirectorySystem.cs | 26 + .../Storage/SpoolShards/SpoolArticlePath.cs | 165 ++ .../Storage/SpoolShards/SpoolCapacity.cs | 94 + .../SpoolShardInitializationService.cs | 309 +++ .../Storage/SpoolShards/SpoolShardLayout.cs | 90 + .../SpoolShards/SpoolShardLogMessages.cs | 27 + .../Storage/StorageCapacity.cs | 29 + .../StorageServerAdvertisementLogMessages.cs | 37 + ...rageServerAdvertisementPublisherService.cs | 375 ++++ .../VectorNNTPD.RabbitMQ.json | 36 + ...ectorNNTPD.StorageService.Development.json | 2 + .../VectorNNTPD.StorageService.csproj | 62 + .../VectorNNTPD.StorageService.json | 82 + .../.editorconfig | 3 + .../ApplicationVersionTests.cs | 49 + .../StorageServerCloudflareHostnameTests.cs | 87 + .../ArticleIndexConfigurationTests.cs | 163 ++ .../Configuration/SegmentPoolsOptionsTests.cs | 242 +++ .../SpoolPersistenceOptionsTests.cs | 35 + ...torageServerBindAddressPropagationTests.cs | 318 ++++ .../StorageServerJsonStorageRemovalTests.cs | 32 + .../StorageServerOptionsTests.cs | 180 ++ .../Fixtures/SerilogCollection.cs | 12 + .../Fixtures/StorageServerTestOptions.cs | 136 ++ .../Fixtures/startup-smoke.json | 73 + .../GlobalUsings.cs | 1 + .../StorageDnsPublicationRollbackTests.cs | 343 ++++ .../StorageServerFleetLifecycleOrderTests.cs | 387 ++++ .../StorageServerHostCompositionTests.cs | 319 ++++ .../Hosting/StorageServerStartupSmokeTests.cs | 202 ++ .../Ingress/AcceptedArticleQueueTests.cs | 400 ++++ .../Ingress/AcceptedArticleWorkerTests.cs | 255 +++ .../Ingress/AdaptiveArticleWorkerTests.cs | 302 +++ .../DirectStoragePlacementPathTests.cs | 108 ++ .../Lifecycle/StorageServerLifecycleTests.cs | 164 ++ .../Listener/ArticlePlacementClientBench.cs | 212 +++ .../StorageVatpListenerServiceTests.cs | 188 ++ .../StorageVatpLiveBackpressureTests.cs | 1025 ++++++++++ .../Listener/StorageVatpQueueIngressTests.cs | 872 +++++++++ .../StorageVatpSingleConnectionBench.cs | 925 +++++++++ .../Listener/StorageVatpThroughputBench.cs | 1285 +++++++++++++ .../StorageVatpTransportTimeoutTests.cs | 187 ++ .../Logging/FileLoggingTests.cs | 332 ++++ .../Logging/SerilogLoggingTests.cs | 238 +++ .../NntpDb/StorageServerNntpDbTests.cs | 296 +++ .../Storage/ArticleIndexBenchTests.cs | 494 +++++ .../Storage/ArticleIndexPersistenceTests.cs | 390 ++++ .../Storage/ArticleIndexTestSupport.cs | 113 ++ .../Storage/ArticleIndexTests.cs | 1693 +++++++++++++++++ .../Storage/BlockDeviceMonitorTests.cs | 683 +++++++ .../Storage/SegmentPoolValidationTests.cs | 549 ++++++ .../Storage/SpoolArticleIdentityGateTests.cs | 76 + .../Storage/SpoolArticlePathTests.cs | 97 + .../Storage/SpoolArticlePublisherTests.cs | 400 ++++ .../Storage/SpoolCapacityTests.cs | 122 ++ .../Storage/SpoolPersistenceBenchTests.cs | 916 +++++++++ .../SpoolPersistenceIntegrationTests.cs | 713 +++++++ .../Storage/SpoolSelectorTests.cs | 209 ++ .../Storage/SpoolShardInitializationTests.cs | 344 ++++ ...erverAdvertisementPublisherServiceTests.cs | 340 ++++ .../Support/Placement/ArticlePlacement.cs | 146 ++ .../Placement/ArticlePlacementClient.cs | 666 +++++++ .../Placement/ArticlePlacementConnection.cs | 901 +++++++++ .../ArticlePlacementTransportLogMessages.cs | 63 + .../Placement/IArticlePlacementClient.cs | 18 + .../Placement/StorageServerFleetEntry.cs | 57 + .../Support/Placement/VatpClientOptions.cs | 22 + .../Support/Placement/VatpTlsClient.cs | 56 + .../FailingSegmentPoolInspector.cs | 18 + .../TestDoubles/FakeCloudflareDnsClient.cs | 339 ++++ .../TestDoubles/FakeNntpDb.cs | 76 + .../TestDoubles/FakeStorageServerRabbitMq.cs | 231 +++ .../MemorySpoolShardDirectorySystem.cs | 79 + .../PermissiveSegmentPoolInspector.cs | 23 + .../RecordingStorageServerRabbitMq.cs | 370 ++++ .../TestDoubles/SegmentPoolTestSupport.cs | 65 + .../TestDoubles/TestDoubles.cs | 160 ++ .../VectorNNTPD.StorageService.Tests.csproj | 41 + tools/Publish-Release.ps1 | 215 ++- 193 files changed, 39750 insertions(+), 179 deletions(-) create mode 100644 deploy/systemd/storageservice.env.example create mode 100644 deploy/systemd/vectornntp-storageservice.service create mode 100644 src/VectorNNTPD.StorageService/.editorconfig create mode 100644 src/VectorNNTPD.StorageService/Acme/AcmeCertificateApplicationService.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/SegmentPoolsOptions.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/SpoolPersistenceOptions.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerAcmeCloudflareOptionsAdapter.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerIdentity.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerOptions.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerOptionsValidator.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptions.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsFactory.cs create mode 100644 src/VectorNNTPD.StorageService/Configuration/StorageServerRuntimeOptionsSource.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/StorageServerHostShutdown.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/StorageServerHostedService.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/StorageServerServiceCollectionExtensions.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/StorageServerStartupSmoke.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/ApplicationHealth.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/IApplicationHealth.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/ISystemdNotifyBridge.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/ISystemdRuntime.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/SystemdLifecycleNotifier.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/SystemdNotifyBridge.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/SystemdRuntime.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/SystemdWatchdogInterval.cs create mode 100644 src/VectorNNTPD.StorageService/Hosting/Systemd/SystemdWatchdogService.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/AcceptedArticle.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/AcceptedArticlePathProcessor.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/AcceptedArticleQueue.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorker.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/AcceptedArticleWorkerLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleProcessor.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/IAcceptedArticleQueue.cs create mode 100644 src/VectorNNTPD.StorageService/Ingress/QueueReservation.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/IStorageArticleOpenBoundary.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/IStorageVatpTransport.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/StorageVatpListenerService.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/StorageVatpLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/StorageVatpSession.Store.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/StorageVatpSession.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/StoreAssemblyAdmission.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/ThreadCpuTimes.cs create mode 100644 src/VectorNNTPD.StorageService/Listener/VatpStoreOutcome.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/HostingLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/StorageServerConfiguredSinks.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/StorageServerFileLogging.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/StorageServerLogCategories.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/StorageServerLoggingExtensions.cs create mode 100644 src/VectorNNTPD.StorageService/Logging/SystemdLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/NntpDb/StorageServerSharedConfigurationLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/NntpDb/StorageServerSharedConfigurationReader.cs create mode 100644 src/VectorNNTPD.StorageService/NntpDb/StorageServerSharedConfigurationService.cs create mode 100644 src/VectorNNTPD.StorageService/Program.cs create mode 100644 src/VectorNNTPD.StorageService/Properties/launchSettings.json create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleAccessCoalescer.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexCounters.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexDatabase.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLegacyMigration.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexModels.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexService.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexSettings.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIndexWriter.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleIngestionReservations.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePayloadProbe.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticlePresenceIndex.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/ArticleSpoolRegistry.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/ArticleIndex/IArticleIndex.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceCounters.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceMonitorService.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/BlockDeviceRates.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/FileSegmentPoolWriteProbe.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/IBlockDeviceCounterSource.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/ILinuxSysfs.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/ISegmentPoolInspector.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/ISegmentPoolWriteProbe.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/IWindowsStorageQuery.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/LinuxBlockDeviceIdentity.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/LinuxDiskstats.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/LinuxMountTable.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/LinuxSysfsReader.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/PlatformBlockDeviceCounterSource.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/PlatformSegmentPoolInspector.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolCatalog.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolInspection.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolPaths.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SegmentPoolValidationService.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/SpoolObservationSnapshot.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/WindowsDiskPerformanceReader.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SegmentPools/WindowsStorageNative.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/FileSpoolArticleFiles.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/FilesystemSpoolFreeSpaceProbe.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/ISpoolArticleFiles.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticleIdentityGate.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePersistenceProcessor.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolArticlePublisher.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolCandidate.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolCandidateBuilder.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolHealth.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceCounters.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPersistenceLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolPressure.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelectionPolicy.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolSelector.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolPersistence/SpoolWriteGate.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/FileSpoolShardDirectorySystem.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/ISpoolShardDirectorySystem.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/SpoolArticlePath.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/SpoolCapacity.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/SpoolShardInitializationService.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/SpoolShardLayout.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/SpoolShards/SpoolShardLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/StorageCapacity.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/StorageServerAdvertisementLogMessages.cs create mode 100644 src/VectorNNTPD.StorageService/Storage/StorageServerAdvertisementPublisherService.cs create mode 100644 src/VectorNNTPD.StorageService/VectorNNTPD.RabbitMQ.json create mode 100644 src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.Development.json create mode 100644 src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj create mode 100644 src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.json create mode 100644 tests/VectorNNTPD.StorageService.Tests/.editorconfig create mode 100644 tests/VectorNNTPD.StorageService.Tests/ApplicationVersionTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Cloudflare/StorageServerCloudflareHostnameTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/ArticleIndexConfigurationTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/SegmentPoolsOptionsTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/SpoolPersistenceOptionsTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/StorageServerBindAddressPropagationTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/StorageServerJsonStorageRemovalTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Configuration/StorageServerOptionsTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Fixtures/SerilogCollection.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Fixtures/StorageServerTestOptions.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json create mode 100644 tests/VectorNNTPD.StorageService.Tests/GlobalUsings.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Hosting/StorageDnsPublicationRollbackTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerFleetLifecycleOrderTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerHostCompositionTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Hosting/StorageServerStartupSmokeTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleQueueTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Ingress/AcceptedArticleWorkerTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Ingress/AdaptiveArticleWorkerTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Ingress/DirectStoragePlacementPathTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Lifecycle/StorageServerLifecycleTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/ArticlePlacementClientBench.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpListenerServiceTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpLiveBackpressureTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpQueueIngressTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpSingleConnectionBench.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpThroughputBench.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Listener/StorageVatpTransportTimeoutTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Logging/FileLoggingTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Logging/SerilogLoggingTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/NntpDb/StorageServerNntpDbTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexBenchTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexPersistenceTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexTestSupport.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/ArticleIndexTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/BlockDeviceMonitorTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SegmentPoolValidationTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticleIdentityGateTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticlePathTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolArticlePublisherTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolCapacityTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceBenchTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolPersistenceIntegrationTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolSelectorTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/SpoolShardInitializationTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Storage/StorageServerAdvertisementPublisherServiceTests.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacement.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementClient.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementConnection.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/ArticlePlacementTransportLogMessages.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/IArticlePlacementClient.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/StorageServerFleetEntry.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/VatpClientOptions.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/Support/Placement/VatpTlsClient.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/FailingSegmentPoolInspector.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/FakeCloudflareDnsClient.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/FakeNntpDb.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/FakeStorageServerRabbitMq.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/MemorySpoolShardDirectorySystem.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/PermissiveSegmentPoolInspector.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/RecordingStorageServerRabbitMq.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/SegmentPoolTestSupport.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/TestDoubles/TestDoubles.cs create mode 100644 tests/VectorNNTPD.StorageService.Tests/VectorNNTPD.StorageService.Tests.csproj diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee25567f..ffffe185 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -231,19 +231,26 @@ jobs: if-no-files-found: error retention-days: 14 - - name: Publish BackFiller linux-x64 - # Same Native AOT invocation as the VectorNNTP release script, plus the + - name: Publish production services linux-x64 + # Same Native AOT invocation as tools/Publish-Release.ps1, plus the # x64 platform contract. Publish restores the ILCompiler pack and the # StuDev.AotAnywhere Zig toolchain for this runtime identifier. + # Both services share one flat artifact directory (identical RabbitMQ JSON). run: | - dotnet publish src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj \ - --configuration Release \ - -p:Platform=x64 \ - --runtime linux-x64 \ - --self-contained true \ - -p:PublishAot=true \ - -p:SelfContained=true \ - --output ./.artifacts/linux-x64 + set -euo pipefail + for proj in \ + src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj \ + src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj + do + dotnet publish "$proj" \ + --configuration Release \ + -p:Platform=x64 \ + --runtime linux-x64 \ + --self-contained true \ + -p:PublishAot=true \ + -p:SelfContained=true \ + --output ./.artifacts/linux-x64 + done - name: Add Linux deployment files shell: bash @@ -252,55 +259,71 @@ jobs: dir=./.artifacts/linux-x64 test -f deploy/systemd/vectornntp-backfiller.service test -f deploy/systemd/backfiller.env.example + test -f deploy/systemd/vectornntp-storageservice.service + test -f deploy/systemd/storageservice.env.example test -f README.md cp deploy/systemd/vectornntp-backfiller.service "$dir/vectornntp-backfiller.service" cp deploy/systemd/backfiller.env.example "$dir/backfiller.env.example" + cp deploy/systemd/vectornntp-storageservice.service "$dir/vectornntp-storageservice.service" + cp deploy/systemd/storageservice.env.example "$dir/storageservice.env.example" cp README.md "$dir/README.md" test -f "$dir/VectorNNTPD.BackFillerService.json" + test -f "$dir/VectorNNTPD.StorageService.json" test -f "$dir/VectorNNTPD.RabbitMQ.json" test ! -e "$dir/VectorNNTP.RabbitMQ.json" + test ! -e "$dir/VectorNNTP.StorageServer.json" test ! -e "$dir/appsettings.json" test ! -e "$dir/RabbitMq.json" - - name: Validate linux-x64 Native AOT executable + - name: Validate linux-x64 Native AOT executables shell: bash run: | set -euo pipefail dir=./.artifacts/linux-x64 - exe="$dir/VectorNNTPD.BackFillerService" - test -f "$exe" - test -x "$exe" + for exe_name in VectorNNTPD.BackFillerService VectorNNTPD.StorageService; do + exe="$dir/$exe_name" + test -f "$exe" + test -x "$exe" + file "$exe" + readelf -h "$exe" + objdump -f "$exe" + file -b "$exe" | grep -q "ELF 64-bit LSB" + file -b "$exe" | grep -q "x86-64" + readelf -h "$exe" | grep -q "Class:[[:space:]]*ELF64" + readelf -h "$exe" | grep -q "Data:[[:space:]]*2's complement, little endian" + readelf -h "$exe" | grep -q "Machine:[[:space:]]*Advanced Micro Devices X86-64" + readelf -h "$exe" | grep -E -q "Type:[[:space:]]*(EXEC|DYN)" + objdump -f "$exe" | grep -q "elf64-x86-64" + if ldd "$exe" | grep -E -q 'coreclr|hostfxr'; then + echo "Executable depends on the shared CLR: $exe" + ldd "$exe" + exit 1 + fi + done test -f "$dir/vectornntp-backfiller.service" test -f "$dir/backfiller.env.example" + test -f "$dir/vectornntp-storageservice.service" + test -f "$dir/storageservice.env.example" test -f "$dir/README.md" test -f "$dir/VectorNNTPD.BackFillerService.json" + test -f "$dir/VectorNNTPD.StorageService.json" test -f "$dir/VectorNNTPD.RabbitMQ.json" test ! -e "$dir/VectorNNTP.RabbitMQ.json" + test ! -e "$dir/VectorNNTP.StorageServer.json" test ! -e "$dir/appsettings.json" test ! -e "$dir/RabbitMq.json" - file "$exe" - readelf -h "$exe" - objdump -f "$exe" - file -b "$exe" | grep -q "ELF 64-bit LSB" - file -b "$exe" | grep -q "x86-64" - readelf -h "$exe" | grep -q "Class:[[:space:]]*ELF64" - readelf -h "$exe" | grep -q "Data:[[:space:]]*2's complement, little endian" - readelf -h "$exe" | grep -q "Machine:[[:space:]]*Advanced Micro Devices X86-64" - readelf -h "$exe" | grep -E -q "Type:[[:space:]]*(EXEC|DYN)" - objdump -f "$exe" | grep -q "elf64-x86-64" - for forbidden in VectorNNTPD.BackFillerService.dll libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll; do + for forbidden in \ + VectorNNTPD.BackFillerService.dll \ + VectorNNTPD.StorageService.dll \ + libcoreclr.so libhostfxr.so coreclr.dll hostfxr.dll + do if [ -e "$dir/$forbidden" ]; then echo "Native AOT output contains $forbidden." exit 1 fi done - if ldd "$exe" | grep -E -q 'coreclr|hostfxr'; then - echo "Executable depends on the shared CLR." - ldd "$exe" - exit 1 - fi - - name: Smoke-test linux-x64 Native AOT startup + - name: Smoke-test linux-x64 Native AOT startup (BackFiller) shell: bash run: | set -euo pipefail @@ -311,28 +334,59 @@ jobs: export VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1 export VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG="$config" set +e - timeout 30 "$exe" >./.artifacts/linux-x64-smoke-stdout.txt 2>./.artifacts/linux-x64-smoke-stderr.txt + timeout 30 "$exe" >./.artifacts/linux-x64-backfiller-smoke-stdout.txt 2>./.artifacts/linux-x64-backfiller-smoke-stderr.txt code=$? set -e - echo "linux-x64 startup smoke exit=$code" + echo "linux-x64 BackFiller startup smoke exit=$code" echo "----- stdout -----" - cat ./.artifacts/linux-x64-smoke-stdout.txt + cat ./.artifacts/linux-x64-backfiller-smoke-stdout.txt echo "----- stderr -----" - cat ./.artifacts/linux-x64-smoke-stderr.txt + cat ./.artifacts/linux-x64-backfiller-smoke-stderr.txt if [ "$code" -eq 124 ]; then - echo "linux-x64 startup smoke timed out." + echo "linux-x64 BackFiller startup smoke timed out." exit 1 fi if [ "$code" -ne 0 ]; then - echo "linux-x64 startup smoke failed." + echo "linux-x64 BackFiller startup smoke failed." exit 1 fi - grep -q "BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja" ./.artifacts/linux-x64-smoke-stdout.txt + grep -q "BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja" ./.artifacts/linux-x64-backfiller-smoke-stdout.txt + + - name: Smoke-test linux-x64 Native AOT startup (StorageService) + shell: bash + run: | + set -euo pipefail + exe=./.artifacts/linux-x64/VectorNNTPD.StorageService + config="$PWD/tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json" + test -f "$exe" + test -f "$config" + export VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE=1 + export VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG="$config" + set +e + timeout 30 "$exe" >./.artifacts/linux-x64-storageservice-smoke-stdout.txt 2>./.artifacts/linux-x64-storageservice-smoke-stderr.txt + code=$? + set -e + echo "linux-x64 StorageService startup smoke exit=$code" + echo "----- stdout -----" + cat ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + echo "----- stderr -----" + cat ./.artifacts/linux-x64-storageservice-smoke-stderr.txt + if [ "$code" -eq 124 ]; then + echo "linux-x64 StorageService startup smoke timed out." + exit 1 + fi + if [ "$code" -ne 0 ]; then + echo "linux-x64 StorageService startup smoke failed." + exit 1 + fi + grep -q "StorageService startup smoke: configuration serverId=1 fqdn=cache01.usenet.ninja" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + grep -q "Application logging initialized" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt + grep -q "VectorNNTPD.StorageService host starting" ./.artifacts/linux-x64-storageservice-smoke-stdout.txt - name: Upload Linux build artifact uses: actions/upload-artifact@v7 with: - name: backfiller-linux-x64-ci-build + name: vectornntpd-linux-x64-ci-build path: .artifacts/linux-x64/ if-no-files-found: error retention-days: 7 @@ -361,58 +415,76 @@ jobs: # VectorNNTPD.slnx is the canonical CI entrypoint for the production and test projects. run: dotnet build VectorNNTPD.slnx --configuration Release -p:Platform=x64 --no-restore - - name: Publish BackFiller win-x64 + - name: Publish production services win-x64 # Native AOT. PublishAot is true only for this runtime identifier, and the # ILCompiler pack is restored only when that identifier is present. # The solution restore above does not pass a runtime identifier, so this # publish restores itself. --no-restore here would publish a self-contained # managed app and the RequireNativeAotCompiler target would fail. + # Both services share one flat artifact directory (identical RabbitMQ JSON). shell: pwsh run: | - dotnet publish src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj ` - --configuration Release ` - -p:Platform=x64 ` - --runtime win-x64 ` - --self-contained true ` - -p:PublishAot=true ` - -p:SelfContained=true ` - --output ./.artifacts/win-x64 - - - name: Validate win-x64 Native AOT executable + $ErrorActionPreference = 'Stop' + foreach ($proj in @( + 'src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj', + 'src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj' + )) { + dotnet publish $proj ` + --configuration Release ` + -p:Platform=x64 ` + --runtime win-x64 ` + --self-contained true ` + -p:PublishAot=true ` + -p:SelfContained=true ` + --output ./.artifacts/win-x64 + if ($LASTEXITCODE -ne 0) { throw "dotnet publish failed for $proj" } + } + + - name: Validate win-x64 Native AOT executables shell: pwsh run: | $ErrorActionPreference = 'Stop' $dir = './.artifacts/win-x64' - $exe = Join-Path $dir 'VectorNNTPD.BackFillerService.exe' - if (-not (Test-Path -LiteralPath $exe)) { throw "Missing $exe" } - foreach ($required in @('VectorNNTPD.BackFillerService.json', 'VectorNNTPD.RabbitMQ.json')) { + foreach ($exeName in @('VectorNNTPD.BackFillerService.exe', 'VectorNNTPD.StorageService.exe')) { + $exe = Join-Path $dir $exeName + if (-not (Test-Path -LiteralPath $exe)) { throw "Missing $exe" } + $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path $exe)) + if ($bytes.Length -lt 512 -or $bytes[0] -ne 0x4D -or $bytes[1] -ne 0x5A) { throw "Expected a Windows PE executable: $exe" } + $peOffset = [System.BitConverter]::ToInt32($bytes, 0x3C) + if ($bytes[$peOffset] -ne 0x50 -or $bytes[$peOffset + 1] -ne 0x45) { throw "Missing PE signature: $exe" } + $machine = [System.BitConverter]::ToUInt16($bytes, $peOffset + 4) + if ($machine -ne 0x8664) { throw "Expected x64 machine 0x8664 and found 0x$('{0:X4}' -f $machine): $exe" } + $optional = $peOffset + 24 + $magic = [System.BitConverter]::ToUInt16($bytes, $optional) + if ($magic -ne 0x20B) { throw "Expected PE32+ and found magic 0x$('{0:X4}' -f $magic): $exe" } + $clrSize = [System.BitConverter]::ToUInt32($bytes, $optional + 112 + (14 * 8) + 4) + if ($clrSize -ne 0) { throw "Executable still has a CLR runtime header, so it is not Native AOT: $exe" } + } + foreach ($required in @('VectorNNTPD.BackFillerService.json', 'VectorNNTPD.StorageService.json', 'VectorNNTPD.RabbitMQ.json')) { $path = Join-Path $dir $required if (-not (Test-Path -LiteralPath $path)) { throw "Missing $path" } } - foreach ($forbidden in @('VectorNNTPD.BackFillerService.dll', 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', 'vectornntp-backfiller.service', 'backfiller.env.example', 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json')) { + foreach ($forbidden in @( + 'VectorNNTPD.BackFillerService.dll', + 'VectorNNTPD.StorageService.dll', + 'coreclr.dll', 'hostfxr.dll', 'libcoreclr.so', 'libhostfxr.so', + 'vectornntp-backfiller.service', 'backfiller.env.example', + 'vectornntp-storageservice.service', 'storageservice.env.example', + 'appsettings.json', 'RabbitMq.json', 'VectorNNTP.RabbitMQ.json', 'VectorNNTP.StorageServer.json', + 'VectorNNTPD.StorageService.Development.json', 'VectorNNTPD.BackFillerService.Development.json' + )) { $path = Join-Path $dir $forbidden if (Test-Path -LiteralPath $path) { throw "Native AOT output contains $forbidden." } } - $bytes = [System.IO.File]::ReadAllBytes((Resolve-Path $exe)) - if ($bytes.Length -lt 512 -or $bytes[0] -ne 0x4D -or $bytes[1] -ne 0x5A) { throw "Expected a Windows PE executable." } - $peOffset = [System.BitConverter]::ToInt32($bytes, 0x3C) - if ($bytes[$peOffset] -ne 0x50 -or $bytes[$peOffset + 1] -ne 0x45) { throw "Missing PE signature." } - $machine = [System.BitConverter]::ToUInt16($bytes, $peOffset + 4) - if ($machine -ne 0x8664) { throw "Expected x64 machine 0x8664 and found 0x$('{0:X4}' -f $machine)." } - $optional = $peOffset + 24 - $magic = [System.BitConverter]::ToUInt16($bytes, $optional) - if ($magic -ne 0x20B) { throw "Expected PE32+ and found magic 0x$('{0:X4}' -f $magic)." } - $clrSize = [System.BitConverter]::ToUInt32($bytes, $optional + 112 + (14 * 8) + 4) - if ($clrSize -ne 0) { throw "Executable still has a CLR runtime header, so it is not Native AOT." } - - - name: Smoke-test win-x64 Native AOT startup + + - name: Smoke-test win-x64 Native AOT startup (BackFiller) shell: pwsh run: | $ErrorActionPreference = 'Stop' $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.BackFillerService.exe' $config = (Resolve-Path './tests/VectorNNTPD.BackFillerService.Tests/Fixtures/startup-smoke.json').Path - $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-smoke-stdout.txt' - $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-smoke-stderr.txt' + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-backfiller-smoke-stderr.txt' $process = [System.Diagnostics.Process]::new() $process.StartInfo.FileName = $exe $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe @@ -421,32 +493,76 @@ jobs: $process.StartInfo.RedirectStandardError = $true $process.StartInfo.Environment['VECTORNNTP_BACKFILLER_STARTUP_SMOKE'] = '1' $process.StartInfo.Environment['VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG'] = $config - if (-not $process.Start()) { throw 'Failed to start the win-x64 startup smoke process.' } + if (-not $process.Start()) { throw 'Failed to start the win-x64 BackFiller startup smoke process.' } $stdoutTask = $process.StandardOutput.ReadToEndAsync() $stderrTask = $process.StandardError.ReadToEndAsync() if (-not $process.WaitForExit(30000)) { try { $process.Kill($true) } catch { } - throw 'win-x64 startup smoke timed out.' + throw 'win-x64 BackFiller startup smoke timed out.' } $process.WaitForExit() $stdout = $stdoutTask.GetAwaiter().GetResult() $stderr = $stderrTask.GetAwaiter().GetResult() Set-Content -LiteralPath $stdoutPath -Value $stdout Set-Content -LiteralPath $stderrPath -Value $stderr - Write-Host "win-x64 startup smoke exit=$($process.ExitCode)" + Write-Host "win-x64 BackFiller startup smoke exit=$($process.ExitCode)" Write-Host '----- stdout -----' Write-Host $stdout Write-Host '----- stderr -----' Write-Host $stderr - if ($process.ExitCode -ne 0) { throw 'win-x64 startup smoke failed.' } + if ($process.ExitCode -ne 0) { throw 'win-x64 BackFiller startup smoke failed.' } if ($stdout -notmatch 'BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01\.usenet\.ninja') { - throw 'win-x64 startup smoke did not reach the configuration checkpoint.' + throw 'win-x64 BackFiller startup smoke did not reach the configuration checkpoint.' + } + + - name: Smoke-test win-x64 Native AOT startup (StorageService) + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $exe = Join-Path (Resolve-Path './.artifacts/win-x64') 'VectorNNTPD.StorageService.exe' + $config = (Resolve-Path './tests/VectorNNTPD.StorageService.Tests/Fixtures/startup-smoke.json').Path + $stdoutPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stdout.txt' + $stderrPath = Join-Path (Resolve-Path './.artifacts') 'win-x64-storageservice-smoke-stderr.txt' + $process = [System.Diagnostics.Process]::new() + $process.StartInfo.FileName = $exe + $process.StartInfo.WorkingDirectory = Split-Path -Parent $exe + $process.StartInfo.UseShellExecute = $false + $process.StartInfo.RedirectStandardOutput = $true + $process.StartInfo.RedirectStandardError = $true + $process.StartInfo.Environment['VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE'] = '1' + $process.StartInfo.Environment['VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG'] = $config + if (-not $process.Start()) { throw 'Failed to start the win-x64 StorageService startup smoke process.' } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit(30000)) { + try { $process.Kill($true) } catch { } + throw 'win-x64 StorageService startup smoke timed out.' + } + $process.WaitForExit() + $stdout = $stdoutTask.GetAwaiter().GetResult() + $stderr = $stderrTask.GetAwaiter().GetResult() + Set-Content -LiteralPath $stdoutPath -Value $stdout + Set-Content -LiteralPath $stderrPath -Value $stderr + Write-Host "win-x64 StorageService startup smoke exit=$($process.ExitCode)" + Write-Host '----- stdout -----' + Write-Host $stdout + Write-Host '----- stderr -----' + Write-Host $stderr + if ($process.ExitCode -ne 0) { throw 'win-x64 StorageService startup smoke failed.' } + if ($stdout -notmatch 'StorageService startup smoke: configuration serverId=1 fqdn=cache01\.usenet\.ninja') { + throw 'win-x64 StorageService startup smoke did not reach the configuration checkpoint.' + } + if ($stdout -notmatch 'Application logging initialized') { + throw 'win-x64 StorageService startup smoke did not initialize logging.' + } + if ($stdout -notmatch 'VectorNNTPD\.StorageService host starting') { + throw 'win-x64 StorageService startup smoke did not log host starting.' } - name: Upload Windows build artifact uses: actions/upload-artifact@v7 with: - name: backfiller-win-x64-ci-build + name: vectornntpd-win-x64-ci-build path: .artifacts/win-x64/ if-no-files-found: error retention-days: 7 diff --git a/Directory.Build.targets b/Directory.Build.targets index cfa3624d..849cc76d 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -11,7 +11,7 @@ - + ©2026 Chris Knipe <cknipe@opticnetworks.net> @@ -22,7 +22,7 @@ --> + Condition="'$(DesignTimeBuild)' != 'true' AND ('$(MSBuildProjectName)' == 'VectorNNTPD.BackFillerService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.StorageService')"> <_ApplicationVersionNow>$([System.DateTime]::Now.Ticks) <_ApplicationVersionBuild>$([System.DateTime]::new($([System.Int64]::Parse($(_ApplicationVersionNow)))).Subtract($([System.DateTime]::new(2000,1,1))).Days) @@ -38,21 +38,36 @@ Production publish is self-contained Native AOT for win-x64 and linux-x64 only. A normal build does not set RuntimeIdentifier, so libraries, tests, and tools stay portable. --> - + win-x64;linux-x64 true true + + + $(NoWarn);IL2104;IL3050;IL3000;IL3002 + $(WarningsNotAsErrors);IL2104;IL3050 + false + false + + + Text="VectorNNTPD builds only for Platform x64. Platform '$(Platform)' is not supported." /> + Condition="('$(MSBuildProjectName)' == 'VectorNNTPD.BackFillerService' OR '$(MSBuildProjectName)' == 'VectorNNTPD.StorageService') AND '$(RuntimeIdentifier)' != '' AND '$(RuntimeIdentifier)' != 'win-x64' AND '$(RuntimeIdentifier)' != 'linux-x64'"> diff --git a/README.md b/README.md index 659cbaac..b92585e1 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,15 @@ -# BackFiller (Standalone) +# VectorNNTPD -Standalone migration of `VectorNNTPD.BackFillerService` into an independent repository. +Standalone VectorNNTPD hosts extracted from VectorNNTP.NNTPD. This repository currently ships BackFiller and StorageService, plus shared `VectorNNTPD.Common`. ## Structure -- `src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj` - standalone BackFiller worker app (net10.0) -- `src/VectorNNTPD.BackFillerService/` - production code organized by responsibility (for example `Acme`, `Articles`, `Configuration`, `Core`, `Hosting`, `Nntp`, `Transport`) -- `tests/VectorNNTPD.BackFillerService.Tests/VectorNNTPD.BackFillerService.Tests.csproj` - xUnit test suite, namespace `VectorNNTPD.BackFillerService.Tests` -- `VectorNNTPD.slnx` - solution including app + tests +- `src/VectorNNTPD.BackFillerService/VectorNNTPD.BackFillerService.csproj` - BackFiller worker app (net10.0) +- `src/VectorNNTPD.StorageService/VectorNNTPD.StorageService.csproj` - StorageService / VATP cache host (net10.0) +- `src/VectorNNTPD.Common/VectorNNTPD.Common.csproj` - shared library +- `tests/VectorNNTPD.BackFillerService.Tests/` / `tests/VectorNNTPD.StorageService.Tests/` / `tests/VectorNNTPD.Common.Tests/` - xUnit suites +- `VectorNNTPD.slnx` - solution including apps + tests +- `tools/Publish-Release.ps1` - Native AOT release publish for both production hosts - `tools/Invoke-CodeQlAnalysis.ps1` - local traced C# CodeQL analysis ## Build @@ -24,21 +26,27 @@ dotnet test .\tests\VectorNNTPD.BackFillerService.Tests\VectorNNTPD.BackFillerSe ## Publish -Windows x64: +Preferred release path (both hosts into one flat directory): ```powershell -dotnet publish .\src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj -c Release -p:Platform=x64 -r win-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true +.\tools\Publish-Release.ps1 -Platform win-64 +.\tools\Publish-Release.ps1 -Platform linux-64 ``` -Linux x64: +Per-project Native AOT publish still works. Example for StorageService: ```powershell -dotnet publish .\src\VectorNNTPD.BackFillerService\VectorNNTPD.BackFillerService.csproj -c Release -p:Platform=x64 -r linux-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true +dotnet publish .\src\VectorNNTPD.StorageService\VectorNNTPD.StorageService.csproj -c Release -p:Platform=x64 -r win-x64 --self-contained true -p:PublishAot=true -p:SelfContained=true ``` -The production project references the `StuDev.AotAnywhere` 1.0.4 MSBuild SDK. On Windows that SDK links `linux-x64` with the Zig toolchain restored by NuGet. A globalization-enabled Linux host still needs `libicu`. +Production hosts reference the `StuDev.AotAnywhere` 1.0.4 MSBuild SDK. On Windows that SDK links `linux-x64` with the Zig toolchain restored by NuGet. A globalization-enabled Linux host still needs `libicu`. -Native AOT publish keeps trim and AOT analysis enabled. CI then runs the published executable with `VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1` and `VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG` pointing at an isolated JSON file. The process must print `BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja` and exit 0. That checkpoint is configuration binding and host construction. It does not open a listener or contact a database, broker, DNS provider, or certificate authority. +Native AOT publish keeps trim and AOT analysis enabled. CI runs each published executable against an isolated JSON fixture: + +- BackFiller: `VECTORNNTP_BACKFILLER_STARTUP_SMOKE=1` / `VECTORNNTP_BACKFILLER_STARTUP_SMOKE_CONFIG` → checkpoint `BackFiller startup smoke: configuration serverId=1 fqdn=backfiller01.usenet.ninja` +- StorageService: `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE=1` / `VECTORNNTP_STORAGESERVICE_STARTUP_SMOKE_CONFIG` → checkpoint `StorageService startup smoke: configuration serverId=1 fqdn=cache01.usenet.ninja` + +Those checkpoints prove configuration binding and host construction (and StorageService logging initialization). They do not open a listener or contact a database, broker, DNS provider, or certificate authority. StorageService AOT publish still scopes Serilog.Settings.Configuration IL suppressions (`IL2104`/`IL3050`/`IL3000`/`IL3002`) because MinimumLevel is applied via `ReadFrom.Configuration` while sinks are registered explicitly. BackFiller references `VectorNNTPD.Common` for `IApplicationService`, NntpDB, ACME certificate management, the Cloudflare DNS API client, authoritative DNS queries used for DNS-01, RabbitMQ connectivity, and article parsing and validation. It keeps its own `ApplicationServiceManager` and does not use Common's `ApplicationLifecycle`. BackFiller still owns certificate watchdog health, the decision to publish listener-bound A/AAAA records only after the cache listener binds, Article Work topology, the RabbitMQ hosted adapter, and the retained article image that dot-stuffs body lines after Common's parser accepts the article. @@ -48,9 +56,11 @@ BackFiller references `VectorNNTPD.Common` for `IApplicationService`, NntpDB, AC ## Linux service -`tools/Publish-Release.ps1 -Platform linux-64` writes `release/linux-64/`. The Linux CI job writes `.artifacts/linux-x64/` and uploads that directory. Both contain `VectorNNTPD.BackFillerService`, `VectorNNTPD.BackFillerService.json`, `VectorNNTPD.RabbitMQ.json`, `README.md`, `vectornntp-backfiller.service`, and `backfiller.env.example`. +`tools/Publish-Release.ps1 -Platform linux-64` writes `release/linux-64/`. The Linux CI job writes `.artifacts/linux-x64/` and uploads that directory. Both contain the flat Native AOT outputs for `VectorNNTPD.BackFillerService` and `VectorNNTPD.StorageService`, their primary JSON files, shared `VectorNNTPD.RabbitMQ.json`, `README.md`, and the BackFiller and StorageService systemd units plus environment-file examples. + +Install BackFiller at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. -Install that directory at `/opt/vectornntp/backfiller`. `deploy/systemd/vectornntp-backfiller.service` uses that path for `ExecStart`, `WorkingDirectory`, and `Documentation=`. `ReadWritePaths` lists only `logs/` and `certs/`. Under `ProtectSystem=strict` that re-opens those two directories for writing. It does not change Unix ownership or mode. The shipped JSON writes file logs under `logs/` and ACME state under `certs/`, and startup creates files inside directories that must already exist. +Install StorageService at `/opt/vectornntp/storageservice` using `VectorNNTPD.StorageService`, `VectorNNTPD.StorageService.json`, `VectorNNTPD.RabbitMQ.json`, `vectornntp-storageservice.service`, and `storageservice.env.example` from the same release directory. That unit’s `ReadWritePaths` also include `db/` and `spool/` for the article index and segment-pool roots. Create the account and install the release as root-owned files. This step copies the placeholder environment file and reloads systemd. It does not start the service. diff --git a/VectorNNTPD.sln.DotSettings b/VectorNNTPD.sln.DotSettings index 4bbb1760..88332f6c 100644 --- a/VectorNNTPD.sln.DotSettings +++ b/VectorNNTPD.sln.DotSettings @@ -1,4 +1,5 @@  True + True True True \ No newline at end of file diff --git a/VectorNNTPD.slnx b/VectorNNTPD.slnx index 5ab1b747..ae1f5980 100644 --- a/VectorNNTPD.slnx +++ b/VectorNNTPD.slnx @@ -9,6 +9,9 @@ + + + @@ -17,5 +20,9 @@ + + + + diff --git a/deploy/systemd/storageservice.env.example b/deploy/systemd/storageservice.env.example new file mode 100644 index 00000000..5779a1e3 --- /dev/null +++ b/deploy/systemd/storageservice.env.example @@ -0,0 +1,16 @@ +# systemd EnvironmentFile for vectornntp-storageservice.service. +# One KEY=VALUE per line. A line whose first non-space character is # is a comment. +# systemd parses this file itself. source does not reproduce that parse. +# An unquoted backslash escapes the next character. \\ is one backslash. +# Confirm a value that contains a backslash, a quote, or leading or trailing whitespace on the target host. +# Copy to /etc/vectornntp/storageservice.env as root:root mode 0600, then replace the placeholder with sudoedit. +# Do not commit the installed file. +ConnectionStrings__NntpDB=Server=REPLACE_WITH_MYSQL_HOST;User ID=REPLACE_WITH_MYSQL_USER;Password=REPLACE_WITH_MYSQL_PASSWORD;Database=REPLACE_WITH_DATABASE_NAME +# These shared names are not required in /etc/vectornntp/storageservice.env. +# StorageService does not take them from this file. nntpsharedconfig supplies them after the connection string works. +# VECTOR__RABBITMQ__USERNAME +# VECTOR__RABBITMQ__PASSWORD +# VECTOR__CLOUDFLAREAPIKEY +# VECTOR__CLOUDFLAREZONEID +# VECTOR__ACMEACCOUNT +# VECTOR__ACMECERTIFICATEPASSWORD diff --git a/deploy/systemd/vectornntp-storageservice.service b/deploy/systemd/vectornntp-storageservice.service new file mode 100644 index 00000000..30fe9d21 --- /dev/null +++ b/deploy/systemd/vectornntp-storageservice.service @@ -0,0 +1,77 @@ +[Unit] +Description=VectorNNTP StorageService +# Install the flat linux-x64 release directory contents needed for this service +# at this path. The shared release also contains BackFiller binaries and units. +# This unit uses only VectorNNTPD.StorageService, VectorNNTPD.StorageService.json, +# VectorNNTPD.RabbitMQ.json, README.md, vectornntp-storageservice.service, and +# storageservice.env.example. +# Copy the unit to /etc/systemd/system/. Create user and group vectornntp, +# create logs/, certs/, db/, and spool/ owned by that account, and install +# /etc/vectornntp/storageservice.env before enable. +Documentation=file:///opt/vectornntp/storageservice/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=notify +NotifyAccess=main + +# Dedicated non-root service account (create before enabling the unit). +# Same account as BackFiller is acceptable when both share /opt/vectornntp. +User=vectornntp +Group=vectornntp + +# The executable directory is the configuration root. The process loads +# VectorNNTPD.StorageService.json and VectorNNTPD.RabbitMQ.json from that directory, +# not appsettings.json, RabbitMq.json, or the process working directory. +# Relative logs (logs/), ACME state (certs/), article index (db/), and spool +# roots also resolve there. +WorkingDirectory=/opt/vectornntp/storageservice +ExecStart=/opt/vectornntp/storageservice/VectorNNTPD.StorageService + +# Host GracefulShutdownTimeout follows StorageServer:GracefulShutdownTimeout. +# The shipped value is 00:00:30. TimeoutStopSec must stay above that setting +# or systemd SIGKILLs an in-progress drain. +TimeoutStartSec=180 +TimeoutStopSec=60 + +Restart=on-failure +RestartSec=5 + +# systemd reads this file as root before exec. The leading '-' suppresses errors +# reading the file and still starts the process. Current systemd ignores a missing, +# unreadable, or invalid file. That does not insert ConnectionStrings__NntpDB. +# Startup validation still fails when the variable is absent. The file is KEY=VALUE, +# not a shell script. An unquoted backslash is an escape. The connection string has +# no VECTOR__ prefix. RabbitMQ, ACME, and Cloudflare secrets come from +# nntpsharedconfig; this unit does not set them. +EnvironmentFile=-/etc/vectornntp/storageservice.env +# DOTNET_ENVIRONMENT selects the optional VectorNNTPD.StorageService.Production.json overlay. +Environment=DOTNET_ENVIRONMENT=Production + +# Watchdog heartbeats run only when EnableWatchdog is true (the shipped default) +# and this unit sets WatchdogSec, which supplies WATCHDOG_USEC. +# Leave WatchdogSec unset until readiness has been checked on the target host. +# Example (enable only after validation): +# WatchdogSec=30 +# Restart=on-watchdog + +# File logging is the shipped destination (StorageServer:LogDir=logs). +# The shipped VectorNNTPD.StorageService.json also enables the Console sink, so +# Serilog Information+ lines reach stdout and the journal under this unit. +# Hardening. StorageService binds TLS and opens outbound RabbitMQ, MySQL, and Cloudflare. +# ProtectSystem=strict mounts the filesystem read-only. ReadWritePaths re-opens only +# logs/, certs/, db/, and spool/. It does not grant Unix ownership; those directories +# must already exist and be writable by the vectornntp account. The executable and +# JSON stay read-only. +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/vectornntp/storageservice/logs /opt/vectornntp/storageservice/certs /opt/vectornntp/storageservice/db /opt/vectornntp/storageservice/spool + +KillSignal=SIGTERM +SendSIGKILL=yes + +[Install] +WantedBy=multi-user.target diff --git a/docs/common/configuration.md b/docs/common/configuration.md index d6b47f71..6af693be 100644 --- a/docs/common/configuration.md +++ b/docs/common/configuration.md @@ -18,7 +18,7 @@ NNTPD, BackFiller, and StorageServer publish the ACME account email, ACME certif Top-level `RabbitMQ` section. The Common sample is `src/VectorNNTPD.Common/VectorNNTPD.RabbitMQ.json`. `ApplicationJsonConfiguration.AddSharedRabbitMqJsonFile` loads only the filename the caller passes. BackFiller passes `VectorNNTPD.RabbitMQ.json`. The file is inserted after the entry-assembly JSON files and before environment variables and command-line arguments. Options types and connection validation live in `VectorNNTPD.Common.Messaging.RabbitMq`. Applications bind the section and call `AddRabbitMqInfrastructure()`. Nested `Management` is validated only when `BaseUrl` is set. -NNTPD and StorageServer are not projects in this repository, so their build output is not part of the contract here. BackFiller ships connectivity in `src/VectorNNTPD.BackFillerService/VectorNNTPD.RabbitMQ.json`. Its primary file, `VectorNNTPD.BackFillerService.json`, still must not repeat a top-level `RabbitMQ` section. `BackFiller:Logging:RabbitMQ` is the log sink, not broker connectivity. +NNTPD is not a project in this repository, so its build output is not part of the contract here. BackFiller and StorageService ship connectivity as `VectorNNTPD.RabbitMQ.json` beside each host project. Primary host files (`VectorNNTPD.BackFillerService.json`, `VectorNNTPD.StorageService.json`) must not repeat a top-level `RabbitMQ` section. `BackFiller:Logging:RabbitMQ` is the log sink, not broker connectivity. `RabbitMqService` owns one process-wide connection, verifies that it is open, and replaces it on connectivity loss or when a broker block lasts longer than `ConnectionBlockedTimeoutSeconds`, while incrementing a monotonic generation. Client automatic recovery is disabled. The AMQP client name comes from `IRabbitMqConnectionNameProvider`. Callers obtain the current connection with `TryGetCurrent` and do not dispose it. diff --git a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj index d19de2d7..e648b0b5 100644 --- a/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj +++ b/src/VectorNNTPD.Common/VectorNNTPD.Common.csproj @@ -16,8 +16,8 @@ - - + +