Improve integration test container startup reliability

Shared.EventStoreContext.Tests/EventStoreContextRetryIntegrationTests.cs

@@ -42,7 +42,7 @@ public EventStoreContextRetryIntegrationTests()
     public async Task EventStoreContext_InsertEvents_RetriesAfterDockerOutage()
     {
         this.InitialiseRetryConfiguration();
-        await this.EventStoreDockerHelper.StartContainers(false, nameof(this.EventStoreContext_InsertEvents_RetriesAfterDockerOutage));
+        await this.EventStoreDockerHelper.StartContainers(false, nameof(this.EventStoreContext_InsertEvents_RetriesAfterDockerOutage), CancellationToken.None);
 
         TimeSpan deadline = TimeSpan.FromSeconds(5);
         IEventStoreContext context = this.CreateContext(false, deadline);

Shared.EventStoreContext.Tests/EventStoreContextTests.cs

@@ -103,7 +103,7 @@ public void EventStoreContext_GetStatusFrom_ProjectionDetailsIsNull_CorrectValue
     [TestCase(false)]
     public async Task EventStoreContext_InsertEvents_EventsAreWritten(Boolean secureEventStore){
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_InsertEvents_EventsAreWritten_{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_InsertEvents_EventsAreWritten_{secureEventStore}", CancellationToken.None);
 
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
@@ -132,7 +132,7 @@ public async Task EventStoreContext_ReadEvents_EventsAreRead(Boolean secureEvent
         TypeMap.AddType<EstateCreatedEvent>("EstateCreatedEvent");
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEvents_EventsAreRead{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEvents_EventsAreRead{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 
@@ -175,7 +175,7 @@ public async Task EventStoreContext_ReadEvents_EventsAreRead(Boolean secureEvent
     public async Task EventStoreContext_ReadEventsBackwards_EventsAreRead(Boolean secureEventStore){
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEventsBackwards_EventsAreRead{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEventsBackwards_EventsAreRead{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 
@@ -206,7 +206,7 @@ public async Task EventStoreContext_ReadEventsBackwards_EventsAreRead(Boolean se
     public async Task EventStoreContext_ReadEventsBackwards_StreamNotFound_EventsAreRead(Boolean secureEventStore){
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEventsBackwards_StreamNotFound_EventsAreRead{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_ReadEventsBackwards_StreamNotFound_EventsAreRead{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 
@@ -225,7 +225,7 @@ public async Task EventStoreContext_ReadEventsBackwards_StreamNotFound_EventsAre
     public async Task EventStoreContext_RunTransientQuery_Faulted_ErrorThrown(Boolean secureEventStore){
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_Faulted_ErrorThrown{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_Faulted_ErrorThrown{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 
@@ -274,7 +274,7 @@ public async Task EventStoreContext_RunTransientQuery_Faulted_ErrorThrown(Boolea
     public async Task EventStoreContext_RunTransientQuery_QueryIsRun(Boolean secureEventStore){
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_QueryIsRun{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_QueryIsRun{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 
@@ -320,7 +320,7 @@ public async Task EventStoreContext_RunTransientQuery_QueryIsRun(Boolean secureE
     public async Task EventStoreContext_RunTransientQuery_ResultIsEmpty_ErrorThrown(Boolean secureEventStore){
         TimeSpan deadline = TimeSpan.FromMinutes(2);
 
-        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_ResultIsEmpty_ErrorThrown{secureEventStore}");
+        await this.EventStoreDockerHelper.StartContainers(secureEventStore, $"EventStoreContext_RunTransientQuery_ResultIsEmpty_ErrorThrown{secureEventStore}", CancellationToken.None);
 
         IEventStoreContext context = this.CreateContext(secureEventStore, deadline);
 

Shared.EventStoreContext.Tests/EventStoreDockerHelper.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using System.Threading;
 using KurrentDB.Client;
 using Shared.IntegrationTesting;
 using Shared.IntegrationTesting.TestContainers;
@@ -10,11 +11,14 @@ namespace Shared.EventStoreContext.Tests;
 
 public class EventStoreDockerHelper : Shared.IntegrationTesting.TestContainers.DockerHelper
 {
-    public async Task StartContainers(Boolean isSecureEventStore, String testName)
+    public async Task StartContainers(
+        Boolean isSecureEventStore,
+        String testName,
+        CancellationToken cancellationToken)
     {
         this.SetHostTraceFolder(testName);
         this.ScenarioName = testName;
-        await this.StartContainersForScenarioRun(testName, DockerServices.EventStore);
+        await this.StartContainersForScenarioRun(testName, DockerServices.EventStore, cancellationToken);
     }
 
     public override async Task StopContainersForScenarioRun(DockerServices sharedDockerServices)
@@ -42,13 +46,17 @@ public override async Task StopContainersForScenarioRun(DockerServices sharedDoc
         }
     }
 
-    public override async Task CreateSubscriptions()
+    public override async Task CreateSubscriptions(CancellationToken cancellationToken)
     {
         // Nothing actually needed here
     }
 
-    public override async Task StartContainersForScenarioRun(String scenarioName, DockerServices services)
+    public override async Task StartContainersForScenarioRun(
+        String scenarioName,
+        DockerServices services,
+        CancellationToken cancellationToken)
     {
+        cancellationToken.ThrowIfCancellationRequested();
         this.TestId = Guid.NewGuid();
         String networkName = $"eventstoretestnetwork{this.TestId:N}";
         INetwork networkService = await this.SetupTestNetwork(networkName, true);
@@ -78,13 +86,15 @@ await this.StartContainer2(SetupSecureEventStoreContainerLocal,
                                   new List<INetwork> {
                                                                 networkService
                                                             },
-                                  DockerServices.EventStore);
+                                  DockerServices.EventStore,
+                                  cancellationToken);
 
         await this.StartContainer2(SetupInsecureEventStoreContainerLocal,
             new List<INetwork> {
                 networkService
             },
-            DockerServices.EventStore);
+            DockerServices.EventStore,
+            cancellationToken);
     }
 
     public KurrentDBClientSettings CreateEventStoreClientSettings(Boolean secureEventStore, TimeSpan? deadline = null, String userName = "admin", String password = "changeit")

Shared.IntegrationTesting.Tests/DockerHelperStartupPlanTests.cs

@@ -1,7 +1,17 @@
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using DotNet.Testcontainers.Builders;
+using DotNet.Testcontainers.Containers;
+using EventStore.Client;
+using NLog;
 using NUnit.Framework;
+using Shared.Logger;
 using Shared.IntegrationTesting;
+using Shared.IntegrationTesting.TestContainers;
 using Shouldly;
 
 namespace Shared.IntegrationTesting.Tests;
@@ -10,22 +20,72 @@ public class DockerHelperStartupPlanTests
 {
     private sealed class InspectableDockerHelper : TestDockerHelper
     {
-        public void UseLinuxPlatform()
+        public IReadOnlyList<IReadOnlyList<DockerServices>> GetStartupGroupsForTesting()
         {
-            this.DockerPlatform = DockerEnginePlatform.Linux;
+            return base.GetStartupGroups();
         }
 
-        public IReadOnlyList<IReadOnlyList<DockerServices>> GetStartupGroupsForTesting()
+        public HttpClient GetEventStoreHealthCheckClientForTesting()
         {
-            return base.GetStartupGroups();
+            return BaseDockerHelper.EventStoreHealthCheckClient;
+        }
+
+        public Task CaptureContainerLogsForTesting(IContainer container)
+        {
+            return base.CaptureContainerLogs(container, CancellationToken.None);
+        }
+
+        public static IReadOnlyList<(DockerServices service, IContainer container)> GetRetainedSharedContainers(
+            IEnumerable<(DockerServices service, IContainer container)> containers,
+            DockerServices sharedDockerServices)
+        {
+            return DockerHelper.RetainSharedContainers(containers, sharedDockerServices);
+        }
+
+    }
+
+    private sealed class FailingProjectionDockerHelper : TestDockerHelper
+    {
+        private readonly FileInfo projectionFile;
+
+        public FailingProjectionDockerHelper(FileInfo projectionFile)
+        {
+            this.projectionFile = projectionFile;
+
+            NlogLogger logger = new();
+            logger.Initialise(LogManager.GetLogger(nameof(FailingProjectionDockerHelper)), nameof(FailingProjectionDockerHelper));
+            this.Logger = logger;
+        }
+
+        public Task LoadProjectionsForTesting()
+        {
+            return base.LoadEventStoreProjections(CancellationToken.None);
+        }
+
+        protected override IReadOnlyList<FileInfo> GetProjectionFiles()
+        {
+            return [this.projectionFile];
+        }
+
+        protected override List<string> GetRequiredProjections()
+        {
+            return [this.projectionFile.Name];
+        }
+
+        protected override Task CreateContinuousProjection(
+            EventStoreProjectionManagementClient projectionClient,
+            string projectionName,
+            string projection,
+            CancellationToken cancellationToken)
+        {
+            throw new InvalidOperationException("projection creation failed");
         }
     }
 
     [Test]
     public void StartupGroups_place_foundation_services_first_and_estate_management_ui_last_on_linux()
     {
         InspectableDockerHelper helper = new();
-        helper.UseLinuxPlatform();
 
         IReadOnlyList<IReadOnlyList<DockerServices>> startupGroups = helper.GetStartupGroupsForTesting();
 
@@ -46,4 +106,73 @@ public void StartupGroups_place_foundation_services_first_and_estate_management_
             [DockerServices.EstateManagementUI]
         ]);
     }
+
+    [Test]
+    public void Shared_containers_are_retained_for_later_cleanup()
+    {
+        List<(DockerServices service, IContainer container)> containers =
+        [
+            (DockerServices.SqlServer, null!),
+            (DockerServices.EventStore, null!)
+        ];
+
+        IReadOnlyList<(DockerServices service, IContainer container)> remaining =
+            InspectableDockerHelper.GetRetainedSharedContainers(containers, DockerServices.SqlServer);
+
+        remaining.Select(container => container.service).ShouldBe([DockerServices.SqlServer]);
+    }
+
+    [Test]
+    public void EventStore_health_checks_reuse_their_http_client()
+    {
+        InspectableDockerHelper helper = new();
+
+        HttpClient firstClient = helper.GetEventStoreHealthCheckClientForTesting();
+        HttpClient secondClient = helper.GetEventStoreHealthCheckClientForTesting();
+
+        secondClient.ShouldBeSameAs(firstClient);
+    }
+
+    [Test]
+    public void Host_folder_mount_does_not_require_docker_platform()
+    {
+        ContainerBuilder builder = new();
+
+        ContainerBuilder mountedBuilder = builder.MountHostFolder("trace");
+
+        mountedBuilder.ShouldNotBeNull();
+    }
+
+    [Test]
+    public async Task Startup_honours_cancellation_before_creating_resources()
+    {
+        using CancellationTokenSource cancellationTokenSource = new();
+        cancellationTokenSource.Cancel();
+        InspectableDockerHelper helper = new();
+
+        await Should.ThrowAsync<OperationCanceledException>(() =>
+            helper.StartContainersForScenarioRun(
+                "cancelled-startup",
+                DockerServices.None,
+                cancellationTokenSource.Token));
+    }
+
+    [Test]
+    public async Task Projection_creation_failure_is_propagated()
+    {
+        string path = Path.Combine(Path.GetTempPath(), $"projection-{Guid.NewGuid():N}.js");
+        await File.WriteAllTextAsync(path, "fromAll().when({});");
+
+        try
+        {
+            FailingProjectionDockerHelper helper = new(new FileInfo(path));
+
+            await Should.ThrowAsync<InvalidOperationException>(
+                () => helper.LoadProjectionsForTesting());
+        }
+        finally
+        {
+            File.Delete(path);
+        }
+    }
 }

Shared.IntegrationTesting.Tests/GenericSteps.cs

@@ -11,7 +11,7 @@ namespace Shared.IntegrationTesting.Tests;
 using Reqnroll;
 
 public class TestDockerHelper : DockerHelper{
-    public override async Task CreateSubscriptions(){
+    public override async Task CreateSubscriptions(CancellationToken cancellationToken){
         // Nothing here
     }
 }
@@ -50,7 +50,10 @@ public async Task StartSystem() {
         this.TestingContext.Logger = logger;
         this.TestingContext.Logger.LogInformation("About to Start Containers for Scenario Run");
         this.TestingContext.DockerHelper.ScenarioName = scenarioName;
-        await this.TestingContext.DockerHelper.StartContainersForScenarioRun(scenarioName, services).ConfigureAwait(false);
+        await this.TestingContext.DockerHelper.StartContainersForScenarioRun(
+            scenarioName,
+            services,
+            CancellationToken.None).ConfigureAwait(false);
         this.TestingContext.Logger.LogInformation("Containers for Scenario Run Started");
     }
 

Shared.IntegrationTesting.UnitTests/RetryTests.cs

@@ -1,6 +1,7 @@
 namespace Shared.IntegrationTesting.UnitTests;
 
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 using Shared.IntegrationTesting;
 using Shouldly;
@@ -28,4 +29,36 @@ await Retry.For(async () =>
 
         attempts.ShouldBe(2);
     }
+
+    [Fact]
+    public async Task Retry_For_stops_immediately_when_cancelled()
+    {
+        using CancellationTokenSource cancellationTokenSource = new();
+        cancellationTokenSource.Cancel();
+
+        await Should.ThrowAsync<OperationCanceledException>(() => Retry.For(
+            _ => Task.CompletedTask,
+            cancellationTokenSource.Token,
+            TimeSpan.FromMinutes(1),
+            TimeSpan.FromSeconds(1)));
+    }
+
+    [Fact]
+    public async Task Retry_For_does_not_retry_when_exception_is_not_retryable()
+    {
+        Int32 attempts = 0;
+
+        await Should.ThrowAsync<InvalidOperationException>(() => Retry.For(
+            _ =>
+            {
+                attempts++;
+                throw new InvalidOperationException("terminal failure");
+            },
+            CancellationToken.None,
+            _ => false,
+            TimeSpan.FromMinutes(1),
+            TimeSpan.FromSeconds(1)));
+
+        attempts.ShouldBe(1);
+    }
 }

Shared.IntegrationTesting/ContainerType.cs

@@ -36,10 +36,3 @@ public enum DockerServices
     EstateManagementUI = 1024,
     EstateReporting = 2048
 }
-
-public enum DockerEnginePlatform
-{
-    Unknown,
-    Linux,
-    Windows
-}