diff --git a/Shared.IntegrationTesting/Retry.cs b/Shared.IntegrationTesting/Retry.cs index b485fd4..4f78256 100644 --- a/Shared.IntegrationTesting/Retry.cs +++ b/Shared.IntegrationTesting/Retry.cs @@ -57,7 +57,7 @@ public static async Task For(Func action, lastException = e; // wait before retrying - Thread.Sleep(retryInterval ?? Retry.DefaultRetryInterval); + await Task.Delay(retryInterval ?? Retry.DefaultRetryInterval); } } diff --git a/Shared.IntegrationTesting/TestContainers/BaseDockerHelper.cs b/Shared.IntegrationTesting/TestContainers/BaseDockerHelper.cs index 60e4346..900754d 100644 --- a/Shared.IntegrationTesting/TestContainers/BaseDockerHelper.cs +++ b/Shared.IntegrationTesting/TestContainers/BaseDockerHelper.cs @@ -794,28 +794,34 @@ protected virtual async Task CreatePersistentSubscription((String streamName, St protected async Task DoSqlServerHealthCheck(IContainer containerService){ // Try opening a connection - Int32 maxRetries = 10; - Int32 counter = 1; + const int maxRetries = 10; + bool connected = false; + + for (int attempt = 1; attempt <= maxRetries; attempt++) + { + try + { + this.Trace($"Connection attempt {attempt}"); - while (counter <= maxRetries){ - try{ - this.Trace($"Connection attempt {counter}"); CheckSqlConnection(containerService); + connected = true; break; } - catch(SqlException ex){ + catch (SqlException ex) + { this.Logger.LogError(ex); - await Task.Delay(30000); - } - finally{ - counter++; + + if (attempt < maxRetries) + { + await Task.Delay(TimeSpan.FromSeconds(30)); + } } } - if (counter >= maxRetries) + if (!connected) { - // We have got to the end and still not opened the connection - throw new ApplicationException($"Database container not started in {maxRetries} retries"); + throw new ApplicationException( + $"Database container not started after {maxRetries} attempts"); } } protected async Task DoEventStoreHealthCheck(){ @@ -933,47 +939,47 @@ protected virtual async Task LoadEventStoreProjections(){ //Start our Continuous Projections - we might decide to do this at a different stage, but now lets try here String projectionsFolder = "projections/continuous"; - if (!String.IsNullOrWhiteSpace(projectionsFolder)){ + if (!String.IsNullOrWhiteSpace(projectionsFolder)) + { DirectoryInfo di = new(projectionsFolder); - if (di.Exists){ + if (di.Exists) + { FileInfo[] files = di.GetFiles(); var requiredProjections = this.GetRequiredProjections(); EventStoreProjectionManagementClient projectionClient = new(this.ConfigureEventStoreSettings()); List projectionNames = new(); - - foreach (FileInfo file in files){ + + foreach (FileInfo file in files) + { if (!requiredProjections.Contains(file.Name)) continue; String projection = await BaseDockerHelper.RemoveProjectionTestSetup(file); String projectionName = file.Name.Replace(".js", String.Empty); - Should.NotThrow(async () => { - this.Trace($"Creating projection [{projectionName}] from file [{file.FullName}]"); - try{ - await projectionClient.CreateContinuousAsync(projectionName, projection, trackEmittedStreams:true).ConfigureAwait(false); - } - catch (Exception ex) { - // ignored - } - - projectionNames.Add(projectionName); - this.Trace($"Projection [{projectionName}] created"); - }, - $"Projection [{projectionName}] error"); - } - // Now check the create status of each - foreach (String projectionName in projectionNames){ - Should.NotThrow(async () => { - ProjectionDetails projectionDetails = await projectionClient.GetStatusAsync(projectionName); + this.Trace($"Creating projection [{projectionName}] from file [{file.FullName}]"); + try + { + await projectionClient.CreateContinuousAsync(projectionName, projection, trackEmittedStreams: true).ConfigureAwait(false); + } + catch (Exception ex) + { + // ignored + this.Trace($"Projection [{projectionName}] create error {ex.Message}"); + } - projectionDetails.Status.ShouldBe("Running", $"Projection [{projectionName}] is {projectionDetails.Status}"); + projectionNames.Add(projectionName); + this.Trace($"Projection [{projectionName}] created"); + } - this.Trace($"Projection [{projectionName}] running"); - }, - "Error getting Projection [{projectionName}] status"); + // Now check the create status of each + foreach (String projectionName in projectionNames) + { + ProjectionDetails projectionDetails = await projectionClient.GetStatusAsync(projectionName); + projectionDetails.Status.ShouldBe("Running", $"Projection [{projectionName}] is {projectionDetails.Status}"); + this.Trace($"Projection [{projectionName}] running"); } } } @@ -1029,8 +1035,7 @@ await Retry.For(async () => }); this.Trace($"{dockerService} Container Started"); - this.Containers.Add((dockerService, builtContainer)); - + // Do a health check here ContainerType type = dockerService switch{ DockerServices.CallbackHandler => ContainerType.CallbackHandler, diff --git a/Shared.IntegrationTesting/TestContainers/DockerHelper.cs b/Shared.IntegrationTesting/TestContainers/DockerHelper.cs index d28ac00..24f4728 100644 --- a/Shared.IntegrationTesting/TestContainers/DockerHelper.cs +++ b/Shared.IntegrationTesting/TestContainers/DockerHelper.cs @@ -124,49 +124,70 @@ public void DisposeIntegrationTestCertificate() this.IntegrationTestCertificate = null; } - public override async Task StopContainersForScenarioRun(DockerServices sharedDockerServices) { + public override async Task StopContainersForScenarioRun( + DockerServices sharedDockerServices) + { + List cleanupErrors = []; + try { - if (this.Containers.Any()) { - this.Containers.Reverse(); - - foreach ((DockerServices, IContainer) containerService in this.Containers) { - - if ((sharedDockerServices & containerService.Item1) == containerService.Item1){ - continue; - } - - String? name; - try - { - name = containerService.Item2.Name; - } - catch (InvalidOperationException ex) - { - this.Trace($"Skipping container entry that is no longer available [{containerService.Item1}] ({ex.Message})"); - continue; - } - this.Trace($"Stopping container [{name}]"); - await containerService.Item2.StopAsync(CancellationToken.None); - await containerService.Item2.DisposeAsync(); - this.Trace($"Container [{name}] stopped"); + foreach ((DockerServices service, IContainer container) + in this.Containers.AsEnumerable().Reverse()) + { + if ((sharedDockerServices & service) == service) + { + continue; + } + String containerName = container.Name; + try + { + this.Trace($"Stopping container [{containerName}]"); + + await container.StopAsync(CancellationToken.None); + await container.DisposeAsync(); + + this.Trace($"Container [{containerName}] stopped"); + } + catch (Exception ex) + { + this.Trace($"Failed to stop container [{containerName}]"); + cleanupErrors.Add(ex); } } - if (this.TestNetworks.Any()) { - foreach (INetwork networkService in this.TestNetworks){ - await networkService.DeleteAsync(CancellationToken.None); - await networkService.DisposeAsync(); + foreach (INetwork network in this.TestNetworks.AsEnumerable().Reverse()) + { + try + { + await network.DeleteAsync(CancellationToken.None); + await network.DisposeAsync(); + } + catch (Exception ex) + { + this.Trace("Failed to remove Docker network"); + cleanupErrors.Add(ex); } } } finally { - if ((sharedDockerServices & DockerServices.SecurityService) != DockerServices.SecurityService) + // Prevent a second cleanup attempt from using disposed objects. + this.Containers.Clear(); + this.TestNetworks.Clear(); + + if ((sharedDockerServices & DockerServices.SecurityService) + != DockerServices.SecurityService) { this.DisposeIntegrationTestCertificate(); } } + + if (cleanupErrors.Count > 0) + { + throw new AggregateException( + "One or more integration-test resources failed to clean up.", + cleanupErrors); + } } public abstract Task CreateSubscriptions();