Configurable retry for ACL token acquisition

TransactionProcessorACL.BusinessLogic.Tests/TransactionProcessorACLApplicationServiceTests.cs

@@ -3,6 +3,8 @@
 
 namespace TransactionProcessorACL.BusinesssLogic.Tests
 {
+    using System.Collections.Generic;
+    using System.Globalization;
     using System.Threading;
     using System.Threading.Tasks;
     using BusinessLogic.Services;
@@ -41,11 +43,18 @@ public TransactionProcessorACLApplicationServiceTests()
 
         private void SetupMemoryConfiguration()
         {
-            if (ConfigurationReader.IsInitialised == false)
-            {
-                IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(TestData.DefaultAppSettings).Build();
-                ConfigurationReader.Initialise(configuration);
+            this.InitialiseConfiguration();
+        }
+
+        private void InitialiseConfiguration(int? securityServiceTokenRetryCount = null)
+        {
+            Dictionary<String, String> settings = new(TestData.DefaultAppSettings);
+            if (securityServiceTokenRetryCount.HasValue) {
+                settings["AppSettings:SecurityServiceTokenRetryCount"] = securityServiceTokenRetryCount.Value.ToString(CultureInfo.InvariantCulture);
             }
+
+            IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
+            ConfigurationReader.Initialise(configuration);
         }
 
         private Mock<ITransactionProcessorClient> transactionProcessorClient;
@@ -521,6 +530,60 @@ public async Task TransactionProcessorACLApplicationService_GetMerchant_GetToken
 
         }
 
+        [Fact]
+        public async Task TransactionProcessorACLApplicationService_GetMerchant_GetTokenThrowsOnceThenSucceeds_RetriesAndReturnsSuccess()
+        {
+            this.InitialiseConfiguration(1);
+
+            transactionProcessorClient.Setup(v => v.GetMerchant(It.IsAny<String>(), It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()))
+                                      .ReturnsAsync(Result.Success(TestData.MerchantResponse(TransactionProcessor.DataTransferObjects.Responses.Merchant.SettlementSchedule.Monthly)));
+            securityServiceClient.SetupSequence(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()))
+                                 .ThrowsAsync(new TaskCanceledException("Transient token failure"))
+                                 .ReturnsAsync(Result.Success(TestData.TokenResponse));
+
+            Result<MerchantResponse> merchantResponse = await applicationService.GetMerchant(TestData.EstateId, TestData.MerchantId, CancellationToken.None);
+
+            merchantResponse.IsSuccess.ShouldBeTrue();
+            securityServiceClient.Verify(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
+        }
+
+        [Fact]
+        public async Task TransactionProcessorACLApplicationService_GetMerchant_GetTokenKeepsFailing_StopsAfterConfiguredRetries()
+        {
+            this.InitialiseConfiguration(2);
+
+            transactionProcessorClient.Setup(v => v.GetMerchant(It.IsAny<String>(), It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()))
+                                      .ReturnsAsync(Result.Success(TestData.MerchantResponse(TransactionProcessor.DataTransferObjects.Responses.Merchant.SettlementSchedule.Monthly)));
+            securityServiceClient.SetupSequence(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()))
+                                 .ThrowsAsync(new TaskCanceledException("Transient token failure"))
+                                 .ThrowsAsync(new TaskCanceledException("Transient token failure"))
+                                 .ThrowsAsync(new TaskCanceledException("Transient token failure"));
+
+            Result<MerchantResponse> merchantResponse = await applicationService.GetMerchant(TestData.EstateId, TestData.MerchantId, CancellationToken.None);
+
+            merchantResponse.IsFailed.ShouldBeTrue();
+            securityServiceClient.Verify(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()), Times.Exactly(3));
+        }
+
+        [Fact]
+        public async Task TransactionProcessorACLApplicationService_GetMerchant_GetTokenCanceled_DoesNotRetry()
+        {
+            this.InitialiseConfiguration(2);
+
+            using CancellationTokenSource cancellationTokenSource = new();
+            cancellationTokenSource.Cancel();
+
+            securityServiceClient.Setup(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()))
+                                 .ThrowsAsync(new Exception("GetToken should not be called when the request is already canceled"));
+
+            await Should.ThrowAsync<OperationCanceledException>(async () =>
+            {
+                await applicationService.GetMerchant(TestData.EstateId, TestData.MerchantId, cancellationTokenSource.Token);
+            });
+
+            securityServiceClient.Verify(s => s.GetToken(It.IsAny<String>(), It.IsAny<String>(), It.IsAny<CancellationToken>()), Times.Never);
+        }
+
         [Fact]
         public async Task TransactionProcessorACLApplicationService_GetMerchant_GetMerchantFailed_ResultIsFailed()
         {

TransactionProcessorACL.BusinessLogic/Services/TransactionProcessorACLApplicationService.cs

@@ -115,7 +115,50 @@ private async Task<Result<TokenResponse>> GetAccessToken(CancellationToken cance
             String clientId = ConfigurationReader.GetValue("AppSettings", "ClientId");
             String clientSecret = ConfigurationReader.GetValue("AppSettings", "ClientSecret");
 
-            return await this.SecurityServiceClient.GetToken(clientId, clientSecret, cancellationToken);
+            Int32 retryCount = GetSecurityServiceTokenRetryCount();
+
+            for (Int32 attempt = 0; attempt <= retryCount; attempt++) {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try {
+                    Result<TokenResponse> accessTokenResult = await this.SecurityServiceClient.GetToken(clientId, clientSecret, cancellationToken);
+                    if (accessTokenResult.IsSuccess) {
+                        return accessTokenResult;
+                    }
+
+                    if (attempt == retryCount) {
+                        return accessTokenResult;
+                    }
+
+                    Logger.LogWarning($"Security service token request failed on attempt {attempt + 1} of {retryCount + 1}: {accessTokenResult.Message}");
+                }
+                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {
+                    throw;
+                }
+                catch (Exception ex) {
+                    if (attempt == retryCount) {
+                        String errorMessage = String.Join(" | ", ex.GetExceptionMessages());
+                        Logger.LogWarning($"Security service token request failed after {retryCount + 1} attempts: {errorMessage}");
+                        return Result.Failure(errorMessage);
+                    }
+
+                    Logger.LogWarning($"Security service token request threw on attempt {attempt + 1} of {retryCount + 1}: {ex.GetExceptionMessages()}");
+                }
+            }
+
+            return Result.Failure("Unable to acquire security service token.");
+        }
+
+        private static Int32 GetSecurityServiceTokenRetryCount()
+        {
+            const Int32 DefaultSecurityServiceTokenRetryCount = 3;
+
+            String configuredRetryCount = ConfigurationReader.GetValue("AppSettings", "SecurityServiceTokenRetryCount");
+            if (Int32.TryParse(configuredRetryCount, out Int32 retryCount) && retryCount >= 0) {
+                return retryCount;
+            }
+
+            return DefaultSecurityServiceTokenRetryCount;
         }
 
         private SaleTransactionRequest BuildSaleTransactionRequest((Guid estateId, Guid merchantId) merchantData,

TransactionProcessorACL.Testing/TestData.cs

@@ -192,6 +192,7 @@ public class TestData
                 ["AppSettings:EstateReportingApi"] = "http://192.168.1.133:5011",
                 ["AppSettings:ClientId"] = "ClientId",
                 ["AppSettings:ClientSecret"] = "secret",
+                ["AppSettings:SecurityServiceTokenRetryCount"] = "3",
                 ["SecurityConfiguration:Authority"] = "https://127.0.0.1",
                 ["SecurityConfiguration:ApiName"] = "ApiName",
                 ["EventStoreSettings:ConnectionString"] = "esdb://127.0.0.1:2113"
@@ -206,6 +207,7 @@ public class TestData
                 ["AppSettings:EstateReportingApi"] = "http://192.168.1.133:5011",
                 ["AppSettings:ClientId"] = "ClientId",
                 ["AppSettings:ClientSecret"] = "secret",
+                ["AppSettings:SecurityServiceTokenRetryCount"] = "3",
                 ["SecurityConfiguration:Authority"] = "https://127.0.0.1",
                 ["EventStoreSettings:ConnectionString"] = "https://127.0.0.1:2113",
                 ["AppSettings:SkipVersionCheck"] = "true"

TransactionProcessorACL/appsettings.json

@@ -2,6 +2,7 @@
   "AppSettings": {
     "ClientId": "serviceClient",
     "ClientSecret": "d192cbc46d834d0da90e8a9d50ded543",
+    "SecurityServiceTokenRetryCount": 3,
     "MinimumSupportedApplicationVersion":  "1.0.5",
     "EstateReportingApi": "http://127.0.0.1:5011"
   },