Make AutoLogonWorkerService resilient to transient operator logon gRPC failures

TransactionProcessor.Tests/General/AutoLogonWorkerServiceTests.cs

@@ -0,0 +1,73 @@
+using Grpc.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Shared.Logger;
+using Shouldly;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using TransactionProcessor.BusinessLogic.OperatorInterfaces;
+using Xunit;
+
+namespace TransactionProcessor.Tests.General;
+
+public class AutoLogonWorkerServiceTests {
+    public AutoLogonWorkerServiceTests() {
+        Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
+    }
+
+    [Fact]
+    public async Task OperatorLogonAsync_TransientGrpcUnavailable_ExceptionIsHandled() {
+        Mock<IOperatorProxy> proxy = new Mock<IOperatorProxy>();
+        proxy.Setup(x => x.ProcessLogonMessage(It.IsAny<CancellationToken>()))
+             .ThrowsAsync(new RpcException(new Status(StatusCode.Unavailable, "RPC server unavailable")));
+
+        ServiceCollection services = new ServiceCollection();
+        services.AddSingleton<Func<String, IOperatorProxy>>(_ => _ => proxy.Object);
+        IServiceProvider previousServiceProvider = Startup.ServiceProvider;
+
+        try {
+            Startup.ServiceProvider = services.BuildServiceProvider();
+
+            AutoLogonWorkerService worker = new AutoLogonWorkerService();
+
+            Exception exception = await Record.ExceptionAsync(async () =>
+                                                              {
+                                                                  await worker.OperatorLogonAsync("operator-1", CancellationToken.None);
+                                                              });
+
+            exception.ShouldBeNull();
+        }
+        finally {
+            Startup.ServiceProvider = previousServiceProvider;
+        }
+    }
+
+    [Fact]
+    public async Task OperatorLogonAsync_NonTransientException_ExceptionIsHandled() {
+        Mock<IOperatorProxy> proxy = new Mock<IOperatorProxy>();
+        proxy.Setup(x => x.ProcessLogonMessage(It.IsAny<CancellationToken>()))
+             .ThrowsAsync(new InvalidOperationException("boom"));
+
+        ServiceCollection services = new ServiceCollection();
+        services.AddSingleton<Func<String, IOperatorProxy>>(_ => _ => proxy.Object);
+        IServiceProvider previousServiceProvider = Startup.ServiceProvider;
+
+        try {
+            Startup.ServiceProvider = services.BuildServiceProvider();
+
+            AutoLogonWorkerService worker = new AutoLogonWorkerService();
+
+            Exception exception = await Record.ExceptionAsync(async () =>
+                                                              {
+                                                                  await worker.OperatorLogonAsync("operator-2", CancellationToken.None);
+                                                              });
+
+            exception.ShouldBeNull();
+        }
+        finally {
+            Startup.ServiceProvider = previousServiceProvider;
+        }
+    }
+}

TransactionProcessor/AssemblyInfo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("TransactionProcessor.Tests")]

TransactionProcessor/Extensions.cs

@@ -7,11 +7,12 @@
 
 namespace TransactionProcessor
 {
-    using EventStore.Client;
-    using Microsoft.AspNetCore.Builder;
-    using Microsoft.Extensions.Caching.Memory;
-    using Microsoft.Extensions.Configuration;
-    using Microsoft.Extensions.DependencyInjection;
+using EventStore.Client;
+using Grpc.Core;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
     using Microsoft.Extensions.Hosting;
     using Shared.EventStore.Aggregate;
     using Shared.EventStore.EventHandling;
@@ -207,7 +208,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) {
 
                 if (Startup.AutoApiLogonOperators.Any()) {
                     foreach (String autoApiLogonOperator in Startup.AutoApiLogonOperators) {
-                        OperatorLogon(autoApiLogonOperator);
+                        await this.OperatorLogonAsync(autoApiLogonOperator, stoppingToken);
                     }
                 }
 
@@ -217,20 +218,37 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) {
             }
         }
 
-        private static void OperatorLogon(String operatorId) {
-                Logger.LogInformation($"About to do auto logon for operator Id [{operatorId}]");
-                Func<String, IOperatorProxy> resolver = Startup.ServiceProvider.GetService<Func<String, IOperatorProxy>>();
+        internal async Task OperatorLogonAsync(String operatorId, CancellationToken cancellationToken) {
+            Logger.LogInformation($"About to do auto logon for operator Id [{operatorId}]");
+
+            try {
+                Func<String, IOperatorProxy> resolver = Startup.ServiceProvider?.GetService<Func<String, IOperatorProxy>>();
+                if (resolver == null) {
+                    Logger.LogWarning($"Auto logon for operator Id [{operatorId}] skipped because no operator proxy resolver was registered");
+                    return;
+                }
+
                 IOperatorProxy proxy = resolver(operatorId);
+                if (proxy == null) {
+                    Logger.LogWarning($"Auto logon for operator Id [{operatorId}] skipped because no operator proxy was resolved");
+                    return;
+                }
 
-                Result<OperatorResponse> logonResult = proxy.ProcessLogonMessage(CancellationToken.None).Result;
+                Result<OperatorResponse> logonResult = await proxy.ProcessLogonMessage(cancellationToken);
 
                 if (logonResult.IsSuccess) {
                     Logger.LogInformation($"Auto logon for operator Id [{operatorId}] status [{logonResult.Data.IsSuccessful}]");
+                    return;
                 }
 
-                if (logonResult.IsFailed) {
-                    Logger.LogWarning($"Auto logon for operator Id [{operatorId}] status [{logonResult.Message}]");
-                }
+                Logger.LogWarning($"Auto logon for operator Id [{operatorId}] status [{logonResult.Message}]");
+            }
+            catch (RpcException ex) when (ex.StatusCode == StatusCode.Unavailable) {
+                Logger.LogWarning($"Auto logon for operator Id [{operatorId}] deferred because the operator gRPC service is unavailable: {ex.Status.Detail}");
+            }
+            catch (Exception ex) {
+                Logger.LogError(new Exception($"Auto logon for operator Id [{operatorId}] failed", ex));
+            }
         }
     }
-}
+}