Add FileProfileDirectoryRecoveryService for recovery

FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Abstractions.TestingHelpers;
+using System.Threading;
+using System.Threading.Tasks;
+using FileProcessor.BusinessLogic.Managers;
+using FileProcessor.BusinessLogic.Requests;
+using FileProcessor.BusinessLogic.Services;
+using FileProcessor.Models;
+using MediatR;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Shared.Results;
+using Shouldly;
+using SimpleResults;
+using FileProcessor.Testing;
+using FileProfileModel = global::FileProcessor.Models.FileProfile;
+using Xunit;
+
+namespace FileProcessor.BusinessLogic.Tests;
+
+public class FileProfileDirectoryRecoveryServiceTests
+{
+    [Fact]
+    public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirectory()
+    {
+        Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
+
+        Mock<IFileProcessorManager> fileProcessorManager = new();
+        Mock<IMediator> mediator = new();
+        MockFileSystem fileSystem = new();
+
+        FileProfileModel fileProfile = TestData.FileProfile;
+        string inProgressDirectory = $"{fileProfile.ListeningDirectory}/inprogress";
+        string inProgressFilePath = $"{inProgressDirectory}/{TestData.EstateId:N}-{TestData.FileId:N}";
+
+        fileSystem.AddDirectory(inProgressDirectory);
+        fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1"));
+
+        fileProcessorManager
+            .Setup(manager => manager.GetAllFileProfiles(It.IsAny<CancellationToken>()))
+            .ReturnsAsync(Result.Success(new List<FileProfileModel> { fileProfile }));
+
+        fileProcessorManager
+            .Setup(manager => manager.GetFile(TestData.FileId, TestData.EstateId, It.IsAny<CancellationToken>()))
+            .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails()));
+
+        mediator
+            .Setup(send => send.Send(It.IsAny<FileCommands.ProcessUploadedFileCommand>(), It.IsAny<CancellationToken>()))
+            .ReturnsAsync(Result.Success());
+
+        FileProfileDirectoryRecoveryService service = new(
+            fileProcessorManager.Object,
+            mediator.Object,
+            fileSystem);
+
+        Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        mediator.Verify(send => send.Send(
+            It.Is<FileCommands.ProcessUploadedFileCommand>(command =>
+                command.FileId == TestData.FileId &&
+                Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath) &&
+                command.FileProfileId == TestData.FileProfileId &&
+                command.EstateId == TestData.EstateId),
+            It.IsAny<CancellationToken>()), Times.Once);
+    }
+}

FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs

@@ -0,0 +1,117 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Abstractions;
+using System.Threading;
+using System.Threading.Tasks;
+using FileProcessor.BusinessLogic.Managers;
+using FileProcessor.BusinessLogic.Requests;
+using FileProcessor.Models;
+using MediatR;
+using Shared.Results;
+using SimpleResults;
+using FileProfileModel = global::FileProcessor.Models.FileProfile;
+
+namespace FileProcessor.BusinessLogic.Services;
+
+public sealed class FileProfileDirectoryRecoveryService : IFileProfileDirectoryRecoveryService
+{
+    private const string InProgressDirectoryName = "inprogress";
+
+    private readonly IFileProcessorManager FileProcessorManager;
+    private readonly IMediator Mediator;
+    private readonly IFileSystem FileSystem;
+
+    public FileProfileDirectoryRecoveryService(
+        IFileProcessorManager fileProcessorManager,
+        IMediator mediator,
+        IFileSystem fileSystem)
+    {
+        this.FileProcessorManager = fileProcessorManager;
+        this.Mediator = mediator;
+        this.FileSystem = fileSystem;
+    }
+
+    public async Task<Result> RecoverInProgressFilesAsync(CancellationToken cancellationToken)
+    {
+        Result<List<FileProfileModel>> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfiles(cancellationToken);
+        if (fileProfilesResult.IsFailed)
+        {
+            Shared.Logger.Logger.LogWarning($"Unable to load file profiles for in-progress recovery: {fileProfilesResult.Message}");
+            return ResultHelpers.CreateFailure(fileProfilesResult);
+        }
+
+        foreach (FileProfileModel fileProfile in fileProfilesResult.Data)
+        {
+            string inProgressDirectory = Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName);
+            if (this.FileSystem.Directory.Exists(inProgressDirectory) == false)
+            {
+                continue;
+            }
+
+            foreach (string filePath in this.FileSystem.Directory.GetFiles(inProgressDirectory))
+            {
+                await this.RecoverFileAsync(fileProfile, filePath, cancellationToken);
+            }
+        }
+
+        return Result.Success();
+    }
+
+    private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePath, CancellationToken cancellationToken)
+    {
+        string fileName = Path.GetFileNameWithoutExtension(filePath);
+        if (TryParseIdentifiers(fileName, out Guid estateId, out Guid fileId) == false)
+        {
+            Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because the name does not contain recoverable identifiers");
+            return;
+        }
+
+        Result<FileDetails> fileResult = await this.FileProcessorManager.GetFile(fileId, estateId, cancellationToken);
+        if (fileResult.IsFailed)
+        {
+            Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] could not be loaded: {fileResult.Message}");
+            return;
+        }
+
+        FileDetails fileDetails = fileResult.Data;
+        if (fileDetails == null)
+        {
+            Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] returned no details");
+            return;
+        }
+
+        FileCommands.ProcessUploadedFileCommand command = new(
+            fileDetails.EstateId,
+            fileDetails.MerchantId,
+            fileDetails.FileImportLogId,
+            fileDetails.FileId,
+            fileDetails.UserId,
+            filePath,
+            fileDetails.FileProfileId,
+            fileDetails.FileReceivedDateTime);
+
+        Result result = await this.Mediator.Send(command, cancellationToken);
+        if (result.IsFailed)
+        {
+            Shared.Logger.Logger.LogWarning($"Recovery processing failed for leftover in-progress file [{filePath}]: {result.Message}");
+            return;
+        }
+
+        Shared.Logger.Logger.LogInformation($"Recovered leftover in-progress file [{filePath}] for file profile [{fileProfile.FileProfileId}]");
+    }
+
+    private static bool TryParseIdentifiers(string fileName, out Guid estateId, out Guid fileId)
+    {
+        estateId = Guid.Empty;
+        fileId = Guid.Empty;
+
+        string[] parts = fileName.Split('-', 2, StringSplitOptions.RemoveEmptyEntries);
+        if (parts.Length != 2)
+        {
+            return false;
+        }
+
+        return Guid.TryParse(parts[0], out estateId) && Guid.TryParse(parts[1], out fileId);
+    }
+}

FileProcessor.BusinessLogic/Services/IFileProfileDirectoryRecoveryService.cs

@@ -0,0 +1,10 @@
+using SimpleResults;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace FileProcessor.BusinessLogic.Services;
+
+public interface IFileProfileDirectoryRecoveryService
+{
+    Task<Result> RecoverInProgressFilesAsync(CancellationToken cancellationToken);
+}

FileProcessor.Testing/TestData.cs

@@ -278,6 +278,23 @@ public static FileAggregate GetCreatedFileAggregate()
             return fileAggregate;
         }
 
+        public static FileDetails GetCreatedFileDetails()
+        {
+            return new FileDetails
+            {
+                FileReceivedDateTime = TestData.FileUploadedDateTime,
+                ProcessingCompleted = false,
+                FileLines = new List<FileLine>(),
+                EstateId = TestData.EstateId,
+                MerchantId = TestData.MerchantId,
+                FileProfileId = TestData.FileProfileId,
+                FileId = TestData.FileId,
+                FileImportLogId = TestData.FileImportLogId,
+                FileLocation = TestData.FilePathWithName,
+                UserId = TestData.UserId
+            };
+        }
+
         public static FileAggregate GetFileAggregateWithLines()
         {
             FileAggregate fileAggregate = new FileAggregate();

FileProcessor/Bootstrapper/MiscRegistry.cs

@@ -25,6 +25,7 @@ public MiscRegistry()
         this.AddSingleton<IModelFactory, ModelFactory>();
         this.AddSingleton<IFileProcessorDomainService, FileProcessorDomainService>();
         this.AddSingleton<IFileProfileDirectorySynchronizer, FileProfileDirectorySynchronizer>();
+        this.AddSingleton<IFileProfileDirectoryRecoveryService, FileProfileDirectoryRecoveryService>();
 
         bool logRequests = ConfigurationReader.GetValueOrDefault<Boolean>("MiddlewareLogging", "LogRequests", true);
         bool logResponses = ConfigurationReader.GetValueOrDefault<Boolean>("MiddlewareLogging", "LogResponses", true);

FileProcessor/Common/Extensions.cs

@@ -69,7 +69,7 @@ public static IServiceCollection AddInSecureEventStoreClient(
     public static void PreWarm(this IApplicationBuilder applicationBuilder){
         IFileSystem fileSystem = Startup.Container.GetInstance<IFileSystem>();
         IFileProfileDirectorySynchronizer directorySynchronizer = Startup.Container.GetInstance<IFileProfileDirectorySynchronizer>();
-        // TODO: Do we poll here for files incase they have been left from a previous run
+        IFileProfileDirectoryRecoveryService recoveryService = Startup.Container.GetInstance<IFileProfileDirectoryRecoveryService>();
         var temporaryFileLocation = ConfigurationReader.GetValue("AppSettings", "TemporaryFileLocation");
         Logger.LogInformation($"Starting up, TemporaryFileLocation is [{temporaryFileLocation}]");
 
@@ -82,6 +82,12 @@ public static void PreWarm(this IApplicationBuilder applicationBuilder){
             throw new ApplicationStartupException(syncResult.Message);
         }
 
+        Result recoveryResult = recoveryService.RecoverInProgressFilesAsync(CancellationToken.None).GetAwaiter().GetResult();
+        if (recoveryResult.IsFailed)
+        {
+            Logger.LogWarning($"Error recovering in-progress files {recoveryResult.Message}");
+        }
+
         TypeProvider.LoadDomainEventsTypeDynamically();
 
         IConfigurationSection subscriptionConfigSection = Startup.Configuration.GetSection("AppSettings:SubscriptionConfiguration");