diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs new file mode 100644 index 0000000..c5e5227 --- /dev/null +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions.TestingHelpers; +using System.Threading; +using System.Threading.Tasks; +using FileProcessor.BusinessLogic.Managers; +using FileProcessor.BusinessLogic.Requests; +using FileProcessor.BusinessLogic.Services; +using FileProcessor.Models; +using MediatR; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shared.Results; +using Shouldly; +using SimpleResults; +using FileProcessor.Testing; +using FileProfileModel = global::FileProcessor.Models.FileProfile; +using Xunit; + +namespace FileProcessor.BusinessLogic.Tests; + +public class FileProfileDirectoryRecoveryServiceTests +{ + [Fact] + public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirectory() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + Mock fileProcessorManager = new(); + Mock mediator = new(); + MockFileSystem fileSystem = new(); + + FileProfileModel fileProfile = TestData.FileProfile; + string inProgressDirectory = $"{fileProfile.ListeningDirectory}/inprogress"; + string inProgressFilePath = $"{inProgressDirectory}/{TestData.EstateId:N}-{TestData.FileId:N}"; + + fileSystem.AddDirectory(inProgressDirectory); + fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); + + fileProcessorManager + .Setup(manager => manager.GetAllFileProfiles(It.IsAny())) + .ReturnsAsync(Result.Success(new List { fileProfile })); + + fileProcessorManager + .Setup(manager => manager.GetFile(TestData.FileId, TestData.EstateId, It.IsAny())) + .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails())); + + mediator + .Setup(send => send.Send(It.IsAny(), It.IsAny())) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Object, + mediator.Object, + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Verify(send => send.Send( + It.Is(command => + command.FileId == TestData.FileId && + Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath) && + command.FileProfileId == TestData.FileProfileId && + command.EstateId == TestData.EstateId), + It.IsAny()), Times.Once); + } +} diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs new file mode 100644 index 0000000..4a638ae --- /dev/null +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Threading; +using System.Threading.Tasks; +using FileProcessor.BusinessLogic.Managers; +using FileProcessor.BusinessLogic.Requests; +using FileProcessor.Models; +using MediatR; +using Shared.Results; +using SimpleResults; +using FileProfileModel = global::FileProcessor.Models.FileProfile; + +namespace FileProcessor.BusinessLogic.Services; + +public sealed class FileProfileDirectoryRecoveryService : IFileProfileDirectoryRecoveryService +{ + private const string InProgressDirectoryName = "inprogress"; + + private readonly IFileProcessorManager FileProcessorManager; + private readonly IMediator Mediator; + private readonly IFileSystem FileSystem; + + public FileProfileDirectoryRecoveryService( + IFileProcessorManager fileProcessorManager, + IMediator mediator, + IFileSystem fileSystem) + { + this.FileProcessorManager = fileProcessorManager; + this.Mediator = mediator; + this.FileSystem = fileSystem; + } + + public async Task RecoverInProgressFilesAsync(CancellationToken cancellationToken) + { + Result> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfiles(cancellationToken); + if (fileProfilesResult.IsFailed) + { + Shared.Logger.Logger.LogWarning($"Unable to load file profiles for in-progress recovery: {fileProfilesResult.Message}"); + return ResultHelpers.CreateFailure(fileProfilesResult); + } + + foreach (FileProfileModel fileProfile in fileProfilesResult.Data) + { + string inProgressDirectory = Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName); + if (this.FileSystem.Directory.Exists(inProgressDirectory) == false) + { + continue; + } + + foreach (string filePath in this.FileSystem.Directory.GetFiles(inProgressDirectory)) + { + await this.RecoverFileAsync(fileProfile, filePath, cancellationToken); + } + } + + return Result.Success(); + } + + private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePath, CancellationToken cancellationToken) + { + string fileName = Path.GetFileNameWithoutExtension(filePath); + if (TryParseIdentifiers(fileName, out Guid estateId, out Guid fileId) == false) + { + Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because the name does not contain recoverable identifiers"); + return; + } + + Result fileResult = await this.FileProcessorManager.GetFile(fileId, estateId, cancellationToken); + if (fileResult.IsFailed) + { + Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] could not be loaded: {fileResult.Message}"); + return; + } + + FileDetails fileDetails = fileResult.Data; + if (fileDetails == null) + { + Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] returned no details"); + return; + } + + FileCommands.ProcessUploadedFileCommand command = new( + fileDetails.EstateId, + fileDetails.MerchantId, + fileDetails.FileImportLogId, + fileDetails.FileId, + fileDetails.UserId, + filePath, + fileDetails.FileProfileId, + fileDetails.FileReceivedDateTime); + + Result result = await this.Mediator.Send(command, cancellationToken); + if (result.IsFailed) + { + Shared.Logger.Logger.LogWarning($"Recovery processing failed for leftover in-progress file [{filePath}]: {result.Message}"); + return; + } + + Shared.Logger.Logger.LogInformation($"Recovered leftover in-progress file [{filePath}] for file profile [{fileProfile.FileProfileId}]"); + } + + private static bool TryParseIdentifiers(string fileName, out Guid estateId, out Guid fileId) + { + estateId = Guid.Empty; + fileId = Guid.Empty; + + string[] parts = fileName.Split('-', 2, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2) + { + return false; + } + + return Guid.TryParse(parts[0], out estateId) && Guid.TryParse(parts[1], out fileId); + } +} diff --git a/FileProcessor.BusinessLogic/Services/IFileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/IFileProfileDirectoryRecoveryService.cs new file mode 100644 index 0000000..3171305 --- /dev/null +++ b/FileProcessor.BusinessLogic/Services/IFileProfileDirectoryRecoveryService.cs @@ -0,0 +1,10 @@ +using SimpleResults; +using System.Threading; +using System.Threading.Tasks; + +namespace FileProcessor.BusinessLogic.Services; + +public interface IFileProfileDirectoryRecoveryService +{ + Task RecoverInProgressFilesAsync(CancellationToken cancellationToken); +} diff --git a/FileProcessor.Testing/TestData.cs b/FileProcessor.Testing/TestData.cs index 64d5e5b..03bef31 100644 --- a/FileProcessor.Testing/TestData.cs +++ b/FileProcessor.Testing/TestData.cs @@ -278,6 +278,23 @@ public static FileAggregate GetCreatedFileAggregate() return fileAggregate; } + public static FileDetails GetCreatedFileDetails() + { + return new FileDetails + { + FileReceivedDateTime = TestData.FileUploadedDateTime, + ProcessingCompleted = false, + FileLines = new List(), + EstateId = TestData.EstateId, + MerchantId = TestData.MerchantId, + FileProfileId = TestData.FileProfileId, + FileId = TestData.FileId, + FileImportLogId = TestData.FileImportLogId, + FileLocation = TestData.FilePathWithName, + UserId = TestData.UserId + }; + } + public static FileAggregate GetFileAggregateWithLines() { FileAggregate fileAggregate = new FileAggregate(); diff --git a/FileProcessor/Bootstrapper/MiscRegistry.cs b/FileProcessor/Bootstrapper/MiscRegistry.cs index 21f3542..05b79e6 100644 --- a/FileProcessor/Bootstrapper/MiscRegistry.cs +++ b/FileProcessor/Bootstrapper/MiscRegistry.cs @@ -25,6 +25,7 @@ public MiscRegistry() this.AddSingleton(); this.AddSingleton(); this.AddSingleton(); + this.AddSingleton(); bool logRequests = ConfigurationReader.GetValueOrDefault("MiddlewareLogging", "LogRequests", true); bool logResponses = ConfigurationReader.GetValueOrDefault("MiddlewareLogging", "LogResponses", true); diff --git a/FileProcessor/Common/Extensions.cs b/FileProcessor/Common/Extensions.cs index b3063ec..6fee4d2 100644 --- a/FileProcessor/Common/Extensions.cs +++ b/FileProcessor/Common/Extensions.cs @@ -69,7 +69,7 @@ public static IServiceCollection AddInSecureEventStoreClient( public static void PreWarm(this IApplicationBuilder applicationBuilder){ IFileSystem fileSystem = Startup.Container.GetInstance(); IFileProfileDirectorySynchronizer directorySynchronizer = Startup.Container.GetInstance(); - // TODO: Do we poll here for files incase they have been left from a previous run + IFileProfileDirectoryRecoveryService recoveryService = Startup.Container.GetInstance(); var temporaryFileLocation = ConfigurationReader.GetValue("AppSettings", "TemporaryFileLocation"); Logger.LogInformation($"Starting up, TemporaryFileLocation is [{temporaryFileLocation}]"); @@ -82,6 +82,12 @@ public static void PreWarm(this IApplicationBuilder applicationBuilder){ throw new ApplicationStartupException(syncResult.Message); } + Result recoveryResult = recoveryService.RecoverInProgressFilesAsync(CancellationToken.None).GetAwaiter().GetResult(); + if (recoveryResult.IsFailed) + { + Logger.LogWarning($"Error recovering in-progress files {recoveryResult.Message}"); + } + TypeProvider.LoadDomainEventsTypeDynamically(); IConfigurationSection subscriptionConfigSection = Startup.Configuration.GetSection("AppSettings:SubscriptionConfiguration");