From 02c81471721b39c5f632b36812c2087d94c74fa2 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 25 Sep 2026 10:23:41 +0100 Subject: [PATCH 1/2] recover in-progress files without file aggregate --- ...ileProfileDirectoryRecoveryServiceTests.cs | 74 +++++++++++++++++++ .../Common/ModelFactory.cs | 5 +- .../FileProfileDirectoryRecoveryService.cs | 70 +++++++++++++----- 3 files changed, 129 insertions(+), 20 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs index ace7a0b..ee2668a 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -16,6 +16,7 @@ using SimpleResults; using FileProcessor.Testing; using FileProfileModel = global::FileProcessor.Models.FileProfile; +using FileImportLogModel = global::FileProcessor.Models.FileImportLog; using Xunit; namespace FileProcessor.BusinessLogic.Tests; @@ -66,4 +67,77 @@ public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirec command.EstateId == TestData.EstateId), Arg.Any()).Called(Count.Once()); } + + [Fact] + public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_ReplaysUsingImportLogDetails() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + IFileProcessorManagerImposter fileProcessorManager = new(); + IMediatorImposter mediator = new(); + MockFileSystem fileSystem = new(); + + FileProfileModel fileProfile = TestData.FileProfile; + string inProgressDirectory = $"{fileProfile.ListeningDirectory}/inprogress"; + string inProgressFilePath = $"{inProgressDirectory}/{TestData.EstateId:N}-{TestData.FileId:N}"; + + fileSystem.AddDirectory(inProgressDirectory); + fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); + + fileProcessorManager + .GetAllFileProfiles(Arg.Any()) + .ReturnsAsync(Result.Success(new List { fileProfile })); + + fileProcessorManager + .GetFile(TestData.FileId, TestData.EstateId, Arg.Any()) + .ReturnsAsync(Result.NotFound("File aggregate not found")); + + fileProcessorManager + .GetFileImportLogs(TestData.EstateId, Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(new List + { + new() + { + EstateId = TestData.EstateId, + FileImportLogId = TestData.FileImportLogId, + FileImportLogDateTime = TestData.FileUploadedDateTime, + Files = new List + { + new() + { + EstateId = TestData.EstateId, + FileId = TestData.FileId, + FileProfileId = TestData.FileProfileId, + MerchantId = TestData.MerchantId, + UserId = TestData.UserId, + UploadedDateTime = TestData.FileUploadedDateTime + } + } + } + })); + + mediator + .Send(Arg>.Any(), Arg.Any()) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Instance(), + mediator.Instance(), + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Send( + Arg>.Is(request => request is FileCommands.ProcessUploadedFileCommand command && + command.FileId == TestData.FileId && + command.FileImportLogId == TestData.FileImportLogId && + command.FileProfileId == TestData.FileProfileId && + command.EstateId == TestData.EstateId && + command.MerchantId == TestData.MerchantId && + command.UserId == TestData.UserId && + command.FileUploadedDateTime == TestData.FileUploadedDateTime && + Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath)), + Arg.Any()).Called(Count.Once()); + } } diff --git a/FileProcessor.BusinessLogic/Common/ModelFactory.cs b/FileProcessor.BusinessLogic/Common/ModelFactory.cs index f0520ef..0dfcde9 100644 --- a/FileProcessor.BusinessLogic/Common/ModelFactory.cs +++ b/FileProcessor.BusinessLogic/Common/ModelFactory.cs @@ -49,7 +49,8 @@ public List ConvertFrom(Guid estateId, FilePath = importLogFile.Item1.FilePath, FileProfileId = importLogFile.Item1.FileProfileId, OriginalFileName = importLogFile.Item1.OriginalFileName, - UserId = importLogFile.Item1.UserId + UserId = importLogFile.Item1.UserId, + UploadedDateTime = importLogFile.Item1.FileUploadedDateTime }); } @@ -97,4 +98,4 @@ public FileImportLog ConvertFrom(Guid estateId, #endregion } -} \ No newline at end of file +} diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs index 4a638ae..8ccdd5d 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -11,6 +11,7 @@ using Shared.Results; using SimpleResults; using FileProfileModel = global::FileProcessor.Models.FileProfile; +using FileImportLogModel = global::FileProcessor.Models.FileImportLog; namespace FileProcessor.BusinessLogic.Services; @@ -68,28 +69,61 @@ private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePat } Result fileResult = await this.FileProcessorManager.GetFile(fileId, estateId, cancellationToken); - if (fileResult.IsFailed) - { - Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] could not be loaded: {fileResult.Message}"); - return; - } + FileCommands.ProcessUploadedFileCommand command; - FileDetails fileDetails = fileResult.Data; - if (fileDetails == null) + if (fileResult.IsSuccess && fileResult.Data != null) { - Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] returned no details"); - return; + FileDetails fileDetails = fileResult.Data; + command = new( + fileDetails.EstateId, + fileDetails.MerchantId, + fileDetails.FileImportLogId, + fileId, + fileDetails.UserId, + filePath, + fileDetails.FileProfileId, + fileDetails.FileReceivedDateTime); } + else + { + Result> importLogsResult = await this.FileProcessorManager.GetFileImportLogs( + estateId, + DateTime.MinValue, + DateTime.MaxValue, + null, + cancellationToken); + + ImportLogFile importLogFile = null; + FileImportLogModel importLog = null; + if (importLogsResult.IsSuccess && importLogsResult.Data != null) + { + foreach (FileImportLogModel candidate in importLogsResult.Data) + { + importLogFile = candidate.Files?.Find(file => file.FileId == fileId); + if (importLogFile != null) + { + importLog = candidate; + break; + } + } + } - FileCommands.ProcessUploadedFileCommand command = new( - fileDetails.EstateId, - fileDetails.MerchantId, - fileDetails.FileImportLogId, - fileDetails.FileId, - fileDetails.UserId, - filePath, - fileDetails.FileProfileId, - fileDetails.FileReceivedDateTime); + if (importLog == null || importLogFile == null) + { + Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] could not be loaded from the file aggregate or import log read model"); + return; + } + + command = new( + estateId, + importLogFile.MerchantId, + importLog.FileImportLogId, + fileId, + importLogFile.UserId, + filePath, + importLogFile.FileProfileId, + importLogFile.UploadedDateTime == default ? importLog.FileImportLogDateTime : importLogFile.UploadedDateTime); + } Result result = await this.Mediator.Send(command, cancellationToken); if (result.IsFailed) From 706e1854f1f38ecea565ddf2038c43db7c2d43b9 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 25 Sep 2026 10:36:39 +0100 Subject: [PATCH 2/2] address in-progress recovery review feedback --- .../DummyFileProcessorDomainService.cs | 6 +++ .../FileProcessingManagerTests.cs | 16 ++++++++ ...ileProfileDirectoryRecoveryServiceTests.cs | 29 +++++++------- .../Managers/FileProcessorManager.cs | 39 +++++++++++++++++++ .../Managers/IFileProcessorManager.cs | 2 + .../FileProfileDirectoryRecoveryService.cs | 24 +++--------- 6 files changed, 81 insertions(+), 35 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs index 6be3735..74f49e0 100644 --- a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs @@ -51,6 +51,12 @@ public async Task> GetFileProfile(Guid fileProfileId, return Result.Success(); } + public async Task> GetFileImportLogForFile(Guid fileId, + Guid estateId, + CancellationToken cancellationToken) { + return Result.Success(); + } + public async Task> GetFile(Guid fileId, Guid estateId, CancellationToken cancellationToken) { diff --git a/FileProcessor.BusinessLogic.Tests/FileProcessingManagerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProcessingManagerTests.cs index 1a0d7c9..e7bf07b 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProcessingManagerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProcessingManagerTests.cs @@ -151,6 +151,22 @@ public async Task FileProcessingManager_GetFileImportLog_WithMerchantId_ImportLo this.VerifyImportLog(TestData.FileImportLogs.First(),importLog, TestData.MerchantId); } + [Fact] + public async Task FileProcessingManager_GetFileImportLogForFile_ReturnsEntryWithoutFileReadModelRow() + { + this.Context.FileImportLogs.Add(TestData.FileImportLog1); + this.Context.FileImportLogFiles.Add(TestData.FileImportLog1Files.First()); + await this.Context.SaveChangesAsync(); + + Result importLog = await this.Manager.GetFileImportLogForFile(TestData.FileId1, TestData.EstateId, CancellationToken.None); + + importLog.IsSuccess.ShouldBeTrue(); + importLog.Data.FileImportLogId.ShouldBe(TestData.FileImportLogId1); + importLog.Data.Files.ShouldHaveSingleItem(); + importLog.Data.Files.Single().FileId.ShouldBe(TestData.FileId1); + importLog.Data.Files.Single().UploadedDateTime.ShouldBe(TestData.FileUploadedDateTime); + } + [Fact] public async Task FileProcessingManager_GetFile_FileReturned() { diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs index ee2668a..b8cd57b 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -93,25 +93,22 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays .ReturnsAsync(Result.NotFound("File aggregate not found")); fileProcessorManager - .GetFileImportLogs(TestData.EstateId, Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .ReturnsAsync(Result.Success(new List + .GetFileImportLogForFile(TestData.FileId, TestData.EstateId, Arg.Any()) + .ReturnsAsync(Result.Success(new FileImportLogModel { - new() + EstateId = TestData.EstateId, + FileImportLogId = TestData.FileImportLogId, + FileImportLogDateTime = TestData.FileUploadedDateTime, + Files = new List { - EstateId = TestData.EstateId, - FileImportLogId = TestData.FileImportLogId, - FileImportLogDateTime = TestData.FileUploadedDateTime, - Files = new List + new() { - new() - { - EstateId = TestData.EstateId, - FileId = TestData.FileId, - FileProfileId = TestData.FileProfileId, - MerchantId = TestData.MerchantId, - UserId = TestData.UserId, - UploadedDateTime = TestData.FileUploadedDateTime - } + EstateId = TestData.EstateId, + FileId = TestData.FileId, + FileProfileId = TestData.FileProfileId, + MerchantId = TestData.MerchantId, + UserId = TestData.UserId, + UploadedDateTime = TestData.FileUploadedDateTime } } })); diff --git a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs index b33db6d..e8fbeda 100644 --- a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs @@ -162,6 +162,45 @@ join merchant in context.Merchants on file.MerchantId equals merchant.MerchantId return Result.Success(x); } + public async Task> GetFileImportLogForFile(Guid fileId, + Guid estateId, + CancellationToken cancellationToken) + { + EstateManagementContext context = await this.GetContext(estateId); + + var importLogFile = await (from fileImportLogFile in context.FileImportLogFiles + join importLog in context.FileImportLogs on fileImportLogFile.FileImportLogId equals importLog.FileImportLogId + where fileImportLogFile.FileId == fileId + select new { fileImportLogFile, importLog }) + .SingleOrDefaultAsync(cancellationToken); + + if (importLogFile == null) + { + return Result.NotFound($"No import log file found for File Id [{fileId}]"); + } + + return Result.Success(new FileImportLog + { + EstateId = estateId, + FileImportLogId = importLogFile.importLog.FileImportLogId, + FileImportLogDateTime = importLogFile.importLog.ImportLogDateTime, + Files = new List + { + new() + { + EstateId = estateId, + FileId = importLogFile.fileImportLogFile.FileId, + FilePath = importLogFile.fileImportLogFile.FilePath, + FileProfileId = importLogFile.fileImportLogFile.FileProfileId, + MerchantId = importLogFile.fileImportLogFile.MerchantId, + OriginalFileName = importLogFile.fileImportLogFile.OriginalFileName, + UploadedDateTime = importLogFile.fileImportLogFile.FileUploadedDateTime, + UserId = importLogFile.fileImportLogFile.UserId + } + } + }); + } + public async Task> GetFile(Guid fileId, Guid estateId, CancellationToken cancellationToken) diff --git a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs index dad8041..fb3a3aa 100644 --- a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs @@ -24,6 +24,8 @@ public interface IFileProcessorManager Task> GetFileImportLog(Guid fileImportLogId, Guid estateId, Guid? merchantId, CancellationToken cancellationToken); + Task> GetFileImportLogForFile(Guid fileId, Guid estateId, CancellationToken cancellationToken); + Task> GetFile(Guid fileId, Guid estateId, CancellationToken cancellationToken); #endregion diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs index 8ccdd5d..bd40135 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -86,29 +86,15 @@ private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePat } else { - Result> importLogsResult = await this.FileProcessorManager.GetFileImportLogs( + Result importLogResult = await this.FileProcessorManager.GetFileImportLogForFile( + fileId, estateId, - DateTime.MinValue, - DateTime.MaxValue, - null, cancellationToken); - ImportLogFile importLogFile = null; - FileImportLogModel importLog = null; - if (importLogsResult.IsSuccess && importLogsResult.Data != null) - { - foreach (FileImportLogModel candidate in importLogsResult.Data) - { - importLogFile = candidate.Files?.Find(file => file.FileId == fileId); - if (importLogFile != null) - { - importLog = candidate; - break; - } - } - } + FileImportLogModel importLog = importLogResult.Data; + ImportLogFile importLogFile = importLog?.Files?.Find(file => file.FileId == fileId); - if (importLog == null || importLogFile == null) + if (importLogResult.IsFailed || importLog == null || importLogFile == null) { Shared.Logger.Logger.LogWarning($"Skipping leftover in-progress file [{filePath}] because file [{fileId}] could not be loaded from the file aggregate or import log read model"); return;