diff --git a/.gitignore b/.gitignore index c952369..d3b6d08 100644 --- a/.gitignore +++ b/.gitignore @@ -350,3 +350,5 @@ MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ +/.superpowers/sdd/2026-10-15-simple-upload-processing +/docs/superpowers diff --git a/FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs b/FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs index d021bd6..9bf39f5 100644 --- a/FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using FileProcessor.BusinessLogic.Requests; @@ -7,9 +10,12 @@ using MediatR; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.Extensions.Configuration; using Shouldly; using SimpleResults; using FileProcessor.Testing; +using Shared.General; +using Shared.Logger; using Xunit; namespace FileProcessor.BusinessLogic.Tests; @@ -29,4 +35,141 @@ public async Task ReplayLine_SendsReplayCommand_AndReturnsOk() response.StatusCode.ShouldBe(StatusCodes.Status204NoContent); mediator.Send(Arg>.Any(), Arg.Any()).Called(Count.Once()); } + + [Fact] + public async Task UploadFile_WhenEventStoreAcceptanceFails_DeletesTemporaryFile() + { + string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temporaryDirectory); + + try + { + Dictionary settings = new(TestData.DefaultAppSettings) + { + ["AppSettings:TemporaryFileLocation"] = temporaryDirectory + }; + IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + ConfigurationReader.Initialise(configuration); + Logger.Initialise(NullLogger.Instance); + + IMediatorImposter mediator = new(); + mediator.Send(Arg>.Any(), Arg.Any()).ReturnsAsync(Result.Failure()); + + await using MemoryStream content = new("D,1,1,1"u8.ToArray()); + FormFile formFile = new(content, 0, content.Length, "file", "source.csv") + { + Headers = new HeaderDictionary + { + ["Content-Disposition"] = "form-data; name=\"file\"; filename=\"source.csv\"" + } + }; + FormFileCollection files = new() { formFile }; + FormCollection formCollection = new(new Dictionary(), files); + + await FileHandlers.UploadFile( + mediator.Instance(), + new FileProcessor.DataTransferObjects.UploadFileRequest + { + FileId = TestData.FileId, + EstateId = TestData.EstateId, + MerchantId = TestData.MerchantId, + UserId = TestData.UserId, + FileProfileId = TestData.FileProfileId + }, + formCollection, + CancellationToken.None); + + Directory.GetFiles(temporaryDirectory).ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(temporaryDirectory)) + Directory.Delete(temporaryDirectory, true); + } + } + + [Fact] + public async Task UploadFile_UsesUniqueTemporaryPathForRepeatedClientFilename() + { + string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temporaryDirectory); + + try + { + Dictionary settings = new(TestData.DefaultAppSettings) + { + ["AppSettings:TemporaryFileLocation"] = temporaryDirectory + }; + IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + ConfigurationReader.Initialise(configuration); + Logger.Initialise(NullLogger.Instance); + + IMediatorImposter mediator = new(); + mediator.Send(Arg>.Any(), Arg.Any()).ReturnsAsync(Result.Success()); + + await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection(), CancellationToken.None); + await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection(), CancellationToken.None); + + Directory.GetFiles(temporaryDirectory).Length.ShouldBe(2); + } + finally + { + if (Directory.Exists(temporaryDirectory)) + Directory.Delete(temporaryDirectory, true); + } + } + + [Fact] + public async Task UploadFile_DoesNotUseClientFilenameCharactersInTemporaryPath() + { + string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}"); + Directory.CreateDirectory(temporaryDirectory); + + try + { + Dictionary settings = new(TestData.DefaultAppSettings) + { + ["AppSettings:TemporaryFileLocation"] = temporaryDirectory + }; + IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + ConfigurationReader.Initialise(configuration); + Logger.Initialise(NullLogger.Instance); + + IMediatorImposter mediator = new(); + mediator.Send(Arg>.Any(), Arg.Any()).ReturnsAsync(Result.Success()); + + await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection("invoice.csv:payload"), CancellationToken.None); + + string storedFileName = Path.GetFileName(Directory.GetFiles(temporaryDirectory).Single()); + storedFileName.ShouldNotContain(":"); + Path.GetExtension(storedFileName).ShouldBeEmpty(); + } + finally + { + if (Directory.Exists(temporaryDirectory)) + Directory.Delete(temporaryDirectory, true); + } + } + + private static FileProcessor.DataTransferObjects.UploadFileRequest CreateUploadRequest() => new() + { + FileId = TestData.FileId, + EstateId = TestData.EstateId, + MerchantId = TestData.MerchantId, + UserId = TestData.UserId, + FileProfileId = TestData.FileProfileId + }; + + private static FormCollection CreateFormCollection(string fileName = "source.csv") + { + MemoryStream content = new("D,1,1,1"u8.ToArray()); + FormFile formFile = new(content, 0, content.Length, "file", fileName) + { + Headers = new HeaderDictionary + { + ["Content-Disposition"] = $"form-data; name=\"file\"; filename=\"{fileName}\"" + } + }; + return new FormCollection(new Dictionary(), new FormFileCollection { formFile }); + } } diff --git a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs index 021ba88..f27d74b 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs @@ -104,6 +104,33 @@ public async Task FileRequestHandler_UploadFileRequest_RequestIsHandled() result.IsSuccess.ShouldBeTrue(); } + [Fact] + public async Task UploadFile_DoesNotRequireListeningDirectoryOrMoveTemporaryFile() + { + this.FileProcessorManager.GetFileProfile(Arg.Any(), Arg.Any()).ReturnsAsync(TestData.FileProfileSafaricom); + + FileImportLogAggregate importLogAggregate = TestData.GetEmptyFileImportLogAggregate(); + this.FileImportLogAggregateRepository.GetLatestVersion(Arg.Any(), Arg.Any()).ReturnsAsync(Result.Success(importLogAggregate)); + this.FileImportLogAggregateRepository.SaveChanges(Arg.Any(), Arg.Any()).ReturnsAsync(Result.Success()); + + var file = new MockFileData("D,1,1,1"); + var fileId = FileProcessorDomainService.CreateGuidFromFileData(Encoding.UTF8.GetString(file.Contents)); + this.FileSystem.AddFile(TestData.FilePathWithName, file); + + FileCommands.UploadFileCommand command = TestData.UploadFileCommand with + { + FileId = fileId, + OriginalFileName = "client-name.csv" + }; + + Result result = await this.FileProcessorDomainService.UploadFile(command, CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + this.FileSystem.File.Exists(TestData.FilePathWithName).ShouldBeTrue(); + importLogAggregate.GetFileImportLog().Files.Single().FilePath.ShouldBe(TestData.FilePathWithName); + importLogAggregate.GetFileImportLog().Files.Single().OriginalFileName.ShouldBe("client-name.csv"); + } + [Fact] public async Task FileRequestHandler_UploadFileRequest_SaveFailed_RequestIsHandled() { @@ -203,22 +230,6 @@ public async Task FileRequestHandler_UploadFileRequest_FileNotFound_RequestIsHan result.Status.ShouldBe(ResultStatus.NotFound); } - [Fact] - public async Task FileRequestHandler_UploadFileRequest_DestinationDirectoryNotFound_RequestIsHandled() - { - this.FileProcessorManager.GetFileProfile(Arg.Any(), Arg.Any()).ReturnsAsync(TestData.FileProfileSafaricom); - - this.FileImportLogAggregateRepository.GetLatestVersion(Arg.Any(), Arg.Any()).ReturnsAsync(Result.Success(TestData.GetCreatedFileImportLogAggregate())); - - this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1")); - - Logger.Initialise(NullLogger.Instance); - - Result result = await this.FileProcessorDomainService.UploadFile(TestData.UploadFileCommand, CancellationToken.None); - result.IsFailed.ShouldBeTrue(); - result.Status.ShouldBe(ResultStatus.NotFound); - } - [Fact] public async Task FileRequestHandler_ProcessUploadedFileRequest_RequestIsHandled() { @@ -241,6 +252,19 @@ public async Task FileRequestHandler_ProcessUploadedFileRequest_RequestIsHandled Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None); result.IsSuccess.ShouldBeTrue(); } + + [Fact] + public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileToFailedDirectory() + { + this.FileProcessorManager.GetFileProfile(Arg.Any(), Arg.Any()).ReturnsAsync(TestData.FileProfileSafaricom); + this.SecurityServiceClient.GetToken(Arg.Any(), Arg.Any(), Arg.Any()).ReturnsAsync(Result.Failure()); + this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1")); + + Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None); + + result.IsFailed.ShouldBeTrue(); + this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue(); + } [Fact] public async Task FileRequestHandler_ProcessUploadedFileRequest_FileNotFound_RequestIsHandled() diff --git a/FileProcessor.BusinessLogic/Requests/FileCommands.cs b/FileProcessor.BusinessLogic/Requests/FileCommands.cs index da4364f..19cbb2e 100644 --- a/FileProcessor.BusinessLogic/Requests/FileCommands.cs +++ b/FileProcessor.BusinessLogic/Requests/FileCommands.cs @@ -25,9 +25,10 @@ public record UploadFileCommand(Guid FileId, Guid UserId, String FilePath, Guid FileProfileId, - DateTime FileUploadedDateTime) : IRequest; + DateTime FileUploadedDateTime, + String OriginalFileName = null) : IRequest; public record ReplayFileLineCommand(Guid FileId, Guid EstateId, Int32 LineNumber) : IRequest; -} \ No newline at end of file +} diff --git a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs index 1f730c7..17815bb 100644 --- a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs @@ -162,28 +162,21 @@ public async Task UploadFile(FileCommands.UploadFileCommand command, } } - // Move the file Result getFileProfileResult = await this.FileProcessorManager.GetFileProfile(command.FileProfileId, cancellationToken); - if (getFileProfileResult.IsFailed) { + if (getFileProfileResult.IsFailed) return ResultHelpers.CreateFailure(getFileProfileResult); - } - FileProfileModel fileProfile = getFileProfileResult.Data; - if (fileProfile == null) { + if (getFileProfileResult.Data == null) return Result.NotFound($"No file profile found with Id {command.FileProfileId}"); - } - // Copy file from the temp location to file processing listening directory IFileInfo file = this.FileSystem.FileInfo.New(command.FilePath); if (file.Exists == false) { return Result.NotFound($"File {file.FullName} not found"); } - String originalName = file.Name; - - if (this.FileSystem.Directory.Exists(fileProfile.ListeningDirectory) == false) { - return Result.NotFound($"Directory {fileProfile.ListeningDirectory} not found"); - } + String originalName = String.IsNullOrWhiteSpace(command.OriginalFileName) + ? file.Name + : command.OriginalFileName; // Read the file data String fileContent = null; @@ -201,11 +194,8 @@ public async Task UploadFile(FileCommands.UploadFileCommand command, return Result.Invalid("File content does not match the file id provided"); } - String fileDestination = $"{fileProfile.ListeningDirectory}//{command.EstateId:N}-{command.FileId:N}"; - file.MoveTo(fileDestination, overwrite: true); - // Update Import log aggregate - Result stateResult= fileImportLogAggregate.AddImportedFile(command.FileId, command.MerchantId, command.UserId, command.FileProfileId, originalName, fileDestination, command.FileUploadedDateTime); + Result stateResult= fileImportLogAggregate.AddImportedFile(command.FileId, command.MerchantId, command.UserId, command.FileProfileId, originalName, command.FilePath, command.FileUploadedDateTime); if (stateResult.IsFailed) return stateResult; @@ -263,14 +253,34 @@ public async Task ProcessUploadedFile(FileCommands.ProcessUploadedFileCo }, command.FileId, cancellationToken, false ); - if (result.IsFailed) + if (result.IsFailed) { + await MoveAcceptedFileToFailedDirectory(command.FileProfileId, command.FilePath, cancellationToken); return result; + } Result processResult = await this.ProcessFile(command.FileId, command.FileProfileId, command.FilePath, cancellationToken); return processResult; } + private async Task MoveAcceptedFileToFailedDirectory(Guid fileProfileId, + String filePath, + CancellationToken cancellationToken) + { + Result fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken); + if (fileProfileResult.IsFailed || fileProfileResult.Data == null) + return; + + IFileInfo file = this.FileSystem.FileInfo.New(filePath); + if (file.Exists == false) + return; + + if (this.FileSystem.Directory.Exists(fileProfileResult.Data.FailedDirectory) == false) + this.FileSystem.Directory.CreateDirectory(fileProfileResult.Data.FailedDirectory); + + file.MoveTo($"{fileProfileResult.Data.FailedDirectory}/{file.Name}", overwrite: true); + } + private async Task> GetOperatorIdForFileProfile(Guid estateId, Guid fileProfileId, CancellationToken cancellationToken){ Result fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken); diff --git a/FileProcessor/Handlers/FileHandlers.cs b/FileProcessor/Handlers/FileHandlers.cs index b5a640c..58e10be 100644 --- a/FileProcessor/Handlers/FileHandlers.cs +++ b/FileProcessor/Handlers/FileHandlers.cs @@ -29,31 +29,57 @@ public static async Task UploadFile(IMediator mediator, string temporaryFileLocation = ConfigurationReader.GetValue("AppSettings", "TemporaryFileLocation"); string fileName = ContentDispositionHeaderValue.Parse(file.ContentDisposition).FileName.Trim('"'); - string fullPath = Path.Combine(temporaryFileLocation, fileName); + string temporaryFileName = Guid.NewGuid().ToString("N"); + string fullPath = Path.Combine(temporaryFileLocation, temporaryFileName); + bool uploadAccepted = false; + bool temporaryFileCreated = false; - await using (FileStream stream = new(fullPath, FileMode.Create)) + try { - await file.CopyToAsync(stream, cancellationToken); - } + Directory.CreateDirectory(temporaryFileLocation); + + await using (FileStream stream = new(fullPath, FileMode.CreateNew)) + { + temporaryFileCreated = true; + await file.CopyToAsync(stream, cancellationToken); + } - if (request.UploadDateTime == DateTime.MinValue) - request.UploadDateTime = DateTime.Now; + if (request.UploadDateTime == DateTime.MinValue) + request.UploadDateTime = DateTime.Now; - FileCommands.UploadFileCommand command = new(request.FileId, - request.EstateId, - request.MerchantId, - request.UserId, - fullPath, - request.FileProfileId, - request.UploadDateTime); + FileCommands.UploadFileCommand command = new(request.FileId, + request.EstateId, + request.MerchantId, + request.UserId, + fullPath, + request.FileProfileId, + request.UploadDateTime, + fileName); - Result result = await mediator.Send(command, cancellationToken); + Result result = await mediator.Send(command, cancellationToken); + uploadAccepted = result.IsSuccess; - Shared.Logger.Logger.LogDebug($"Day is {request.UploadDateTime.Day}"); - Shared.Logger.Logger.LogDebug($"Month is {request.UploadDateTime.Month}"); - Shared.Logger.Logger.LogDebug($"Year is {request.UploadDateTime.Year}"); + Shared.Logger.Logger.LogDebug($"Day is {request.UploadDateTime.Day}"); + Shared.Logger.Logger.LogDebug($"Month is {request.UploadDateTime.Month}"); + Shared.Logger.Logger.LogDebug($"Year is {request.UploadDateTime.Year}"); - return ResponseFactory.FromResult(result, r => r); + return ResponseFactory.FromResult(result, r => r); + } + finally + { + if (uploadAccepted == false && temporaryFileCreated) + { + try + { + if (System.IO.File.Exists(fullPath)) + System.IO.File.Delete(fullPath); + } + catch (Exception exception) + { + Shared.Logger.Logger.LogWarning($"Failed to remove temporary upload file [{fullPath}]. Error [{exception.Message}]"); + } + } + } } public static async Task GetFile(IMediator mediator, @@ -78,4 +104,4 @@ public static async Task ReplayLine(IMediator mediator, return ResponseFactory.FromResult(result); } -} \ No newline at end of file +}