Fix upload temporary file handling

.gitignore

@@ -350,3 +350,5 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+/.superpowers/sdd/2026-10-15-simple-upload-processing
+/docs/superpowers

FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs

@@ -1,4 +1,7 @@
 using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using FileProcessor.BusinessLogic.Requests;
@@ -7,9 +10,12 @@
 using MediatR;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.HttpResults;
+using Microsoft.Extensions.Configuration;
 using Shouldly;
 using SimpleResults;
 using FileProcessor.Testing;
+using Shared.General;
+using Shared.Logger;
 using Xunit;
 
 namespace FileProcessor.BusinessLogic.Tests;
@@ -29,4 +35,141 @@ public async Task ReplayLine_SendsReplayCommand_AndReturnsOk()
         response.StatusCode.ShouldBe(StatusCodes.Status204NoContent);
         mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).Called(Count.Once());
     }
+
+    [Fact]
+    public async Task UploadFile_WhenEventStoreAcceptanceFails_DeletesTemporaryFile()
+    {
+        string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(temporaryDirectory);
+
+        try
+        {
+            Dictionary<string, string> settings = new(TestData.DefaultAppSettings)
+            {
+                ["AppSettings:TemporaryFileLocation"] = temporaryDirectory
+            };
+            IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
+            ConfigurationReader.Initialise(configuration);
+            Logger.Initialise(NullLogger.Instance);
+
+            IMediatorImposter mediator = new();
+            mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Failure());
+
+            await using MemoryStream content = new("D,1,1,1"u8.ToArray());
+            FormFile formFile = new(content, 0, content.Length, "file", "source.csv")
+            {
+                Headers = new HeaderDictionary
+                {
+                    ["Content-Disposition"] = "form-data; name=\"file\"; filename=\"source.csv\""
+                }
+            };
+            FormFileCollection files = new() { formFile };
+            FormCollection formCollection = new(new Dictionary<string, Microsoft.Extensions.Primitives.StringValues>(), files);
+
+            await FileHandlers.UploadFile(
+                mediator.Instance(),
+                new FileProcessor.DataTransferObjects.UploadFileRequest
+                {
+                    FileId = TestData.FileId,
+                    EstateId = TestData.EstateId,
+                    MerchantId = TestData.MerchantId,
+                    UserId = TestData.UserId,
+                    FileProfileId = TestData.FileProfileId
+                },
+                formCollection,
+                CancellationToken.None);
+
+            Directory.GetFiles(temporaryDirectory).ShouldBeEmpty();
+        }
+        finally
+        {
+            if (Directory.Exists(temporaryDirectory))
+                Directory.Delete(temporaryDirectory, true);
+        }
+    }
+
+    [Fact]
+    public async Task UploadFile_UsesUniqueTemporaryPathForRepeatedClientFilename()
+    {
+        string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(temporaryDirectory);
+
+        try
+        {
+            Dictionary<string, string> settings = new(TestData.DefaultAppSettings)
+            {
+                ["AppSettings:TemporaryFileLocation"] = temporaryDirectory
+            };
+            IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
+            ConfigurationReader.Initialise(configuration);
+            Logger.Initialise(NullLogger.Instance);
+
+            IMediatorImposter mediator = new();
+            mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
+
+            await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection(), CancellationToken.None);
+            await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection(), CancellationToken.None);
+
+            Directory.GetFiles(temporaryDirectory).Length.ShouldBe(2);
+        }
+        finally
+        {
+            if (Directory.Exists(temporaryDirectory))
+                Directory.Delete(temporaryDirectory, true);
+        }
+    }
+
+    [Fact]
+    public async Task UploadFile_DoesNotUseClientFilenameCharactersInTemporaryPath()
+    {
+        string temporaryDirectory = Path.Combine(Path.GetTempPath(), $"file-processor-tests-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(temporaryDirectory);
+
+        try
+        {
+            Dictionary<string, string> settings = new(TestData.DefaultAppSettings)
+            {
+                ["AppSettings:TemporaryFileLocation"] = temporaryDirectory
+            };
+            IConfigurationRoot configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build();
+            ConfigurationReader.Initialise(configuration);
+            Logger.Initialise(NullLogger.Instance);
+
+            IMediatorImposter mediator = new();
+            mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
+
+            await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection("invoice.csv:payload"), CancellationToken.None);
+
+            string storedFileName = Path.GetFileName(Directory.GetFiles(temporaryDirectory).Single());
+            storedFileName.ShouldNotContain(":");
+            Path.GetExtension(storedFileName).ShouldBeEmpty();
+        }
+        finally
+        {
+            if (Directory.Exists(temporaryDirectory))
+                Directory.Delete(temporaryDirectory, true);
+        }
+    }
+
+    private static FileProcessor.DataTransferObjects.UploadFileRequest CreateUploadRequest() => new()
+    {
+        FileId = TestData.FileId,
+        EstateId = TestData.EstateId,
+        MerchantId = TestData.MerchantId,
+        UserId = TestData.UserId,
+        FileProfileId = TestData.FileProfileId
+    };
+
+    private static FormCollection CreateFormCollection(string fileName = "source.csv")
+    {
+        MemoryStream content = new("D,1,1,1"u8.ToArray());
+        FormFile formFile = new(content, 0, content.Length, "file", fileName)
+        {
+            Headers = new HeaderDictionary
+            {
+                ["Content-Disposition"] = $"form-data; name=\"file\"; filename=\"{fileName}\""
+            }
+        };
+        return new FormCollection(new Dictionary<string, Microsoft.Extensions.Primitives.StringValues>(), new FormFileCollection { formFile });
+    }
 }

FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs

@@ -104,6 +104,33 @@ public async Task FileRequestHandler_UploadFileRequest_RequestIsHandled()
         result.IsSuccess.ShouldBeTrue();
     }
 
+    [Fact]
+    public async Task UploadFile_DoesNotRequireListeningDirectoryOrMoveTemporaryFile()
+    {
+        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
+
+        FileImportLogAggregate importLogAggregate = TestData.GetEmptyFileImportLogAggregate();
+        this.FileImportLogAggregateRepository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success(importLogAggregate));
+        this.FileImportLogAggregateRepository.SaveChanges(Arg<FileImportLogAggregate>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
+
+        var file = new MockFileData("D,1,1,1");
+        var fileId = FileProcessorDomainService.CreateGuidFromFileData(Encoding.UTF8.GetString(file.Contents));
+        this.FileSystem.AddFile(TestData.FilePathWithName, file);
+
+        FileCommands.UploadFileCommand command = TestData.UploadFileCommand with
+        {
+            FileId = fileId,
+            OriginalFileName = "client-name.csv"
+        };
+
+        Result result = await this.FileProcessorDomainService.UploadFile(command, CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        this.FileSystem.File.Exists(TestData.FilePathWithName).ShouldBeTrue();
+        importLogAggregate.GetFileImportLog().Files.Single().FilePath.ShouldBe(TestData.FilePathWithName);
+        importLogAggregate.GetFileImportLog().Files.Single().OriginalFileName.ShouldBe("client-name.csv");
+    }
+
     [Fact]
     public async Task FileRequestHandler_UploadFileRequest_SaveFailed_RequestIsHandled()
     {
@@ -203,22 +230,6 @@ public async Task FileRequestHandler_UploadFileRequest_FileNotFound_RequestIsHan
         result.Status.ShouldBe(ResultStatus.NotFound);
     }
 
-    [Fact]
-    public async Task FileRequestHandler_UploadFileRequest_DestinationDirectoryNotFound_RequestIsHandled()
-    {
-        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
-
-        this.FileImportLogAggregateRepository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success(TestData.GetCreatedFileImportLogAggregate()));
-
-        this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1"));
-
-        Logger.Initialise(NullLogger.Instance);
-
-        Result<Guid> result = await this.FileProcessorDomainService.UploadFile(TestData.UploadFileCommand, CancellationToken.None);
-        result.IsFailed.ShouldBeTrue();
-        result.Status.ShouldBe(ResultStatus.NotFound);
-    }
-
     [Fact]
     public async Task FileRequestHandler_ProcessUploadedFileRequest_RequestIsHandled()
     {
@@ -241,6 +252,19 @@ public async Task FileRequestHandler_ProcessUploadedFileRequest_RequestIsHandled
         Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None);
         result.IsSuccess.ShouldBeTrue();
     }
+
+    [Fact]
+    public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileToFailedDirectory()
+    {
+        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
+        this.SecurityServiceClient.GetToken(Arg<String>.Any(), Arg<String>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Failure());
+        this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1"));
+
+        Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None);
+
+        result.IsFailed.ShouldBeTrue();
+        this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
+    }
 
     [Fact]
     public async Task FileRequestHandler_ProcessUploadedFileRequest_FileNotFound_RequestIsHandled()

FileProcessor.BusinessLogic/Requests/FileCommands.cs

@@ -25,9 +25,10 @@ public record UploadFileCommand(Guid FileId,
                                     Guid UserId,
                                     String FilePath,
                                     Guid FileProfileId,
-                                    DateTime FileUploadedDateTime) : IRequest<Result>;
+                                    DateTime FileUploadedDateTime,
+                                    String OriginalFileName = null) : IRequest<Result>;
 
     public record ReplayFileLineCommand(Guid FileId,
                                     Guid EstateId,
                                     Int32 LineNumber) : IRequest<Result>;
-}
+}

FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs

@@ -162,28 +162,21 @@ public async Task<Result> UploadFile(FileCommands.UploadFileCommand command,
                 }
             }
 
-            // Move the file
             Result<FileProfileModel> getFileProfileResult = await this.FileProcessorManager.GetFileProfile(command.FileProfileId, cancellationToken);
-            if (getFileProfileResult.IsFailed) {
+            if (getFileProfileResult.IsFailed)
                 return ResultHelpers.CreateFailure(getFileProfileResult);
-            }
 
-            FileProfileModel fileProfile = getFileProfileResult.Data;
-            if (fileProfile == null) {
+            if (getFileProfileResult.Data == null)
                 return Result.NotFound($"No file profile found with Id {command.FileProfileId}");
-            }
 
-            // Copy file from the temp location to file processing listening directory
             IFileInfo file = this.FileSystem.FileInfo.New(command.FilePath);
             if (file.Exists == false) {
                 return Result.NotFound($"File {file.FullName} not found");
             }
 
-            String originalName = file.Name;
-
-            if (this.FileSystem.Directory.Exists(fileProfile.ListeningDirectory) == false) {
-                return Result.NotFound($"Directory {fileProfile.ListeningDirectory} not found");
-            }
+            String originalName = String.IsNullOrWhiteSpace(command.OriginalFileName)
+                ? file.Name
+                : command.OriginalFileName;
 
             // Read the file data
             String fileContent = null;
@@ -201,11 +194,8 @@ public async Task<Result> UploadFile(FileCommands.UploadFileCommand command,
                 return Result.Invalid("File content does not match the file id provided");
             }
 
-            String fileDestination = $"{fileProfile.ListeningDirectory}//{command.EstateId:N}-{command.FileId:N}";
-            file.MoveTo(fileDestination, overwrite: true);
-
             // Update Import log aggregate
-            Result stateResult= fileImportLogAggregate.AddImportedFile(command.FileId, command.MerchantId, command.UserId, command.FileProfileId, originalName, fileDestination, command.FileUploadedDateTime);
+            Result stateResult= fileImportLogAggregate.AddImportedFile(command.FileId, command.MerchantId, command.UserId, command.FileProfileId, originalName, command.FilePath, command.FileUploadedDateTime);
             if (stateResult.IsFailed)
                 return stateResult;
 
@@ -263,14 +253,34 @@ public async Task<Result> ProcessUploadedFile(FileCommands.ProcessUploadedFileCo
 
         }, command.FileId, cancellationToken, false );
 
-        if (result.IsFailed)
+        if (result.IsFailed) {
+            await MoveAcceptedFileToFailedDirectory(command.FileProfileId, command.FilePath, cancellationToken);
             return result;
+        }
 
         Result processResult = await this.ProcessFile(command.FileId, command.FileProfileId, command.FilePath, cancellationToken);
 
         return processResult;
     }
 
+    private async Task MoveAcceptedFileToFailedDirectory(Guid fileProfileId,
+                                                          String filePath,
+                                                          CancellationToken cancellationToken)
+    {
+        Result<FileProfileModel> fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken);
+        if (fileProfileResult.IsFailed || fileProfileResult.Data == null)
+            return;
+
+        IFileInfo file = this.FileSystem.FileInfo.New(filePath);
+        if (file.Exists == false)
+            return;
+
+        if (this.FileSystem.Directory.Exists(fileProfileResult.Data.FailedDirectory) == false)
+            this.FileSystem.Directory.CreateDirectory(fileProfileResult.Data.FailedDirectory);
+
+        file.MoveTo($"{fileProfileResult.Data.FailedDirectory}/{file.Name}", overwrite: true);
+    }
+
     private async Task<Result<Guid>> GetOperatorIdForFileProfile(Guid estateId, Guid fileProfileId, CancellationToken cancellationToken){
 
         Result<FileProfileModel> fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken);