Return accepted for idempotent file uploads

FileProcessor.BusinessLogic.Tests/FileHandlersTests.cs

@@ -138,7 +138,7 @@ public async Task UploadFile_DoesNotUseClientFilenameCharactersInTemporaryPath()
             IMediatorImposter mediator = new();
             mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
 
-            await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection("invoice.csv:payload"), CancellationToken.None);
+            await FileHandlers.UploadFile(mediator.Instance(), CreateUploadRequest(), CreateFormCollection("invoice:payload.csv"), CancellationToken.None);
 
             string storedFileName = Path.GetFileName(Directory.GetFiles(temporaryDirectory).Single());
             storedFileName.ShouldNotContain(":");

FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs

@@ -96,7 +96,6 @@ private async Task<EstateManagementContext> GetContext(Guid estateId)
             return resolvedContext.Context;
         }
 
-
         public async Task<Result<FileImportLogQueryResult>> GetFileImportLogs(Guid estateId,
                                                                  DateOnly startDate,
                                                                  DateOnly endDate,

FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs

@@ -167,17 +167,65 @@ public async Task<Result> UploadFile(FileCommands.UploadFileCommand command,
         if (validateResult.IsFailed)
             return validateResult;
 
-        // This will now create the import log and add an event for the file being uploaded
+        // The import log is created only after validation and duplicate checks succeed.
         Guid importLogId = Helpers.CalculateFileImportLogAggregateId(importLogDateTime.Date, command.EstateId);
 
         Result result = await ApplyFileImportLogUpdates(async (fileImportLogAggregate) => {
-            if (fileImportLogAggregate.IsCreated == false)
+            IFileInfo file = this.FileSystem.FileInfo.New(command.FilePath);
+            if (file.Exists == false) {
+                return Result.NotFound($"File {file.FullName} not found");
+            }
+
+            String originalName = String.IsNullOrWhiteSpace(command.OriginalFileName)
+                ? file.Name
+                : command.OriginalFileName;
+
+            // Read the original bytes so the server validates the same content hash as the uploader.
+            Byte[] fileContent;
+            await using (Stream fs = file.Open(FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) {
+                using MemoryStream memoryStream = new();
+                await fs.CopyToAsync(memoryStream, cancellationToken);
+                fileContent = memoryStream.ToArray();
+            }
+            Guid generatedFileId = CreateGuidFromFileData(fileContent);
+
+            if (generatedFileId != command.FileId) {
+                return Result.Invalid("File content does not match the file id provided");
+            }
+
+            Result<FileAggregate> existingFileResult = await this.FileAggregateRepository.GetLatestVersion(command.FileId, cancellationToken);
+            if (existingFileResult?.IsSuccess == true && existingFileResult.Data?.IsCreated == true)
             {
-                // First file of the day so create
-                Result result = fileImportLogAggregate.CreateImportLog(command.EstateId, importLogDateTime);
-                if (result.IsFailed) {
-                    return result;
+                FileDetails existingFile = existingFileResult.Data.GetFile();
+                if (existingFile.EstateId != command.EstateId ||
+                    existingFile.MerchantId != command.MerchantId ||
+                    existingFile.UserId != command.UserId ||
+                    existingFile.FileProfileId != command.FileProfileId)
+                {
+                    return Result.Invalid($"File Id [{command.FileId}] is already registered with different upload details");
                 }
+
+                this.DeleteDuplicateUploadFile(command.FilePath);
+                return Result.Success();
+            }
+            if (existingFileResult?.IsFailed == true && existingFileResult.Status != ResultStatus.NotFound)
+                return ResultHelpers.CreateFailure(existingFileResult);
+
+            // An identical file already on the current import log is accepted without appending another event.
+            Boolean fileAlreadyRegistered = fileImportLogAggregate.ContainsFile(command.FileId);
+            if (fileAlreadyRegistered)
+            {
+                Result duplicateResult = fileImportLogAggregate.AddImportedFile(new AddImportedFileRequest(command.FileId,
+                                                                                                            command.MerchantId,
+                                                                                                            command.UserId,
+                                                                                                            command.FileProfileId,
+                                                                                                            originalName,
+                                                                                                            command.FilePath,
+                                                                                                            command.FileUploadedDateTime));
+                if (duplicateResult.IsSuccess)
+                    this.DeleteDuplicateUploadFile(command.FilePath);
+
+                return duplicateResult;
             }
 
             Result<FileProfileModel> getFileProfileResult = await this.FileProcessorManager.GetFileProfile(command.FileProfileId, cancellationToken);
@@ -191,40 +239,21 @@ public async Task<Result> UploadFile(FileCommands.UploadFileCommand command,
                 ? getFileProfileResult.Data.FileProfileId
                 : getFileProfileResult.Data.FileProfileVersionId;
 
-            IFileInfo file = this.FileSystem.FileInfo.New(command.FilePath);
-            if (file.Exists == false) {
-                return Result.NotFound($"File {file.FullName} not found");
-            }
-
-            String originalName = String.IsNullOrWhiteSpace(command.OriginalFileName)
-                ? file.Name
-                : command.OriginalFileName;
-
-            // Read the file data
-            String fileContent = null;
-            //Open file for Read\Write
-            await using (Stream fs = file.Open(FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) {
-                //Create object of StreamReader by passing FileStream object on which it needs to operates on
-                using (StreamReader sr = new StreamReader(fs)) {
-                    //Use ReadToEnd method to read all the content from file
-                    fileContent = await sr.ReadToEndAsync(cancellationToken);
-                }
+            if (fileImportLogAggregate.IsCreated == false)
+            {
+                Result createResult = fileImportLogAggregate.CreateImportLog(command.EstateId, importLogDateTime);
+                if (createResult.IsFailed)
+                    return createResult;
             }
-            Guid generatedFileId = CreateGuidFromFileData(fileContent);
 
-            if (generatedFileId != command.FileId) {
-                return Result.Invalid("File content does not match the file id provided");
-            }
-
-            // Update Import log aggregate
-            Result stateResult= fileImportLogAggregate.AddImportedFile(new AddImportedFileRequest(command.FileId,
-                                                                                                  command.MerchantId,
-                                                                                                  command.UserId,
-                                                                                                  command.FileProfileId,
-                                                                                                  originalName,
-                                                                                                  command.FilePath,
-                                                                                                  command.FileUploadedDateTime,
-                                                                                                  fileProfileVersionId));
+            Result stateResult = fileImportLogAggregate.AddImportedFile(new AddImportedFileRequest(command.FileId,
+                                                                                                   command.MerchantId,
+                                                                                                   command.UserId,
+                                                                                                   command.FileProfileId,
+                                                                                                   originalName,
+                                                                                                   command.FilePath,
+                                                                                                   command.FileUploadedDateTime,
+                                                                                                   fileProfileVersionId));
             if (stateResult.IsFailed)
                 return stateResult;
 
@@ -235,11 +264,16 @@ public async Task<Result> UploadFile(FileCommands.UploadFileCommand command,
     }
 
     public static Guid CreateGuidFromFileData(String fileContents)
+    {
+        return CreateGuidFromFileData(Encoding.UTF8.GetBytes(fileContents));
+    }
+
+    public static Guid CreateGuidFromFileData(Byte[] fileContents)
     {
         using (SHA256 sha256Hash = SHA256.Create())
         {
             //Generate hash from the key
-            Byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(fileContents));
+            Byte[] bytes = sha256Hash.ComputeHash(fileContents);
 
             Byte[] j = bytes.Skip(Math.Max(0, bytes.Count() - 16)).ToArray(); //Take last 16
 
@@ -248,6 +282,14 @@ public static Guid CreateGuidFromFileData(String fileContents)
         }
     }
     private Result ValidateRequest(FileCommands.UploadFileCommand command) {
+        if (command.EstateId == Guid.Empty) {
+            return Result.Invalid("No Estate Id provided with file upload");
+        }
+
+        if (command.FileId == Guid.Empty) {
+            return Result.Invalid("No File Id provided with file upload");
+        }
+
         if (command.UserId == Guid.Empty) {
             return Result.Invalid("No User Id provided with file upload");
         }
@@ -263,6 +305,20 @@ private Result ValidateRequest(FileCommands.UploadFileCommand command) {
         return Result.Success();
     }
 
+    private void DeleteDuplicateUploadFile(String filePath)
+    {
+        try
+        {
+            IFileInfo file = this.FileSystem.FileInfo.New(filePath);
+            if (file.Exists)
+                file.Delete();
+        }
+        catch (Exception exception)
+        {
+            Logger.LogWarning($"Failed to remove duplicate temporary upload file [{filePath}]. Error [{exception.Message}]");
+        }
+    }
+
     public async Task<Result> ProcessUploadedFile(FileCommands.ProcessUploadedFileCommand command,
                                                   CancellationToken cancellationToken) {
         Result result = await ApplyFileUpdates(async (FileAggregate fileAggregate) => {

FileProcessor.FileImportLogAggregate.Tests/FileImportLogAggregateTests.cs

@@ -98,14 +98,14 @@ public void FileImportLogAggregate_AddImportedFile_ImportLogNotCreated_ErrorThro
         }
 
         [Fact]
-        public void FileImportLogAggregate_AddImportedFile_DuplicateFileId_ErrorThrown()
+        public void FileImportLogAggregate_AddImportedFile_DuplicateFileId_IsAcceptedIdempotently()
         {
             FileImportLogAggregate fileImportLogAggregate = FileImportLogAggregate.Create(TestData.FileImportLogId);
             fileImportLogAggregate.CreateImportLog(TestData.EstateId, TestData.ImportLogDateTime);
             fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath, TestData.FileUploadedDateTime);
             Result result = fileImportLogAggregate.AddImportedFile(TestData.FileId, TestData.MerchantId, TestData.UserId, TestData.FileProfileId, TestData.OriginalFileName, TestData.FilePath, TestData.FileUploadedDateTime);
-            result.IsFailed.ShouldBeTrue();
-            result.Status.ShouldBe(ResultStatus.Invalid);
+            result.IsSuccess.ShouldBeTrue();
+            fileImportLogAggregate.GetFileImportLog().Files.Count.ShouldBe(1);
         }
     }
 }

FileProcessor.FileImportLogAggregate/FileImportLogAggregate.cs

@@ -52,9 +52,18 @@ public static Result AddImportedFile(this FileImportLogAggregate aggregate, AddI
                 return Result.Invalid("Import log has not been created");
             }
 
-            if (aggregate.Files.Any(f => f.FileId == request.FileId))
+            ImportLogFile existingFile = aggregate.Files.SingleOrDefault(f => f.FileId == request.FileId);
+            if (existingFile != null)
             {
-                return Result.Invalid($"Duplicate file {request.OriginalFileName} detected File Id [{request.FileId}]");
+                if (existingFile.EstateId == aggregate.EstateId &&
+                    existingFile.MerchantId == request.MerchantId &&
+                    existingFile.UserId == request.UserId &&
+                    existingFile.FileProfileId == request.FileProfileId)
+                {
+                    return Result.Success();
+                }
+
+                return Result.Invalid($"File Id [{request.FileId}] is already registered with different upload details");
             }
 
             FileAddedToImportLogEvent fileAddedToImportLogEvent =
@@ -74,6 +83,11 @@ public static Result AddImportedFile(this FileImportLogAggregate aggregate, AddI
             return Result.Success();
         }
 
+        public static Boolean ContainsFile(this FileImportLogAggregate aggregate, Guid fileId)
+        {
+            return aggregate.Files.Any(file => file.FileId == fileId);
+        }
+
         public static void PlayEvent(this FileImportLogAggregate aggregate, ImportLogCreatedEvent domainEvent)
         {
             aggregate.IsCreated = true;
@@ -92,7 +106,8 @@ public static void PlayEvent(this FileImportLogAggregate aggregate, FileAddedToI
                                     MerchantId = domainEvent.MerchantId,
                                     FilePath = domainEvent.FilePath,
                                     OriginalFileName = domainEvent.OriginalFileName,
-                                    UserId = domainEvent.UserId
+                                    UserId = domainEvent.UserId,
+                                    UploadedDateTime = domainEvent.FileUploadedDateTime
                                 });
         }
 

FileProcessor.IntegrationTests/Features/ProcessTopupCSV.feature

@@ -143,7 +143,7 @@ Scenario: Process Duplicate Safaricom Topup File with 1 detail row
 	| D       | 07777777775 | 100     |
 	| D       | 07777777776 | 200     |
 	| T       | 1           |         |
-	And I upload this file for processing an error should be returned indicating the file is a duplicate
+	And I upload this file for processing
 	| EstateName    | MerchantName    | FileProfileId                        | UserId                               |
 	| Test Estate 1 | Test Merchant 1 | B2A59ABF-293D-4A6B-B81B-7007503C3476 | ABA59ABF-293D-4A6B-B81B-7007503C3476 |
 

FileProcessor.IntegrationTests/Features/ProcessVoucherCSV.feature

@@ -157,6 +157,6 @@ Scenario: Process Duplicate Voucher Topup File with 1 detail row
 	| D       | Hospital 1 | testrecipient1@recipient.com | 20      |
 	| T       | 1          |                              |         |
 
-	And I upload this file for processing an error should be returned indicating the file is a duplicate
+	And I upload this file for processing
 	| EstateName    | MerchantName    | FileProfileId                        | UserId                               |
 	| Test Estate 1 | Test Merchant 1 | 8806EDBC-3ED6-406B-9E5F-A9078356BE99 | ABA59ABF-293D-4A6B-B81B-7007503C3476 |

FileProcessor/Bootstrapper/MiddlewareRegistry.cs

@@ -4,6 +4,7 @@
 
 namespace FileProcessor.Bootstrapper
 {
+    using FileProcessor.Common;
     using Lamar;
     using Microsoft.AspNetCore.Http.Features;
     using Microsoft.Extensions.DependencyInjection;
@@ -113,10 +114,9 @@ public MiddlewareRegistry()
 
             this.Configure<FormOptions>(options =>
             {
-                // Allow very large values (adjust to what you need)
-                options.ValueLengthLimit = int.MaxValue;                  // form value length
-                options.MultipartBodyLengthLimit = long.MaxValue;         // multipart body length
-                options.MemoryBufferThreshold = int.MaxValue;             // buffer threshold
+                options.ValueLengthLimit = 16 * 1024;
+                options.MultipartBodyLengthLimit = UploadFileSettings.GetMaximumMultipartRequestSizeBytes();
+                options.MemoryBufferThreshold = 64 * 1024;
             });
 
             this.AddHealthMonitoringRegistration(Startup.Configuration);
@@ -161,4 +161,4 @@ public static void ConfigureMinimalApi(JsonSerializerOptions serializerOptions)
             serializerOptions.Converters.Add(new DateTimeSpaceConverter());
         }
     }
-}
+}

FileProcessor/Common/UploadFileSettings.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Globalization;
+using Shared.General;
+
+namespace FileProcessor.Common;
+
+public static class UploadFileSettings
+{
+    private const Int64 DefaultMaximumFileSizeBytes = 10 * 1024 * 1024;
+    private const Int64 MaximumMultipartOverheadBytes = 1024 * 1024;
+    private const String DefaultAllowedFileExtensions = ".csv,.txt";
+
+    public static Int64 GetMaximumFileSizeBytes()
+    {
+        String configuredValue = ConfigurationReader.GetValueOrDefault("AppSettings", "MaximumUploadFileSizeBytes", DefaultMaximumFileSizeBytes.ToString(CultureInfo.InvariantCulture));
+
+        if (Int64.TryParse(configuredValue, NumberStyles.None, CultureInfo.InvariantCulture, out Int64 maximumFileSizeBytes) == false || maximumFileSizeBytes <= 0)
+            throw new InvalidOperationException("AppSettings:MaximumUploadFileSizeBytes must be a positive integer when configured");
+
+        return maximumFileSizeBytes;
+    }
+
+    public static Int64 GetMaximumMultipartRequestSizeBytes()
+    {
+        try
+        {
+            return checked(GetMaximumFileSizeBytes() + MaximumMultipartOverheadBytes);
+        }
+        catch (OverflowException exception)
+        {
+            throw new InvalidOperationException("AppSettings:MaximumUploadFileSizeBytes is too large", exception);
+        }
+    }
+
+    public static String[] GetAllowedFileExtensions()
+    {
+        String value = ConfigurationReader.GetValueOrDefault("AppSettings", "AllowedUploadFileExtensions", DefaultAllowedFileExtensions);
+
+        String[] extensions = value.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        foreach (String extension in extensions)
+        {
+            if (extension.StartsWith('.') == false || extension.Length < 2)
+                throw new InvalidOperationException("AppSettings:AllowedUploadFileExtensions must contain comma-separated extensions beginning with a dot");
+        }
+
+        if (extensions.Length == 0)
+            throw new InvalidOperationException("AppSettings:AllowedUploadFileExtensions must contain at least one extension");
+
+        return extensions;
+    }
+}