Add file-level validation and duplicate row handling

FileProcessor.BusinessLogic.Tests/FileFormatValidationTests.cs

@@ -0,0 +1,78 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using FileProcessor.BusinessLogic.FileFormatHandlers;
+using Shouldly;
+using Xunit;
+
+namespace FileProcessor.BusinessLogic.Tests;
+
+public class FileFormatValidationTests
+{
+    [Fact]
+    public void Safaricom_ValidFile_IsAccepted()
+    {
+        SafaricomFileFormatHandler handler = new();
+
+        FileValidationResult result = handler.ValidateFile(new StringReader("H,20210508,\nD,07777770001,10\nT,1,"), "\n");
+
+        result.IsValid.ShouldBeTrue();
+        result.Errors.ShouldBeEmpty();
+    }
+
+    [Theory]
+    [InlineData("EmptyFile", new string[0], "EmptyFile")]
+    [InlineData("MissingHeader", new[] { "D,07777770001,10", "T,1," }, "MissingHeader")]
+    [InlineData("MissingTrailer", new[] { "H,20210508,", "D,07777770001,10" }, "MissingTrailer")]
+    [InlineData("HeaderNotFirst", new[] { "D,07777770001,10", "H,20210508,", "T,1," }, "HeaderNotFirst")]
+    [InlineData("TrailerNotLast", new[] { "H,20210508,", "T,1,", "D,07777770001,10" }, "TrailerNotLast")]
+    [InlineData("TrailerCountMismatch", new[] { "H,20210508,", "D,07777770001,10", "T,2," }, "TrailerDetailCountMismatch")]
+    public void Safaricom_InvalidFile_ReturnsSpecificError(string scenario,
+                                                           IReadOnlyList<string> lines,
+                                                           string expectedCode)
+    {
+        _ = scenario;
+        SafaricomFileFormatHandler handler = new();
+
+        FileValidationResult result = handler.ValidateFile(new StringReader(string.Join("\n", lines)), "\n");
+
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(error => error.Code == expectedCode);
+    }
+
+    [Fact]
+    public void Voucher_MalformedDetail_ReturnsRowValidationError()
+    {
+        VoucherFileFormatHandler handler = new();
+
+        FileValidationResult result = handler.ValidateFile(new StringReader("H,20210508,\nD,Issuer,recipient\nT,1,"), "\n");
+
+        result.IsValid.ShouldBeFalse();
+        result.Errors.OfType<RowValidationError>().ShouldContain(error => error.LineNumber == 2 &&
+                                                                            error.Code == "InvalidFieldCount");
+    }
+
+    [Fact]
+    public void Voucher_TrailerValue_DoesNotRepresentDetailRowCount()
+    {
+        VoucherFileFormatHandler handler = new();
+
+        FileValidationResult result = handler.ValidateFile(new StringReader("H,20210508,\nD,Issuer,07777777775,10\nD,Issuer,testrecipient1@recipient.com,20\nT,1,"), "\n");
+
+        result.IsValid.ShouldBeTrue();
+    }
+
+    [Theory]
+    [InlineData("T")]
+    [InlineData("Trailer")]
+    public void Safaricom_TrailerWithoutDetailCount_IsRejected(String trailer)
+    {
+        SafaricomFileFormatHandler handler = new();
+
+        FileValidationResult result = handler.ValidateFile(new StringReader($"H,20210508,\nD,07777770001,10\n{trailer}"), "\n");
+
+        result.IsValid.ShouldBeFalse();
+        result.Errors.ShouldContain(error => error.Code == "InvalidTrailer");
+    }
+}

FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs

@@ -67,6 +67,8 @@ public FileProcessorDomainServiceTests()
         this.TransactionProcessorClient = new ITransactionProcessorClientImposter();
         this.SecurityServiceClient = new ISecurityServiceClientImposter();
         this.FileFormatHandler = new IFileFormatHandlerImposter();
+        this.FileFormatHandler.ValidateFile(Arg<TextReader>.Any(), Arg<String>.Any()).Returns(
+            new FileValidationResult(Guid.Empty, true, Array.Empty<FileValidationError>()));
         this.FileSystem = new MockFileSystem();
         this.DiagnosticLogger = new CapturingLogger<FileProcessorDomainService>();
 
@@ -282,6 +284,61 @@ public async Task ProcessUploadedFile_WhenFileExceedsMaximumDetailRecords_Reject
         this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
     }
 
+    [Fact]
+    public async Task ProcessUploadedFile_WhenFileValidationFails_RejectsBeforeAddingLines()
+    {
+        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
+        this.FileFormatHandler.FileLineCanBeIgnored(Arg<String>.Any()).Returns((String line) => line.StartsWith("H") || line.StartsWith("T"));
+        this.FileFormatHandler.ValidateFile(Arg<TextReader>.Any(), Arg<String>.Any()).Returns(
+            new FileValidationResult(Guid.Empty,
+                                      false,
+                                      new[] { new FileLevelValidationError("InvalidFile", "The file is invalid") }));
+
+        FileAggregate createdAggregate = TestData.GetCreatedFileAggregate();
+        this.FileAggregateRepository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(TestData.GetEmptyFileAggregate()))
+            .Then().ReturnsAsync(Result.Success(createdAggregate));
+        this.FileAggregateRepository.SaveChanges(Arg<FileAggregate>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
+        this.SecurityServiceClient.GetToken(Arg<String>.Any(), Arg<String>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success(TestData.TokenResponse()));
+        this.TransactionProcessorClient.GetOperators(Arg<String>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.OperatorList);
+
+        this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("H,20210508,\nD,invalid\nT,1,"));
+        this.FileSystem.AddDirectory(TestData.FileProfileSafaricom.ProcessedDirectory);
+        this.FileSystem.AddDirectory(TestData.FileProfileSafaricom.FailedDirectory);
+
+        Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        createdAggregate.GetFile().IsRejected.ShouldBeTrue();
+        createdAggregate.GetFile().RejectionCode.ShouldBe("InvalidFile");
+        createdAggregate.GetFile().FileLines.ShouldBeEmpty();
+        this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
+    }
+
+    [Fact]
+    public async Task ProcessUploadedFile_WhenReplayed_DoesNotAppendPersistedLines()
+    {
+        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
+        this.FileFormatHandler.FileLineCanBeIgnored(Arg<String>.Any()).Returns(false);
+
+        FileAggregate createdAggregate = TestData.GetCreatedFileAggregate();
+        createdAggregate.AddFileLine("D,1,1,1");
+        this.FileAggregateRepository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(createdAggregate));
+        this.FileAggregateRepository.SaveChanges(Arg<FileAggregate>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
+        this.SecurityServiceClient.GetToken(Arg<String>.Any(), Arg<String>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success(TestData.TokenResponse()));
+        this.TransactionProcessorClient.GetOperators(Arg<String>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.OperatorList);
+
+        this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1"));
+        this.FileSystem.AddDirectory(TestData.FileProfileSafaricom.ProcessedDirectory);
+        this.FileSystem.AddDirectory(TestData.FileProfileSafaricom.FailedDirectory);
+
+        Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        createdAggregate.GetFile().FileLines.Count.ShouldBe(1);
+    }
+
     [Theory]
     [InlineData("0")]
     [InlineData("-1")]

FileProcessor.BusinessLogic/EventHandling/FileDomainEventHandler.cs

@@ -93,6 +93,14 @@ private Task<Result> HandleSpecificDomainEvent(FileLineTransactionDispatchFailed
             return Task.FromResult(Result.Success());
         }
 
+        private Task<Result> HandleSpecificDomainEvent(DuplicateFileLineAddedEvent domainEvent,
+                                                        CancellationToken cancellationToken)
+        {
+            _ = domainEvent;
+            _ = cancellationToken;
+            return Task.FromResult(Result.Success());
+        }
+
         /// <summary>
         /// Handles the specific domain event.
         /// </summary>

FileProcessor.BusinessLogic/FileFormatHandlers/FileValidationErrors.cs

@@ -0,0 +1,135 @@
+namespace FileProcessor.BusinessLogic.FileFormatHandlers;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+public abstract record FileValidationError(String Code, String Reason);
+
+public record FileLevelValidationError(String Code, String Reason)
+    : FileValidationError(Code, Reason);
+
+public record RowValidationError(Int32 LineNumber,
+                                 String Field,
+                                 String Code,
+                                 String Reason)
+    : FileValidationError(Code, Reason);
+
+public record FileValidationResult(Guid FileId,
+                                   Boolean IsValid,
+                                   IReadOnlyList<FileValidationError> Errors);
+
+internal static class FileFormatValidation
+{
+    public static IReadOnlyList<String> ReadLines(TextReader fileReader, String lineTerminator)
+    {
+        return fileReader.ReadToEnd().Split(lineTerminator);
+    }
+
+    public static FileValidationResult Validate(IReadOnlyList<String> fileLines,
+                                                Func<String, Boolean> isIgnored,
+                                                Func<String, Int32> expectedFieldCount,
+                                                Func<String, String> parseDetailError,
+                                                Boolean validateTrailerDetailCount)
+    {
+        List<(Int32 Number, String Value)> lines = fileLines
+            .Select((line, index) => (Number: index + 1, Value: line?.Trim() ?? String.Empty))
+            .Where(line => String.IsNullOrWhiteSpace(line.Value) == false)
+            .ToList();
+
+        if (lines.Count == 0)
+            return new FileValidationResult(Guid.Empty,
+                                            false,
+                                            new[] { new FileLevelValidationError("EmptyFile", "The file contains no non-empty lines") });
+
+        List<(Int32 Number, String Value)> detailLines = lines.Where(line => isIgnored(line.Value) == false).ToList();
+        List<FileValidationError> errors = ValidateStructure(lines);
+        errors.AddRange(ValidateDetailLines(detailLines, expectedFieldCount, parseDetailError));
+        errors.AddRange(ValidateTrailerDetailCount(lines, detailLines.Count, validateTrailerDetailCount));
+
+        return new FileValidationResult(Guid.Empty, errors.Count == 0, errors);
+    }
+
+    private static List<FileValidationError> ValidateStructure(IReadOnlyList<(Int32 Number, String Value)> lines)
+    {
+        List<FileValidationError> errors = new();
+        Boolean firstLineIsHeader = lines[0].Value.StartsWith("H", StringComparison.Ordinal);
+        Boolean lastLineIsTrailer = lines[^1].Value.StartsWith("T", StringComparison.Ordinal);
+
+        if (firstLineIsHeader == false)
+            errors.Add(new FileLevelValidationError("MissingHeader", "The first non-empty line must be a header"));
+
+        if (lines.Any(line => line.Value.StartsWith("H", StringComparison.Ordinal)) && firstLineIsHeader == false)
+            errors.Add(new FileLevelValidationError("HeaderNotFirst", "The header must be the first non-empty line"));
+
+        if (lastLineIsTrailer == false)
+            errors.Add(new FileLevelValidationError("MissingTrailer", "The last non-empty line must be a trailer"));
+
+        if (lines.Any(line => line.Value.StartsWith("T", StringComparison.Ordinal)) && lastLineIsTrailer == false)
+            errors.Add(new FileLevelValidationError("TrailerNotLast", "The trailer must be the last non-empty line"));
+
+        if (lines.Count(line => line.Value.StartsWith("H", StringComparison.Ordinal)) != 1)
+            errors.Add(new FileLevelValidationError("InvalidHeaderCount", "The file must contain exactly one header"));
+
+        if (lines.Count(line => line.Value.StartsWith("T", StringComparison.Ordinal)) != 1)
+            errors.Add(new FileLevelValidationError("InvalidTrailerCount", "The file must contain exactly one trailer"));
+
+        return errors;
+    }
+
+    private static List<FileValidationError> ValidateDetailLines(IReadOnlyList<(Int32 Number, String Value)> detailLines,
+                                                                 Func<String, Int32> expectedFieldCount,
+                                                                 Func<String, String> parseDetailError)
+    {
+        List<FileValidationError> errors = new();
+        if (detailLines.Count == 0)
+            errors.Add(new FileLevelValidationError("NoDetailRows", "The file contains no detail rows"));
+
+        foreach ((Int32 number, String value) detailLine in detailLines)
+        {
+            String[] fields = detailLine.value.Split(',');
+            Int32 expectedCount = expectedFieldCount(detailLine.value);
+            if (fields.Length != expectedCount)
+            {
+                errors.Add(new RowValidationError(detailLine.number,
+                                                  "Record",
+                                                  "InvalidFieldCount",
+                                                  $"Expected {expectedCount} fields but found {fields.Length}"));
+                continue;
+            }
+
+            String parseError = parseDetailError(detailLine.value);
+            if (String.IsNullOrWhiteSpace(parseError) == false)
+                errors.Add(new RowValidationError(detailLine.number, "Record", "InvalidDetail", parseError));
+        }
+
+        return errors;
+    }
+
+    private static List<FileValidationError> ValidateTrailerDetailCount(IReadOnlyList<(Int32 Number, String Value)> lines,
+                                                                         Int32 detailLineCount,
+                                                                         Boolean validateTrailerDetailCount)
+    {
+        List<FileValidationError> errors = new();
+        if (validateTrailerDetailCount == false || lines[^1].Value.StartsWith("T", StringComparison.Ordinal) == false)
+            return errors;
+
+        String[] trailerFields = lines[^1].Value.Split(',');
+        if (trailerFields.Length > 1 && Int32.TryParse(trailerFields[1].Trim(), out Int32 expectedCount))
+        {
+            if (expectedCount != detailLineCount)
+                errors.Add(new FileLevelValidationError("TrailerDetailCountMismatch",
+                                                        $"Trailer specifies {expectedCount} detail rows but found {detailLineCount}"));
+        }
+        else
+        {
+            errors.Add(new RowValidationError(lines[^1].Number,
+                                              "DetailCount",
+                                              "InvalidTrailer",
+                                              "Trailer detail count is not a valid integer"));
+        }
+
+        return errors;
+    }
+}

FileProcessor.BusinessLogic/FileFormatHandlers/IFileFormatHandler.cs

@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using System.IO;
 
     /// <summary>
     /// 
@@ -21,5 +22,7 @@ public interface IFileFormatHandler
         /// <param name="fileLine">The file line.</param>
         /// <returns></returns>
         Boolean FileLineCanBeIgnored(String fileLine);
+
+        FileValidationResult ValidateFile(TextReader fileReader, String lineTerminator);
     }
-}
+}

FileProcessor.BusinessLogic/FileFormatHandlers/SafaricomFileFormatHandler.cs

@@ -10,6 +10,28 @@
     /// <seealso cref="IFileFormatHandler" />
     public class SafaricomFileFormatHandler : IFileFormatHandler
     {
+        public FileValidationResult ValidateFile(TextReader fileReader, String lineTerminator)
+        {
+            return FileFormatValidation.Validate(FileFormatValidation.ReadLines(fileReader, lineTerminator),
+                                                 this.FileLineCanBeIgnored,
+                                                 _ => 3,
+                                                 line => TryParseDetail(line),
+                                                 true);
+        }
+
+        private String TryParseDetail(String line)
+        {
+            try
+            {
+                this.ParseFileLine(line);
+                return null;
+            }
+            catch (InvalidDataException exception)
+            {
+                return exception.Message;
+            }
+        }
+
         /// <summary>
         /// Parses the file line.
         /// </summary>
@@ -61,4 +83,4 @@ public Boolean FileLineCanBeIgnored(String fileLine)
             return (fileLine.StartsWith("H") || fileLine.StartsWith("T"));
         }
     }
-}
+}

FileProcessor.BusinessLogic/FileFormatHandlers/VoucherFileFormatHandler.cs

@@ -6,6 +6,28 @@
 
     public class VoucherFileFormatHandler : IFileFormatHandler
     {
+        public FileValidationResult ValidateFile(TextReader fileReader, String lineTerminator)
+        {
+            return FileFormatValidation.Validate(FileFormatValidation.ReadLines(fileReader, lineTerminator),
+                                                 this.FileLineCanBeIgnored,
+                                                 _ => 4,
+                                                 line => TryParseDetail(line),
+                                                 false);
+        }
+
+        private String TryParseDetail(String line)
+        {
+            try
+            {
+                this.ParseFileLine(line);
+                return null;
+            }
+            catch (InvalidDataException exception)
+            {
+                return exception.Message;
+            }
+        }
+
         public Dictionary<String, String> ParseFileLine(String fileLine)
         {
             // Format is 
@@ -58,4 +80,4 @@ public Boolean FileLineCanBeIgnored(String fileLine)
             return (fileLine.StartsWith("H") || fileLine.StartsWith("T"));
         }
     }
-}
+}