Make profile directory changes safe for in-flight files

FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs

@@ -33,6 +33,10 @@ public async Task<Result<List<FileProfileModel>>> GetAllFileProfiles(Cancellatio
         return Result.Success();
     }
 
+    public async Task<Result<List<FileProfileModel>>> GetAllFileProfileVersions(CancellationToken cancellationToken) {
+        return Result.Success();
+    }
+
     public async Task<Result<FileProfileModel>> GetFileProfile(Guid fileProfileId,
                                                           CancellationToken cancellationToken) {
         return Result.Success();

FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs

@@ -393,6 +393,48 @@ public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileT
         result.IsFailed.ShouldBeTrue();
         this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
     }
+
+    [Fact]
+    public async Task ProcessUploadedFile_WhenRecordedVersionDiffers_MovesAcceptedFileToRecordedFailedDirectory()
+    {
+        FileProfile currentProfile = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            "/home/txnproc/bulkfiles/current-profile",
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            Guid.NewGuid(),
+            2);
+        FileProfile historicalProfile = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            "/home/txnproc/bulkfiles/historical-profile",
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            TestData.FileProfileVersionId,
+            1);
+        this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(currentProfile);
+        this.FileProcessorManager.GetFileProfileVersion(Arg<Guid>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(historicalProfile);
+        this.SecurityServiceClient.GetToken(Arg<String>.Any(), Arg<String>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Failure());
+        this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1"));
+        this.FileSystem.AddDirectory(currentProfile.FailedDirectory);
+        this.FileSystem.AddDirectory(historicalProfile.FailedDirectory);
+
+        FileCommands.ProcessUploadedFileCommand command = TestData.ProcessUploadedFileCommand with
+        {
+            FileProfileVersionId = historicalProfile.FileProfileVersionId
+        };
+
+        Result result = await this.FileProcessorDomainService.ProcessUploadedFile(command, CancellationToken.None);
+
+        result.IsFailed.ShouldBeTrue();
+        this.FileSystem.File.Exists($"{historicalProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
+        this.FileSystem.File.Exists($"{currentProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeFalse();
+    }
 
     [Fact]
     public async Task FileRequestHandler_ProcessUploadedFileRequest_FileNotFound_RequestIsHandled()

FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs

@@ -40,7 +40,7 @@ public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirec
         fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1"));
 
         fileProcessorManager
-            .GetAllFileProfiles(Arg<CancellationToken>.Any())
+            .GetAllFileProfileVersions(Arg<CancellationToken>.Any())
             .ReturnsAsync(Result.Success(new List<FileProfileModel> { fileProfile }));
 
         fileProcessorManager
@@ -86,7 +86,7 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays
         fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1"));
 
         fileProcessorManager
-            .GetAllFileProfiles(Arg<CancellationToken>.Any())
+            .GetAllFileProfileVersions(Arg<CancellationToken>.Any())
             .ReturnsAsync(Result.Success(new List<FileProfileModel> { fileProfile }));
 
         fileProcessorManager
@@ -140,4 +140,107 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays
                 Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath)),
             Arg<CancellationToken>.Any()).Called(Count.Once());
     }
+
+    [Fact]
+    public async Task RecoverInProgressFilesAsync_ReplaysFilesFromHistoricalAndCurrentDirectories()
+    {
+        Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
+
+        IFileProcessorManagerImposter fileProcessorManager = new();
+        IMediatorImposter mediator = new();
+        MockFileSystem fileSystem = new();
+        FileProfileModel historicalVersion = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            "/home/txnproc/bulkfiles/safaricom-old",
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            TestData.FileProfileVersionId,
+            1);
+        FileProfileModel currentVersion = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            "/home/txnproc/bulkfiles/safaricom-current",
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            Guid.NewGuid(),
+            2);
+        string historicalFilePath = $"{historicalVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}";
+        string currentFilePath = $"{currentVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId2:N}";
+        fileSystem.AddFile(historicalFilePath, new MockFileData("D,1,1,1"));
+        fileSystem.AddFile(currentFilePath, new MockFileData("D,1,1,1"));
+
+        fileProcessorManager.GetAllFileProfileVersions(Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(new List<FileProfileModel> { historicalVersion, currentVersion }));
+        fileProcessorManager.GetFile(Arg<Guid>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails()));
+        mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success());
+
+        FileProfileDirectoryRecoveryService service = new(
+            fileProcessorManager.Instance(),
+            mediator.Instance(),
+            fileSystem);
+
+        Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        mediator.Send(Arg<IRequest<Result>>.Is(request => request is FileCommands.ProcessUploadedFileCommand command &&
+            (Path.GetFileName(command.FilePath) == Path.GetFileName(historicalFilePath) ||
+             Path.GetFileName(command.FilePath) == Path.GetFileName(currentFilePath))),
+            Arg<CancellationToken>.Any()).Called(Count.Exactly(2));
+    }
+
+    [Fact]
+    public async Task RecoverInProgressFilesAsync_WhenVersionsShareDirectory_ReplaysFileOnce()
+    {
+        Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
+
+        IFileProcessorManagerImposter fileProcessorManager = new();
+        IMediatorImposter mediator = new();
+        MockFileSystem fileSystem = new();
+        FileProfileModel firstVersion = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            TestData.SafaricomListeningDirectory,
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            Guid.NewGuid(),
+            1);
+        FileProfileModel secondVersion = new(
+            TestData.FileProfileId,
+            TestData.SafaricomProfileName,
+            TestData.SafaricomListeningDirectory,
+            TestData.SafaricomRequestType,
+            TestData.SafaricomOperatorIdentifier,
+            "\n",
+            TestData.SafaricomFileFormatHandler,
+            Guid.NewGuid(),
+            2);
+        string filePath = $"{firstVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}";
+        fileSystem.AddFile(filePath, new MockFileData("D,1,1,1"));
+        fileProcessorManager.GetAllFileProfileVersions(Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(new List<FileProfileModel> { firstVersion, secondVersion }));
+        fileProcessorManager.GetFile(Arg<Guid>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails()));
+        mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any())
+            .ReturnsAsync(Result.Success());
+
+        FileProfileDirectoryRecoveryService service = new(
+            fileProcessorManager.Instance(),
+            mediator.Instance(),
+            fileSystem);
+
+        Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        mediator.Send(Arg<IRequest<Result>>.Any(), Arg<CancellationToken>.Any()).Called(Count.Once());
+    }
+
 }

FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs

@@ -19,7 +19,7 @@ namespace FileProcessor.BusinessLogic.Tests;
 public class FileProfileDirectorySynchronizerTests
 {
     [Fact]
-    public async Task SyncAsync_CreatesTemporaryAndProfileDirectories()
+    public async Task SyncAsync_CreatesTemporaryAndHistoricalProfileVersionDirectories()
     {
         MockFileSystem fileSystem = new MockFileSystem();
         IFileProfileManagerImposter fileProfileManager = new IFileProfileManagerImposter();
@@ -39,9 +39,22 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories()
             "1",
             "ProfileAHandler");
 
+        string historicalDirectory = @"C:\temp\profile-a-old";
+        FileProfileModel historicalProfileVersion = new(
+            profile.FileProfileId,
+            "Profile A",
+            historicalDirectory,
+            "ProfileARequest",
+            "Operator A",
+            "1",
+            "ProfileAHandler",
+            Guid.NewGuid(),
+            1);
+        fileSystem.AddFile(Path.Combine(historicalDirectory, "inprogress", "existing-file"), new MockFileData("contents"));
+
         fileProfileManager
-            .GetAllFileProfiles(Arg<CancellationToken>.Any())
-            .Returns(Task.FromResult(Result.Success(new List<FileProfileModel> { profile })));
+            .GetAllFileProfileVersions(Arg<CancellationToken>.Any())
+            .Returns(Task.FromResult(Result.Success(new List<FileProfileModel> { historicalProfileVersion, profile })));
 
         FileProfileDirectorySynchronizer synchronizer = new FileProfileDirectorySynchronizer(
             configuration,
@@ -56,5 +69,78 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories()
         fileSystem.Directory.Exists(Path.Combine(profile.ListeningDirectory, "inprogress")).ShouldBeTrue();
         fileSystem.Directory.Exists(profile.ProcessedDirectory).ShouldBeTrue();
         fileSystem.Directory.Exists(profile.FailedDirectory).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "inprogress")).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "processed")).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "failed")).ShouldBeTrue();
+        fileSystem.File.Exists(Path.Combine(historicalDirectory, "inprogress", "existing-file")).ShouldBeTrue();
+    }
+
+    [Fact]
+    public async Task SyncAsync_WhenVersionsShareDirectory_EnsuresSharedDirectoryWithoutDeletingFiles()
+    {
+        MockFileSystem fileSystem = new();
+        IFileProfileManagerImposter fileProfileManager = new();
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary"
+            })
+            .Build();
+        string sharedDirectory = @"C:\temp\shared-profile";
+        FileProfileModel firstVersion = new(
+            Guid.NewGuid(), "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1);
+        FileProfileModel secondVersion = new(
+            firstVersion.FileProfileId, "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2);
+        string existingFile = Path.Combine(sharedDirectory, "failed", "old-file");
+        fileSystem.AddFile(existingFile, new MockFileData("contents"));
+
+        fileProfileManager.GetAllFileProfileVersions(Arg<CancellationToken>.Any())
+            .Returns(Task.FromResult(Result.Success(new List<FileProfileModel> { firstVersion, secondVersion })));
+
+        FileProfileDirectorySynchronizer synchronizer = new(
+            configuration,
+            fileProfileManager.Instance(),
+            fileSystem,
+            NullLogger<FileProfileDirectorySynchronizer>.Instance);
+
+        Result result = await synchronizer.SyncAsync(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        fileSystem.File.Exists(existingFile).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "inprogress")).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "processed")).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "failed")).ShouldBeTrue();
+    }
+
+    [Fact]
+    public async Task SyncAsync_WhenDirectoriesDifferOnlyByCase_UsesFilesystemPathSemantics()
+    {
+        MockFileSystem fileSystem = new();
+        IFileProfileManagerImposter fileProfileManager = new();
+        IConfiguration configuration = new ConfigurationBuilder()
+            .AddInMemoryCollection(new Dictionary<string, string?>
+            {
+                ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary"
+            })
+            .Build();
+        FileProfileModel upperCaseVersion = new(
+            Guid.NewGuid(), "Profile A", @"C:\temp\Profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1);
+        FileProfileModel lowerCaseVersion = new(
+            upperCaseVersion.FileProfileId, "Profile A", @"C:\temp\profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2);
+
+        fileProfileManager.GetAllFileProfileVersions(Arg<CancellationToken>.Any())
+            .Returns(Task.FromResult(Result.Success(new List<FileProfileModel> { upperCaseVersion, lowerCaseVersion })));
+
+        FileProfileDirectorySynchronizer synchronizer = new(
+            configuration,
+            fileProfileManager.Instance(),
+            fileSystem,
+            NullLogger<FileProfileDirectorySynchronizer>.Instance);
+
+        Result result = await synchronizer.SyncAsync(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(upperCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue();
+        fileSystem.Directory.Exists(Path.Combine(lowerCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue();
     }
 }

FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs

@@ -56,6 +56,24 @@ public async Task GetFileProfileVersion_WhenVersionDoesNotExist_ReturnsActionabl
         result.Message.ShouldContain(missingVersionId.ToString());
     }
 
+    [Fact]
+    public async Task GetAllFileProfileVersions_IncludesVersionsForArchivedProfiles()
+    {
+        FileProfileAggregateRoot aggregate = this.CreateArchivedAggregate();
+        IAggregateRepositoryImposter<FileProfileAggregateRoot, DomainEvent> repository = new();
+        repository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any())
+                  .ReturnsAsync(Result.Success(aggregate));
+        FileProfileManager manager = new(repository.Instance());
+
+        Result<List<FileProfileModel>> result = await manager.GetAllFileProfileVersions(CancellationToken.None);
+
+        result.IsSuccess.ShouldBeTrue();
+        result.Data.Count.ShouldBe(2);
+        result.Data[0].FileProfileId.ShouldBe(TestData.FileProfileId);
+        result.Data[0].FileProfileVersion.ShouldBe(1);
+        result.Data[1].FileProfileVersion.ShouldBe(2);
+    }
+
     private FileProfileAggregateRoot CreateArchivedAggregate()
     {
         FileProfileAggregateRoot aggregate = FileProfileAggregateRoot.Create();

FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs

@@ -70,6 +70,11 @@ public async Task<Result<List<FileProfileModel>>> GetAllFileProfiles(Cancellatio
             return await this.FileProfileManager.GetAllFileProfiles(cancellationToken);
         }
 
+        public async Task<Result<List<FileProfileModel>>> GetAllFileProfileVersions(CancellationToken cancellationToken)
+        {
+            return await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken);
+        }
+
         public async Task<Result<FileProfileModel>> GetFileProfile(Guid fileProfileId,
                                                               CancellationToken cancellationToken)
         {

FileProcessor.BusinessLogic/Managers/FileProfileManager.cs

@@ -65,6 +65,15 @@ public async Task<Result<List<FileProfileModel>>> GetFileProfileVersions(Guid fi
         return Result.Success(versions);
     }
 
+    public async Task<Result<List<FileProfileModel>>> GetAllFileProfileVersions(CancellationToken cancellationToken)
+    {
+        Result<FileProfileAggregateRoot> aggregateResult = await this.LoadAggregate(cancellationToken);
+        if (aggregateResult.IsFailed)
+            return ResultHelpers.CreateFailure(aggregateResult);
+
+        return Result.Success(aggregateResult.Data.GetAllProfileVersions());
+    }
+
     public async Task<Result<FileProfileModel>> GetFileProfileVersion(Guid fileProfileId,
                                                                        Guid fileProfileVersionId,
                                                                        CancellationToken cancellationToken)

FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs

@@ -18,6 +18,8 @@ public interface IFileProcessorManager
 
         Task<Result<List<FileProfileModel>>> GetAllFileProfiles(CancellationToken cancellationToken);
 
+        Task<Result<List<FileProfileModel>>> GetAllFileProfileVersions(CancellationToken cancellationToken);
+
         Task<Result<FileProfileModel>> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken);
 
         Task<Result<FileProfileModel>> GetFileProfileVersion(Guid fileProfileId,

FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs

@@ -16,6 +16,8 @@ public interface IFileProfileManager
 
     Task<Result<List<FileProfileModel>>> GetFileProfileVersions(Guid fileProfileId, CancellationToken cancellationToken);
 
+    Task<Result<List<FileProfileModel>>> GetAllFileProfileVersions(CancellationToken cancellationToken);
+
     Task<Result<FileProfileModel>> GetFileProfileVersion(Guid fileProfileId, Guid fileProfileVersionId, CancellationToken cancellationToken);
 
     Task<Result<FileProfileModel>> CreateFileProfile(CreateFileProfileRequest request, CancellationToken cancellationToken);