diff --git a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs index c5e7972..7637ff3 100644 --- a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs @@ -33,6 +33,10 @@ public async Task>> GetAllFileProfiles(Cancellatio return Result.Success(); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) { + return Result.Success(); + } + public async Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken) { return Result.Success(); diff --git a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs index 989dab6..4bde44b 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs @@ -393,6 +393,48 @@ public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileT result.IsFailed.ShouldBeTrue(); this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue(); } + + [Fact] + public async Task ProcessUploadedFile_WhenRecordedVersionDiffers_MovesAcceptedFileToRecordedFailedDirectory() + { + FileProfile currentProfile = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/current-profile", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + FileProfile historicalProfile = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/historical-profile", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + TestData.FileProfileVersionId, + 1); + this.FileProcessorManager.GetFileProfile(Arg.Any(), Arg.Any()).ReturnsAsync(currentProfile); + this.FileProcessorManager.GetFileProfileVersion(Arg.Any(), Arg.Any(), Arg.Any()).ReturnsAsync(historicalProfile); + this.SecurityServiceClient.GetToken(Arg.Any(), Arg.Any(), Arg.Any()).ReturnsAsync(Result.Failure()); + this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1")); + this.FileSystem.AddDirectory(currentProfile.FailedDirectory); + this.FileSystem.AddDirectory(historicalProfile.FailedDirectory); + + FileCommands.ProcessUploadedFileCommand command = TestData.ProcessUploadedFileCommand with + { + FileProfileVersionId = historicalProfile.FileProfileVersionId + }; + + Result result = await this.FileProcessorDomainService.ProcessUploadedFile(command, CancellationToken.None); + + result.IsFailed.ShouldBeTrue(); + this.FileSystem.File.Exists($"{historicalProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue(); + this.FileSystem.File.Exists($"{currentProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeFalse(); + } [Fact] public async Task FileRequestHandler_ProcessUploadedFileRequest_FileNotFound_RequestIsHandled() diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs index d586e4c..c4f0d48 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -40,7 +40,7 @@ public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirec fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); fileProcessorManager - .GetAllFileProfiles(Arg.Any()) + .GetAllFileProfileVersions(Arg.Any()) .ReturnsAsync(Result.Success(new List { fileProfile })); fileProcessorManager @@ -86,7 +86,7 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); fileProcessorManager - .GetAllFileProfiles(Arg.Any()) + .GetAllFileProfileVersions(Arg.Any()) .ReturnsAsync(Result.Success(new List { fileProfile })); fileProcessorManager @@ -140,4 +140,107 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath)), Arg.Any()).Called(Count.Once()); } + + [Fact] + public async Task RecoverInProgressFilesAsync_ReplaysFilesFromHistoricalAndCurrentDirectories() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + IFileProcessorManagerImposter fileProcessorManager = new(); + IMediatorImposter mediator = new(); + MockFileSystem fileSystem = new(); + FileProfileModel historicalVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/safaricom-old", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + TestData.FileProfileVersionId, + 1); + FileProfileModel currentVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/safaricom-current", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + string historicalFilePath = $"{historicalVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}"; + string currentFilePath = $"{currentVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId2:N}"; + fileSystem.AddFile(historicalFilePath, new MockFileData("D,1,1,1")); + fileSystem.AddFile(currentFilePath, new MockFileData("D,1,1,1")); + + fileProcessorManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List { historicalVersion, currentVersion })); + fileProcessorManager.GetFile(Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails())); + mediator.Send(Arg>.Any(), Arg.Any()) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Instance(), + mediator.Instance(), + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Send(Arg>.Is(request => request is FileCommands.ProcessUploadedFileCommand command && + (Path.GetFileName(command.FilePath) == Path.GetFileName(historicalFilePath) || + Path.GetFileName(command.FilePath) == Path.GetFileName(currentFilePath))), + Arg.Any()).Called(Count.Exactly(2)); + } + + [Fact] + public async Task RecoverInProgressFilesAsync_WhenVersionsShareDirectory_ReplaysFileOnce() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + IFileProcessorManagerImposter fileProcessorManager = new(); + IMediatorImposter mediator = new(); + MockFileSystem fileSystem = new(); + FileProfileModel firstVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 1); + FileProfileModel secondVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + string filePath = $"{firstVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}"; + fileSystem.AddFile(filePath, new MockFileData("D,1,1,1")); + fileProcessorManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List { firstVersion, secondVersion })); + fileProcessorManager.GetFile(Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails())); + mediator.Send(Arg>.Any(), Arg.Any()) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Instance(), + mediator.Instance(), + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Send(Arg>.Any(), Arg.Any()).Called(Count.Once()); + } + } diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs index 4402fa6..aa4f916 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs @@ -19,7 +19,7 @@ namespace FileProcessor.BusinessLogic.Tests; public class FileProfileDirectorySynchronizerTests { [Fact] - public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() + public async Task SyncAsync_CreatesTemporaryAndHistoricalProfileVersionDirectories() { MockFileSystem fileSystem = new MockFileSystem(); IFileProfileManagerImposter fileProfileManager = new IFileProfileManagerImposter(); @@ -39,9 +39,22 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() "1", "ProfileAHandler"); + string historicalDirectory = @"C:\temp\profile-a-old"; + FileProfileModel historicalProfileVersion = new( + profile.FileProfileId, + "Profile A", + historicalDirectory, + "ProfileARequest", + "Operator A", + "1", + "ProfileAHandler", + Guid.NewGuid(), + 1); + fileSystem.AddFile(Path.Combine(historicalDirectory, "inprogress", "existing-file"), new MockFileData("contents")); + fileProfileManager - .GetAllFileProfiles(Arg.Any()) - .Returns(Task.FromResult(Result.Success(new List { profile }))); + .GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { historicalProfileVersion, profile }))); FileProfileDirectorySynchronizer synchronizer = new FileProfileDirectorySynchronizer( configuration, @@ -56,5 +69,78 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() fileSystem.Directory.Exists(Path.Combine(profile.ListeningDirectory, "inprogress")).ShouldBeTrue(); fileSystem.Directory.Exists(profile.ProcessedDirectory).ShouldBeTrue(); fileSystem.Directory.Exists(profile.FailedDirectory).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "processed")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "failed")).ShouldBeTrue(); + fileSystem.File.Exists(Path.Combine(historicalDirectory, "inprogress", "existing-file")).ShouldBeTrue(); + } + + [Fact] + public async Task SyncAsync_WhenVersionsShareDirectory_EnsuresSharedDirectoryWithoutDeletingFiles() + { + MockFileSystem fileSystem = new(); + IFileProfileManagerImposter fileProfileManager = new(); + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary" + }) + .Build(); + string sharedDirectory = @"C:\temp\shared-profile"; + FileProfileModel firstVersion = new( + Guid.NewGuid(), "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1); + FileProfileModel secondVersion = new( + firstVersion.FileProfileId, "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2); + string existingFile = Path.Combine(sharedDirectory, "failed", "old-file"); + fileSystem.AddFile(existingFile, new MockFileData("contents")); + + fileProfileManager.GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { firstVersion, secondVersion }))); + + FileProfileDirectorySynchronizer synchronizer = new( + configuration, + fileProfileManager.Instance(), + fileSystem, + NullLogger.Instance); + + Result result = await synchronizer.SyncAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + fileSystem.File.Exists(existingFile).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "processed")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "failed")).ShouldBeTrue(); + } + + [Fact] + public async Task SyncAsync_WhenDirectoriesDifferOnlyByCase_UsesFilesystemPathSemantics() + { + MockFileSystem fileSystem = new(); + IFileProfileManagerImposter fileProfileManager = new(); + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary" + }) + .Build(); + FileProfileModel upperCaseVersion = new( + Guid.NewGuid(), "Profile A", @"C:\temp\Profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1); + FileProfileModel lowerCaseVersion = new( + upperCaseVersion.FileProfileId, "Profile A", @"C:\temp\profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2); + + fileProfileManager.GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { upperCaseVersion, lowerCaseVersion }))); + + FileProfileDirectorySynchronizer synchronizer = new( + configuration, + fileProfileManager.Instance(), + fileSystem, + NullLogger.Instance); + + Result result = await synchronizer.SyncAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(upperCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(lowerCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue(); } } diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs index 5d95d03..8f71afe 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs @@ -56,6 +56,24 @@ public async Task GetFileProfileVersion_WhenVersionDoesNotExist_ReturnsActionabl result.Message.ShouldContain(missingVersionId.ToString()); } + [Fact] + public async Task GetAllFileProfileVersions_IncludesVersionsForArchivedProfiles() + { + FileProfileAggregateRoot aggregate = this.CreateArchivedAggregate(); + IAggregateRepositoryImposter repository = new(); + repository.GetLatestVersion(Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(aggregate)); + FileProfileManager manager = new(repository.Instance()); + + Result> result = await manager.GetAllFileProfileVersions(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + result.Data.Count.ShouldBe(2); + result.Data[0].FileProfileId.ShouldBe(TestData.FileProfileId); + result.Data[0].FileProfileVersion.ShouldBe(1); + result.Data[1].FileProfileVersion.ShouldBe(2); + } + private FileProfileAggregateRoot CreateArchivedAggregate() { FileProfileAggregateRoot aggregate = FileProfileAggregateRoot.Create(); diff --git a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs index f0f5f72..6d58ed1 100644 --- a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs @@ -70,6 +70,11 @@ public async Task>> GetAllFileProfiles(Cancellatio return await this.FileProfileManager.GetAllFileProfiles(cancellationToken); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) + { + return await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken); + } + public async Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken) { diff --git a/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs b/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs index 518f50f..ac283e7 100644 --- a/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs +++ b/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs @@ -65,6 +65,15 @@ public async Task>> GetFileProfileVersions(Guid fi return Result.Success(versions); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) + { + Result aggregateResult = await this.LoadAggregate(cancellationToken); + if (aggregateResult.IsFailed) + return ResultHelpers.CreateFailure(aggregateResult); + + return Result.Success(aggregateResult.Data.GetAllProfileVersions()); + } + public async Task> GetFileProfileVersion(Guid fileProfileId, Guid fileProfileVersionId, CancellationToken cancellationToken) diff --git a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs index 62d3d64..aaab524 100644 --- a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs @@ -18,6 +18,8 @@ public interface IFileProcessorManager Task>> GetAllFileProfiles(CancellationToken cancellationToken); + Task>> GetAllFileProfileVersions(CancellationToken cancellationToken); + Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken); Task> GetFileProfileVersion(Guid fileProfileId, diff --git a/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs b/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs index 55e1c17..81b4a44 100644 --- a/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs +++ b/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs @@ -16,6 +16,8 @@ public interface IFileProfileManager Task>> GetFileProfileVersions(Guid fileProfileId, CancellationToken cancellationToken); + Task>> GetAllFileProfileVersions(CancellationToken cancellationToken); + Task> GetFileProfileVersion(Guid fileProfileId, Guid fileProfileVersionId, CancellationToken cancellationToken); Task> CreateFileProfile(CreateFileProfileRequest request, CancellationToken cancellationToken); diff --git a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs index a4ee46e..28f9128 100644 --- a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs @@ -287,7 +287,11 @@ public async Task ProcessUploadedFile(FileCommands.ProcessUploadedFileCo }, command.FileId, cancellationToken, false ); if (result.IsFailed) { - await MoveAcceptedFileToFailedDirectory(command.FileProfileId, command.FilePath, cancellationToken); + await MoveAcceptedFileToFailedDirectory(command.FileId, + command.FileProfileId, + command.FileProfileVersionId, + command.FilePath, + cancellationToken); return result; } @@ -300,11 +304,16 @@ public async Task ProcessUploadedFile(FileCommands.ProcessUploadedFileCo return processResult; } - private async Task MoveAcceptedFileToFailedDirectory(Guid fileProfileId, + private async Task MoveAcceptedFileToFailedDirectory(Guid fileId, + Guid fileProfileId, + Guid fileProfileVersionId, String filePath, CancellationToken cancellationToken) { - Result fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken); + Result fileProfileResult = await this.GetFileProfileForProcessing(fileId, + fileProfileId, + fileProfileVersionId, + cancellationToken); if (fileProfileResult.IsFailed || fileProfileResult.Data == null) return; diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs index 56896f6..0509071 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -35,16 +35,22 @@ public FileProfileDirectoryRecoveryService( public async Task RecoverInProgressFilesAsync(CancellationToken cancellationToken) { - Result> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfiles(cancellationToken); + Result> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfileVersions(cancellationToken); if (fileProfilesResult.IsFailed) { Shared.Logger.Logger.LogWarning($"Unable to load file profiles for in-progress recovery: {fileProfilesResult.Message}"); return ResultHelpers.CreateFailure(fileProfilesResult); } + HashSet recoveredDirectories = new(GetPathComparer()); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { string inProgressDirectory = Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName); + if (recoveredDirectories.Add(inProgressDirectory) == false) + { + continue; + } + if (this.FileSystem.Directory.Exists(inProgressDirectory) == false) { continue; @@ -59,6 +65,11 @@ public async Task RecoverInProgressFilesAsync(CancellationToken cancella return Result.Success(); } + private static StringComparer GetPathComparer() + { + return OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + } + private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePath, CancellationToken cancellationToken) { string fileName = Path.GetFileNameWithoutExtension(filePath); diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs index fd59969..0bc00bb 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; @@ -45,15 +46,19 @@ public async Task SyncAsync(CancellationToken cancellationToken) this.FileSystem.Directory.CreateDirectory(temporaryFileLocation); this.Logger.LogInformation("Ensured temporary file location exists at [{TemporaryFileLocation}]", temporaryFileLocation); - Result> fileProfilesResult = await this.FileProfileManager.GetAllFileProfiles(cancellationToken); + Result> fileProfilesResult = await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken); if (fileProfilesResult.IsFailed) { this.Logger.LogWarning("Error getting file profiles {Message}", fileProfilesResult.Message); return ResultHelpers.CreateFailure(fileProfilesResult); } + HashSet synchronizedDirectories = new(GetPathComparer()); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { + if (synchronizedDirectories.Add(fileProfile.ListeningDirectory) == false) + continue; + this.EnsureDirectory(Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName), "in progress"); this.EnsureDirectory(fileProfile.ProcessedDirectory, "processed"); this.EnsureDirectory(fileProfile.FailedDirectory, "failed"); @@ -62,6 +67,11 @@ public async Task SyncAsync(CancellationToken cancellationToken) return Result.Success(); } + private static StringComparer GetPathComparer() + { + return OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + } + private void EnsureDirectory(string directoryPath, string directoryName) { this.FileSystem.Directory.CreateDirectory(directoryPath); diff --git a/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs b/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs index 9f3ab31..6725bdf 100644 --- a/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs +++ b/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs @@ -197,6 +197,34 @@ public void FileProfileAggregate_Archive_WhenAlreadyArchived_IsIdempotent() aggregate.GetProfileVersions(TestData.FileProfileId).Count.ShouldBe(1); } + [Fact] + public void FileProfileAggregate_GetAllProfileVersions_IncludesArchivedProfilesAndRetainsDirectories() + { + FileProfileAggregateRoot aggregate = FileProfileAggregateRoot.Create(); + aggregate.CreateProfile(this.CreateProfile( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + LineTerminatorType.LineFeed, + TestData.SafaricomFileFormatHandler)); + + aggregate.UpdateProfile(TestData.FileProfileId, new UpdateFileProfileRequest + { + ListeningDirectory = "/home/txnproc/bulkfiles/safaricom-v2" + }); + aggregate.ArchiveProfile(TestData.FileProfileId); + + List versions = aggregate.GetAllProfileVersions(); + + versions.Count.ShouldBe(2); + versions[0].ListeningDirectory.ShouldBe(TestData.SafaricomListeningDirectory); + versions[0].ProcessedDirectory.ShouldBe($"{TestData.SafaricomListeningDirectory}/processed"); + versions[0].FailedDirectory.ShouldBe($"{TestData.SafaricomListeningDirectory}/failed"); + versions[1].ListeningDirectory.ShouldBe("/home/txnproc/bulkfiles/safaricom-v2"); + } + [Fact] public void FileProfileAggregate_Update_ListeningDirectoryUpdatesDerivedDirectories() { diff --git a/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs b/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs index 8c3d0ad..4e044ad 100644 --- a/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs +++ b/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs @@ -299,6 +299,15 @@ public static List GetProfileVersions(this FileProfileAggregat .ToList(); } + public static List GetAllProfileVersions(this FileProfileAggregate aggregate) + { + return aggregate.FileProfiles.Values + .SelectMany(profile => profile.Versions.Select(version => version.ToModel(profile.FileProfileId))) + .OrderBy(version => version.FileProfileId) + .ThenBy(version => version.FileProfileVersion) + .ToList(); + } + public static FileProfileModel GetProfileVersion(this FileProfileAggregate aggregate, Guid fileProfileId, Guid fileProfileVersionId) diff --git a/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md b/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md new file mode 100644 index 0000000..30dcf37 --- /dev/null +++ b/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md @@ -0,0 +1,61 @@ +# Profile Directory Version Safety Design + +## Context + +Issue #843 addresses directory changes after files have been accepted under an earlier file profile version. Profile versions already capture the listening directory, and issue #842 made historical versions available for lookup. The remaining risk is that directory synchronization, processing failure moves, and startup recovery still use only the active profile directory. + +## Goals + +- Keep accepted files associated with the directories recorded by their profile version. +- Ensure new files use the active profile version's directories. +- Discover historical `inprogress` files after a directory change. +- Resolve processed and failed destinations from the recorded profile version. +- Preserve historical directories and avoid silent file relocation. +- Define cleanup behavior explicitly. + +## Non-goals + +- Do not prohibit directory changes; a directory change creates a new immutable profile version. +- Do not move existing files between version directories automatically. +- Do not add an unarchive operation or a new persistence projection. +- Do not implement automatic deletion of historical directories. + +## Design + +### Historical directory discovery + +Add a profile-manager operation that returns profile versions across the complete aggregate, including archived profiles. Directory synchronization will ensure `inprogress`, `processed`, and `failed` directories for every returned version, deduplicating paths that are shared by multiple versions. + +The synchronizer will create directories only. It will not delete, rename, or migrate files from historical locations. + +### File processing destinations + +Every post-acceptance directory operation will resolve the profile version recorded on the file/import-log command. This includes the failure path that runs before normal processing and the existing processed/failed moves after processing. A missing recorded version remains an explicit failure; it must never fall back to the active profile. + +### In-progress recovery + +Startup recovery will enumerate the `inprogress` directory for every historical profile version, including versions belonging to archived profiles. Duplicate physical paths will be scanned once. Each discovered file will continue to obtain its file profile/version identifiers from the file aggregate or import-log read model, and replay will use those recorded identifiers. + +### Retention and cleanup + +Historical profile-version directories are retained indefinitely by application behavior. No automatic cleanup is performed because files may still be in flight, awaiting recovery, or required for audit/replay. Removing an old directory is an explicit operational action outside this issue, after confirming that no accepted or recoverable files remain there. + +## Error handling + +- A missing historical profile version returns the existing explicit not-found failure with the file, profile, and version identifiers where processing has that context. +- A failure to resolve a recorded version must not redirect the file to a current profile directory. +- Recovery skips malformed or unresolvable files as it does today and logs the reason. +- Directory creation failures continue to surface through the synchronizer result. + +## Verification + +Add tests covering: + +- Synchronization creates current and historical directories, including archived-profile versions, and does not remove old directories. +- A successfully processed file moves to the recorded version's processed directory after the active profile directory changes. +- A rejected or failed file moves to the recorded version's failed directory. +- Recovery finds old and current `inprogress` directories and replays commands with recorded profile/version metadata. +- Duplicate historical versions pointing to one physical directory are processed once. +- Missing historical versions fail without attempting a current-directory fallback. + +Run the focused aggregate/business-logic tests, the relevant integration tests, the complete solution build/test suite, and `git diff --check`.