From 0c880e083b91079bd21ff4fd23f5931996e53945 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 18:55:52 +0100 Subject: [PATCH 1/6] Document profile directory version safety --- ...profile-directory-version-safety-design.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md diff --git a/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md b/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md new file mode 100644 index 0000000..30dcf37 --- /dev/null +++ b/docs/superpowers/specs/2026-10-16-profile-directory-version-safety-design.md @@ -0,0 +1,61 @@ +# Profile Directory Version Safety Design + +## Context + +Issue #843 addresses directory changes after files have been accepted under an earlier file profile version. Profile versions already capture the listening directory, and issue #842 made historical versions available for lookup. The remaining risk is that directory synchronization, processing failure moves, and startup recovery still use only the active profile directory. + +## Goals + +- Keep accepted files associated with the directories recorded by their profile version. +- Ensure new files use the active profile version's directories. +- Discover historical `inprogress` files after a directory change. +- Resolve processed and failed destinations from the recorded profile version. +- Preserve historical directories and avoid silent file relocation. +- Define cleanup behavior explicitly. + +## Non-goals + +- Do not prohibit directory changes; a directory change creates a new immutable profile version. +- Do not move existing files between version directories automatically. +- Do not add an unarchive operation or a new persistence projection. +- Do not implement automatic deletion of historical directories. + +## Design + +### Historical directory discovery + +Add a profile-manager operation that returns profile versions across the complete aggregate, including archived profiles. Directory synchronization will ensure `inprogress`, `processed`, and `failed` directories for every returned version, deduplicating paths that are shared by multiple versions. + +The synchronizer will create directories only. It will not delete, rename, or migrate files from historical locations. + +### File processing destinations + +Every post-acceptance directory operation will resolve the profile version recorded on the file/import-log command. This includes the failure path that runs before normal processing and the existing processed/failed moves after processing. A missing recorded version remains an explicit failure; it must never fall back to the active profile. + +### In-progress recovery + +Startup recovery will enumerate the `inprogress` directory for every historical profile version, including versions belonging to archived profiles. Duplicate physical paths will be scanned once. Each discovered file will continue to obtain its file profile/version identifiers from the file aggregate or import-log read model, and replay will use those recorded identifiers. + +### Retention and cleanup + +Historical profile-version directories are retained indefinitely by application behavior. No automatic cleanup is performed because files may still be in flight, awaiting recovery, or required for audit/replay. Removing an old directory is an explicit operational action outside this issue, after confirming that no accepted or recoverable files remain there. + +## Error handling + +- A missing historical profile version returns the existing explicit not-found failure with the file, profile, and version identifiers where processing has that context. +- A failure to resolve a recorded version must not redirect the file to a current profile directory. +- Recovery skips malformed or unresolvable files as it does today and logs the reason. +- Directory creation failures continue to surface through the synchronizer result. + +## Verification + +Add tests covering: + +- Synchronization creates current and historical directories, including archived-profile versions, and does not remove old directories. +- A successfully processed file moves to the recorded version's processed directory after the active profile directory changes. +- A rejected or failed file moves to the recorded version's failed directory. +- Recovery finds old and current `inprogress` directories and replays commands with recorded profile/version metadata. +- Duplicate historical versions pointing to one physical directory are processed once. +- Missing historical versions fail without attempting a current-directory fallback. + +Run the focused aggregate/business-logic tests, the relevant integration tests, the complete solution build/test suite, and `git diff --check`. From 62ac495070d4a759205ad549940b3444d8131acd Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 18:59:43 +0100 Subject: [PATCH 2/6] Expose all historical profile version directories --- .../DummyFileProcessorDomainService.cs | 4 +++ .../FileProfileManagerTests.cs | 18 ++++++++++++ .../Managers/FileProcessorManager.cs | 5 ++++ .../Managers/FileProfileManager.cs | 9 ++++++ .../Managers/IFileProcessorManager.cs | 2 ++ .../Managers/IFileProfileManager.cs | 2 ++ .../FileProfileAggregateTests.cs | 28 +++++++++++++++++++ .../FileProfileAggregate.cs | 9 ++++++ 8 files changed, 77 insertions(+) diff --git a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs index c5e7972..7637ff3 100644 --- a/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic.Tests/DummyFileProcessorDomainService.cs @@ -33,6 +33,10 @@ public async Task>> GetAllFileProfiles(Cancellatio return Result.Success(); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) { + return Result.Success(); + } + public async Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken) { return Result.Success(); diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs index 5d95d03..8f71afe 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileManagerTests.cs @@ -56,6 +56,24 @@ public async Task GetFileProfileVersion_WhenVersionDoesNotExist_ReturnsActionabl result.Message.ShouldContain(missingVersionId.ToString()); } + [Fact] + public async Task GetAllFileProfileVersions_IncludesVersionsForArchivedProfiles() + { + FileProfileAggregateRoot aggregate = this.CreateArchivedAggregate(); + IAggregateRepositoryImposter repository = new(); + repository.GetLatestVersion(Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(aggregate)); + FileProfileManager manager = new(repository.Instance()); + + Result> result = await manager.GetAllFileProfileVersions(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + result.Data.Count.ShouldBe(2); + result.Data[0].FileProfileId.ShouldBe(TestData.FileProfileId); + result.Data[0].FileProfileVersion.ShouldBe(1); + result.Data[1].FileProfileVersion.ShouldBe(2); + } + private FileProfileAggregateRoot CreateArchivedAggregate() { FileProfileAggregateRoot aggregate = FileProfileAggregateRoot.Create(); diff --git a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs index f0f5f72..6d58ed1 100644 --- a/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/FileProcessorManager.cs @@ -70,6 +70,11 @@ public async Task>> GetAllFileProfiles(Cancellatio return await this.FileProfileManager.GetAllFileProfiles(cancellationToken); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) + { + return await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken); + } + public async Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken) { diff --git a/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs b/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs index 518f50f..ac283e7 100644 --- a/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs +++ b/FileProcessor.BusinessLogic/Managers/FileProfileManager.cs @@ -65,6 +65,15 @@ public async Task>> GetFileProfileVersions(Guid fi return Result.Success(versions); } + public async Task>> GetAllFileProfileVersions(CancellationToken cancellationToken) + { + Result aggregateResult = await this.LoadAggregate(cancellationToken); + if (aggregateResult.IsFailed) + return ResultHelpers.CreateFailure(aggregateResult); + + return Result.Success(aggregateResult.Data.GetAllProfileVersions()); + } + public async Task> GetFileProfileVersion(Guid fileProfileId, Guid fileProfileVersionId, CancellationToken cancellationToken) diff --git a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs index 62d3d64..aaab524 100644 --- a/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs +++ b/FileProcessor.BusinessLogic/Managers/IFileProcessorManager.cs @@ -18,6 +18,8 @@ public interface IFileProcessorManager Task>> GetAllFileProfiles(CancellationToken cancellationToken); + Task>> GetAllFileProfileVersions(CancellationToken cancellationToken); + Task> GetFileProfile(Guid fileProfileId, CancellationToken cancellationToken); Task> GetFileProfileVersion(Guid fileProfileId, diff --git a/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs b/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs index 55e1c17..81b4a44 100644 --- a/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs +++ b/FileProcessor.BusinessLogic/Managers/IFileProfileManager.cs @@ -16,6 +16,8 @@ public interface IFileProfileManager Task>> GetFileProfileVersions(Guid fileProfileId, CancellationToken cancellationToken); + Task>> GetAllFileProfileVersions(CancellationToken cancellationToken); + Task> GetFileProfileVersion(Guid fileProfileId, Guid fileProfileVersionId, CancellationToken cancellationToken); Task> CreateFileProfile(CreateFileProfileRequest request, CancellationToken cancellationToken); diff --git a/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs b/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs index 9f3ab31..6725bdf 100644 --- a/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs +++ b/FileProcessor.FileProfileAggregate.Tests/FileProfileAggregateTests.cs @@ -197,6 +197,34 @@ public void FileProfileAggregate_Archive_WhenAlreadyArchived_IsIdempotent() aggregate.GetProfileVersions(TestData.FileProfileId).Count.ShouldBe(1); } + [Fact] + public void FileProfileAggregate_GetAllProfileVersions_IncludesArchivedProfilesAndRetainsDirectories() + { + FileProfileAggregateRoot aggregate = FileProfileAggregateRoot.Create(); + aggregate.CreateProfile(this.CreateProfile( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + LineTerminatorType.LineFeed, + TestData.SafaricomFileFormatHandler)); + + aggregate.UpdateProfile(TestData.FileProfileId, new UpdateFileProfileRequest + { + ListeningDirectory = "/home/txnproc/bulkfiles/safaricom-v2" + }); + aggregate.ArchiveProfile(TestData.FileProfileId); + + List versions = aggregate.GetAllProfileVersions(); + + versions.Count.ShouldBe(2); + versions[0].ListeningDirectory.ShouldBe(TestData.SafaricomListeningDirectory); + versions[0].ProcessedDirectory.ShouldBe($"{TestData.SafaricomListeningDirectory}/processed"); + versions[0].FailedDirectory.ShouldBe($"{TestData.SafaricomListeningDirectory}/failed"); + versions[1].ListeningDirectory.ShouldBe("/home/txnproc/bulkfiles/safaricom-v2"); + } + [Fact] public void FileProfileAggregate_Update_ListeningDirectoryUpdatesDerivedDirectories() { diff --git a/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs b/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs index 8c3d0ad..4e044ad 100644 --- a/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs +++ b/FileProcessor.FileProfileAggregate/FileProfileAggregate.cs @@ -299,6 +299,15 @@ public static List GetProfileVersions(this FileProfileAggregat .ToList(); } + public static List GetAllProfileVersions(this FileProfileAggregate aggregate) + { + return aggregate.FileProfiles.Values + .SelectMany(profile => profile.Versions.Select(version => version.ToModel(profile.FileProfileId))) + .OrderBy(version => version.FileProfileId) + .ThenBy(version => version.FileProfileVersion) + .ToList(); + } + public static FileProfileModel GetProfileVersion(this FileProfileAggregate aggregate, Guid fileProfileId, Guid fileProfileVersionId) From c76ced4e9ee8004527f772d0e0f02442e20c550b Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 19:00:37 +0100 Subject: [PATCH 3/6] Synchronize historical profile version directories --- .../FileProfileDirectorySynchronizerTests.cs | 60 ++++++++++++++++++- .../FileProfileDirectorySynchronizer.cs | 7 ++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs index 4402fa6..c90e091 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs @@ -19,7 +19,7 @@ namespace FileProcessor.BusinessLogic.Tests; public class FileProfileDirectorySynchronizerTests { [Fact] - public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() + public async Task SyncAsync_CreatesTemporaryAndHistoricalProfileVersionDirectories() { MockFileSystem fileSystem = new MockFileSystem(); IFileProfileManagerImposter fileProfileManager = new IFileProfileManagerImposter(); @@ -39,9 +39,22 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() "1", "ProfileAHandler"); + string historicalDirectory = @"C:\temp\profile-a-old"; + FileProfileModel historicalProfileVersion = new( + profile.FileProfileId, + "Profile A", + historicalDirectory, + "ProfileARequest", + "Operator A", + "1", + "ProfileAHandler", + Guid.NewGuid(), + 1); + fileSystem.AddFile(Path.Combine(historicalDirectory, "inprogress", "existing-file"), new MockFileData("contents")); + fileProfileManager - .GetAllFileProfiles(Arg.Any()) - .Returns(Task.FromResult(Result.Success(new List { profile }))); + .GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { historicalProfileVersion, profile }))); FileProfileDirectorySynchronizer synchronizer = new FileProfileDirectorySynchronizer( configuration, @@ -56,5 +69,46 @@ public async Task SyncAsync_CreatesTemporaryAndProfileDirectories() fileSystem.Directory.Exists(Path.Combine(profile.ListeningDirectory, "inprogress")).ShouldBeTrue(); fileSystem.Directory.Exists(profile.ProcessedDirectory).ShouldBeTrue(); fileSystem.Directory.Exists(profile.FailedDirectory).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "processed")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(historicalDirectory, "failed")).ShouldBeTrue(); + fileSystem.File.Exists(Path.Combine(historicalDirectory, "inprogress", "existing-file")).ShouldBeTrue(); + } + + [Fact] + public async Task SyncAsync_WhenVersionsShareDirectory_EnsuresSharedDirectoryWithoutDeletingFiles() + { + MockFileSystem fileSystem = new(); + IFileProfileManagerImposter fileProfileManager = new(); + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary" + }) + .Build(); + string sharedDirectory = @"C:\temp\shared-profile"; + FileProfileModel firstVersion = new( + Guid.NewGuid(), "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1); + FileProfileModel secondVersion = new( + firstVersion.FileProfileId, "Profile A", sharedDirectory, "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2); + string existingFile = Path.Combine(sharedDirectory, "failed", "old-file"); + fileSystem.AddFile(existingFile, new MockFileData("contents")); + + fileProfileManager.GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { firstVersion, secondVersion }))); + + FileProfileDirectorySynchronizer synchronizer = new( + configuration, + fileProfileManager.Instance(), + fileSystem, + NullLogger.Instance); + + Result result = await synchronizer.SyncAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + fileSystem.File.Exists(existingFile).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "processed")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "failed")).ShouldBeTrue(); } } diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs index fd59969..b52602a 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; @@ -45,15 +46,19 @@ public async Task SyncAsync(CancellationToken cancellationToken) this.FileSystem.Directory.CreateDirectory(temporaryFileLocation); this.Logger.LogInformation("Ensured temporary file location exists at [{TemporaryFileLocation}]", temporaryFileLocation); - Result> fileProfilesResult = await this.FileProfileManager.GetAllFileProfiles(cancellationToken); + Result> fileProfilesResult = await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken); if (fileProfilesResult.IsFailed) { this.Logger.LogWarning("Error getting file profiles {Message}", fileProfilesResult.Message); return ResultHelpers.CreateFailure(fileProfilesResult); } + HashSet synchronizedDirectories = new(StringComparer.OrdinalIgnoreCase); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { + if (synchronizedDirectories.Add(fileProfile.ListeningDirectory) == false) + continue; + this.EnsureDirectory(Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName), "in progress"); this.EnsureDirectory(fileProfile.ProcessedDirectory, "processed"); this.EnsureDirectory(fileProfile.FailedDirectory, "failed"); From 5b3286200d2a1151f72901d5dcdca5ed20d38c6f Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 19:01:57 +0100 Subject: [PATCH 4/6] Move early processing failures using recorded profile version --- .../FileProcessorDomainServiceTests.cs | 42 +++++++++++++++++++ .../Services/FileProcessorDomainService.cs | 15 +++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs index 989dab6..4bde44b 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs @@ -393,6 +393,48 @@ public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileT result.IsFailed.ShouldBeTrue(); this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue(); } + + [Fact] + public async Task ProcessUploadedFile_WhenRecordedVersionDiffers_MovesAcceptedFileToRecordedFailedDirectory() + { + FileProfile currentProfile = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/current-profile", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + FileProfile historicalProfile = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/historical-profile", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + TestData.FileProfileVersionId, + 1); + this.FileProcessorManager.GetFileProfile(Arg.Any(), Arg.Any()).ReturnsAsync(currentProfile); + this.FileProcessorManager.GetFileProfileVersion(Arg.Any(), Arg.Any(), Arg.Any()).ReturnsAsync(historicalProfile); + this.SecurityServiceClient.GetToken(Arg.Any(), Arg.Any(), Arg.Any()).ReturnsAsync(Result.Failure()); + this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData("D,1,1,1")); + this.FileSystem.AddDirectory(currentProfile.FailedDirectory); + this.FileSystem.AddDirectory(historicalProfile.FailedDirectory); + + FileCommands.ProcessUploadedFileCommand command = TestData.ProcessUploadedFileCommand with + { + FileProfileVersionId = historicalProfile.FileProfileVersionId + }; + + Result result = await this.FileProcessorDomainService.ProcessUploadedFile(command, CancellationToken.None); + + result.IsFailed.ShouldBeTrue(); + this.FileSystem.File.Exists($"{historicalProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue(); + this.FileSystem.File.Exists($"{currentProfile.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeFalse(); + } [Fact] public async Task FileRequestHandler_ProcessUploadedFileRequest_FileNotFound_RequestIsHandled() diff --git a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs index a4ee46e..28f9128 100644 --- a/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs @@ -287,7 +287,11 @@ public async Task ProcessUploadedFile(FileCommands.ProcessUploadedFileCo }, command.FileId, cancellationToken, false ); if (result.IsFailed) { - await MoveAcceptedFileToFailedDirectory(command.FileProfileId, command.FilePath, cancellationToken); + await MoveAcceptedFileToFailedDirectory(command.FileId, + command.FileProfileId, + command.FileProfileVersionId, + command.FilePath, + cancellationToken); return result; } @@ -300,11 +304,16 @@ public async Task ProcessUploadedFile(FileCommands.ProcessUploadedFileCo return processResult; } - private async Task MoveAcceptedFileToFailedDirectory(Guid fileProfileId, + private async Task MoveAcceptedFileToFailedDirectory(Guid fileId, + Guid fileProfileId, + Guid fileProfileVersionId, String filePath, CancellationToken cancellationToken) { - Result fileProfileResult = await this.FileProcessorManager.GetFileProfile(fileProfileId, cancellationToken); + Result fileProfileResult = await this.GetFileProfileForProcessing(fileId, + fileProfileId, + fileProfileVersionId, + cancellationToken); if (fileProfileResult.IsFailed || fileProfileResult.Data == null) return; From 2929311265afddf81ba1dc827eaaf566c7d09e17 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 19:05:14 +0100 Subject: [PATCH 5/6] Recover files from historical profile directories --- ...ileProfileDirectoryRecoveryServiceTests.cs | 106 +++++++++++++++++- .../FileProfileDirectoryRecoveryService.cs | 8 +- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs index d586e4c..1731052 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -40,7 +40,7 @@ public async Task RecoverInProgressFilesAsync_ReplaysFilesFoundInInProgressDirec fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); fileProcessorManager - .GetAllFileProfiles(Arg.Any()) + .GetAllFileProfileVersions(Arg.Any()) .ReturnsAsync(Result.Success(new List { fileProfile })); fileProcessorManager @@ -86,7 +86,7 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays fileSystem.AddFile(inProgressFilePath, new MockFileData("D,1,1,1")); fileProcessorManager - .GetAllFileProfiles(Arg.Any()) + .GetAllFileProfileVersions(Arg.Any()) .ReturnsAsync(Result.Success(new List { fileProfile })); fileProcessorManager @@ -140,4 +140,106 @@ public async Task RecoverInProgressFilesAsync_WhenFileAggregateIsMissing_Replays Path.GetFileName(command.FilePath) == Path.GetFileName(inProgressFilePath)), Arg.Any()).Called(Count.Once()); } + + [Fact] + public async Task RecoverInProgressFilesAsync_ReplaysFilesFromHistoricalAndCurrentDirectories() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + IFileProcessorManagerImposter fileProcessorManager = new(); + IMediatorImposter mediator = new(); + MockFileSystem fileSystem = new(); + FileProfileModel historicalVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/safaricom-old", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + TestData.FileProfileVersionId, + 1); + FileProfileModel currentVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + "/home/txnproc/bulkfiles/safaricom-current", + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + string historicalFilePath = $"{historicalVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}"; + string currentFilePath = $"{currentVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId2:N}"; + fileSystem.AddFile(historicalFilePath, new MockFileData("D,1,1,1")); + fileSystem.AddFile(currentFilePath, new MockFileData("D,1,1,1")); + + fileProcessorManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List { historicalVersion, currentVersion })); + fileProcessorManager.GetFile(Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails())); + mediator.Send(Arg>.Any(), Arg.Any()) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Instance(), + mediator.Instance(), + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Send(Arg>.Is(request => request is FileCommands.ProcessUploadedFileCommand command && + (Path.GetFileName(command.FilePath) == Path.GetFileName(historicalFilePath) || + Path.GetFileName(command.FilePath) == Path.GetFileName(currentFilePath))), + Arg.Any()).Called(Count.Exactly(2)); + } + + [Fact] + public async Task RecoverInProgressFilesAsync_WhenVersionsShareDirectory_ReplaysFileOnce() + { + Shared.Logger.Logger.Initialise(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + IFileProcessorManagerImposter fileProcessorManager = new(); + IMediatorImposter mediator = new(); + MockFileSystem fileSystem = new(); + FileProfileModel firstVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 1); + FileProfileModel secondVersion = new( + TestData.FileProfileId, + TestData.SafaricomProfileName, + TestData.SafaricomListeningDirectory, + TestData.SafaricomRequestType, + TestData.SafaricomOperatorIdentifier, + "\n", + TestData.SafaricomFileFormatHandler, + Guid.NewGuid(), + 2); + string filePath = $"{firstVersion.ListeningDirectory}/inprogress/{TestData.EstateId:N}-{TestData.FileId:N}"; + fileSystem.AddFile(filePath, new MockFileData("D,1,1,1")); + fileProcessorManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List { firstVersion, secondVersion })); + fileProcessorManager.GetFile(Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsAsync(Result.Success(TestData.GetCreatedFileDetails())); + mediator.Send(Arg>.Any(), Arg.Any()) + .ReturnsAsync(Result.Success()); + + FileProfileDirectoryRecoveryService service = new( + fileProcessorManager.Instance(), + mediator.Instance(), + fileSystem); + + Result result = await service.RecoverInProgressFilesAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + mediator.Send(Arg>.Any(), Arg.Any()).Called(Count.Once()); + } } diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs index 56896f6..d56c5e7 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -35,16 +35,22 @@ public FileProfileDirectoryRecoveryService( public async Task RecoverInProgressFilesAsync(CancellationToken cancellationToken) { - Result> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfiles(cancellationToken); + Result> fileProfilesResult = await this.FileProcessorManager.GetAllFileProfileVersions(cancellationToken); if (fileProfilesResult.IsFailed) { Shared.Logger.Logger.LogWarning($"Unable to load file profiles for in-progress recovery: {fileProfilesResult.Message}"); return ResultHelpers.CreateFailure(fileProfilesResult); } + HashSet recoveredDirectories = new(StringComparer.OrdinalIgnoreCase); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { string inProgressDirectory = Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName); + if (recoveredDirectories.Add(inProgressDirectory) == false) + { + continue; + } + if (this.FileSystem.Directory.Exists(inProgressDirectory) == false) { continue; From 5434d5614924096dd927bc27336be754ca41aff4 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Fri, 16 Oct 2026 19:11:26 +0100 Subject: [PATCH 6/6] Respect filesystem casing when deduplicating directories --- ...ileProfileDirectoryRecoveryServiceTests.cs | 1 + .../FileProfileDirectorySynchronizerTests.cs | 32 +++++++++++++++++++ .../FileProfileDirectoryRecoveryService.cs | 7 +++- .../FileProfileDirectorySynchronizer.cs | 7 +++- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs index 1731052..c4f0d48 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectoryRecoveryServiceTests.cs @@ -242,4 +242,5 @@ public async Task RecoverInProgressFilesAsync_WhenVersionsShareDirectory_Replays result.IsSuccess.ShouldBeTrue(); mediator.Send(Arg>.Any(), Arg.Any()).Called(Count.Once()); } + } diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs index c90e091..aa4f916 100644 --- a/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs +++ b/FileProcessor.BusinessLogic.Tests/FileProfileDirectorySynchronizerTests.cs @@ -111,4 +111,36 @@ public async Task SyncAsync_WhenVersionsShareDirectory_EnsuresSharedDirectoryWit fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "processed")).ShouldBeTrue(); fileSystem.Directory.Exists(Path.Combine(sharedDirectory, "failed")).ShouldBeTrue(); } + + [Fact] + public async Task SyncAsync_WhenDirectoriesDifferOnlyByCase_UsesFilesystemPathSemantics() + { + MockFileSystem fileSystem = new(); + IFileProfileManagerImposter fileProfileManager = new(); + IConfiguration configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["AppSettings:TemporaryFileLocation"] = @"C:\temp\temporary" + }) + .Build(); + FileProfileModel upperCaseVersion = new( + Guid.NewGuid(), "Profile A", @"C:\temp\Profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 1); + FileProfileModel lowerCaseVersion = new( + upperCaseVersion.FileProfileId, "Profile A", @"C:\temp\profile", "ProfileARequest", "Operator A", "1", "ProfileAHandler", Guid.NewGuid(), 2); + + fileProfileManager.GetAllFileProfileVersions(Arg.Any()) + .Returns(Task.FromResult(Result.Success(new List { upperCaseVersion, lowerCaseVersion }))); + + FileProfileDirectorySynchronizer synchronizer = new( + configuration, + fileProfileManager.Instance(), + fileSystem, + NullLogger.Instance); + + Result result = await synchronizer.SyncAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(upperCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue(); + fileSystem.Directory.Exists(Path.Combine(lowerCaseVersion.ListeningDirectory, "inprogress")).ShouldBeTrue(); + } } diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs index d56c5e7..0509071 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectoryRecoveryService.cs @@ -42,7 +42,7 @@ public async Task RecoverInProgressFilesAsync(CancellationToken cancella return ResultHelpers.CreateFailure(fileProfilesResult); } - HashSet recoveredDirectories = new(StringComparer.OrdinalIgnoreCase); + HashSet recoveredDirectories = new(GetPathComparer()); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { string inProgressDirectory = Path.Combine(fileProfile.ListeningDirectory, InProgressDirectoryName); @@ -65,6 +65,11 @@ public async Task RecoverInProgressFilesAsync(CancellationToken cancella return Result.Success(); } + private static StringComparer GetPathComparer() + { + return OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + } + private async Task RecoverFileAsync(FileProfileModel fileProfile, string filePath, CancellationToken cancellationToken) { string fileName = Path.GetFileNameWithoutExtension(filePath); diff --git a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs index b52602a..0bc00bb 100644 --- a/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs +++ b/FileProcessor.BusinessLogic/Services/FileProfileDirectorySynchronizer.cs @@ -53,7 +53,7 @@ public async Task SyncAsync(CancellationToken cancellationToken) return ResultHelpers.CreateFailure(fileProfilesResult); } - HashSet synchronizedDirectories = new(StringComparer.OrdinalIgnoreCase); + HashSet synchronizedDirectories = new(GetPathComparer()); foreach (FileProfileModel fileProfile in fileProfilesResult.Data) { if (synchronizedDirectories.Add(fileProfile.ListeningDirectory) == false) @@ -67,6 +67,11 @@ public async Task SyncAsync(CancellationToken cancellationToken) return Result.Success(); } + private static StringComparer GetPathComparer() + { + return OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + } + private void EnsureDirectory(string directoryPath, string directoryName) { this.FileSystem.Directory.CreateDirectory(directoryPath);