From aebe73bd26a7e2c78402839972e6b184d6ccee55 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Sat, 17 Oct 2026 07:17:01 +0100 Subject: [PATCH 1/2] Migrate existing files to profile versions --- ...FileProfileVersionMigrationServiceTests.cs | 176 ++++++++++++++++++ .../FileProfileVersionMigrationService.cs | 141 ++++++++++++++ .../FileAggregateDomainEvents.cs.cs | 3 + .../FileAggregateTests.cs | 38 ++++ FileProcessor.FileAggregate/FileAggregate.cs | 23 +++ FileProcessor/Bootstrapper/MiscRegistry.cs | 1 + FileProcessor/Endpoints/MigrationEndpoints.cs | 20 ++ FileProcessor/Handlers/MigrationHandlers.cs | 15 ++ FileProcessor/Startup.cs | 1 + 9 files changed, 418 insertions(+) create mode 100644 FileProcessor.BusinessLogic.Tests/FileProfileVersionMigrationServiceTests.cs create mode 100644 FileProcessor.BusinessLogic/Services/FileProfileVersionMigrationService.cs create mode 100644 FileProcessor/Endpoints/MigrationEndpoints.cs create mode 100644 FileProcessor/Handlers/MigrationHandlers.cs diff --git a/FileProcessor.BusinessLogic.Tests/FileProfileVersionMigrationServiceTests.cs b/FileProcessor.BusinessLogic.Tests/FileProfileVersionMigrationServiceTests.cs new file mode 100644 index 0000000..c5d20c8 --- /dev/null +++ b/FileProcessor.BusinessLogic.Tests/FileProfileVersionMigrationServiceTests.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FileProcessor.BusinessLogic.Managers; +using FileProcessor.BusinessLogic.Services; +using FileProcessor.FileAggregate; +using FileProcessor.Models; +using Imposter.Abstractions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shared.DomainDrivenDesign.EventSourcing; +using Shared.EntityFramework; +using Shared.EventStore.Aggregate; +using Shouldly; +using SimpleResults; +using TransactionProcessor.Database.Contexts; +using Xunit; +using FileProfileModel = global::FileProcessor.Models.FileProfile; +using FileAggregateRoot = FileProcessor.FileAggregate.FileAggregate; + +namespace FileProcessor.BusinessLogic.Tests; + +public class FileProfileVersionMigrationServiceTests +{ + [Fact] + public async Task RunAsync_AssignsInitialVersionToLegacyFile() + { + (EstateManagementContext context, IDbContextResolverImposter resolver) = this.CreateContext(); + TransactionProcessor.Database.Entities.File file = FileProcessor.Testing.TestData.Files1[0]; + context.Files.Add(file); + await context.SaveChangesAsync(); + + IFileProfileManagerImposter profileManager = new(); + profileManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List + { + new FileProfileModel(file.FileProfileId, + "Profile", + "C:/profile", + "Request", + "Operator", + "\n", + "Handler", + FileProcessor.Testing.TestData.FileProfileVersionId, + 1) + })); + + FileAggregateRoot aggregate = FileAggregateRoot.Create(file.FileId); + aggregate.CreateFile(new FileAggregateExtensions.CreateFileRequest(file.FileImportLogId, + FileProcessor.Testing.TestData.EstateId, + file.MerchantId, + file.UserId, + file.FileProfileId, + file.FileLocation, + FileProcessor.Testing.TestData.FileUploadedDateTime, + FileProcessor.Testing.TestData.SafaricomOperatorId)); + IAggregateRepositoryImposter repository = new(); + repository.GetLatestVersion(file.FileId, Arg.Any()).ReturnsAsync(Result.Success(aggregate)); + repository.SaveChanges(Arg.Any(), Arg.Any()).ReturnsAsync(Result.Success()); + + FileProfileVersionMigrationService service = new(profileManager.Instance(), + resolver.Instance(), + repository.Instance()); + + Result result = await service.RunAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + result.Data.FilesAssigned.ShouldBe(1); + result.Data.Failures.ShouldBeEmpty(); + aggregate.GetFile().FileProfileVersionId.ShouldBe(FileProcessor.Testing.TestData.FileProfileVersionId); + } + + [Fact] + public async Task RunAsync_WhenProfileCannotBeResolved_ReportsFailureAndLeavesFileUnchanged() + { + (EstateManagementContext context, IDbContextResolverImposter resolver) = this.CreateContext(); + TransactionProcessor.Database.Entities.File file = FileProcessor.Testing.TestData.Files1[0]; + file.FileProfileId = Guid.NewGuid(); + context.Files.Add(file); + await context.SaveChangesAsync(); + + IFileProfileManagerImposter profileManager = new(); + profileManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List())); + + FileAggregateRoot aggregate = FileAggregateRoot.Create(file.FileId); + aggregate.CreateFile(new FileAggregateExtensions.CreateFileRequest(file.FileImportLogId, + FileProcessor.Testing.TestData.EstateId, + file.MerchantId, + file.UserId, + file.FileProfileId, + file.FileLocation, + FileProcessor.Testing.TestData.FileUploadedDateTime, + FileProcessor.Testing.TestData.SafaricomOperatorId)); + IAggregateRepositoryImposter repository = new(); + repository.GetLatestVersion(file.FileId, Arg.Any()).ReturnsAsync(Result.Success(aggregate)); + + FileProfileVersionMigrationService service = new(profileManager.Instance(), + resolver.Instance(), + repository.Instance()); + + Result result = await service.RunAsync(CancellationToken.None); + + result.IsSuccess.ShouldBeTrue(); + result.Data.FilesAssigned.ShouldBe(0); + result.Data.Failures.Count.ShouldBe(1); + result.Data.Failures.Single().ShouldContain(file.FileId.ToString()); + aggregate.GetFile().FileProfileVersionId.ShouldBe(Guid.Empty); + } + + [Fact] + public async Task RunAsync_WhenRepeated_SkipsAlreadyAssignedFile() + { + (EstateManagementContext context, IDbContextResolverImposter resolver) = this.CreateContext(); + TransactionProcessor.Database.Entities.File file = FileProcessor.Testing.TestData.Files1[0]; + context.Files.Add(file); + await context.SaveChangesAsync(); + + IFileProfileManagerImposter profileManager = new(); + profileManager.GetAllFileProfileVersions(Arg.Any()) + .ReturnsAsync(Result.Success(new List + { + new FileProfileModel(file.FileProfileId, + "Profile", + "C:/profile", + "Request", + "Operator", + "\n", + "Handler", + FileProcessor.Testing.TestData.FileProfileVersionId, + 1) + })); + + FileAggregateRoot aggregate = FileAggregateRoot.Create(file.FileId); + aggregate.CreateFile(new FileAggregateExtensions.CreateFileRequest(file.FileImportLogId, + FileProcessor.Testing.TestData.EstateId, + file.MerchantId, + file.UserId, + file.FileProfileId, + file.FileLocation, + FileProcessor.Testing.TestData.FileUploadedDateTime, + FileProcessor.Testing.TestData.SafaricomOperatorId)); + IAggregateRepositoryImposter repository = new(); + repository.GetLatestVersion(file.FileId, Arg.Any()).ReturnsAsync(Result.Success(aggregate)); + repository.SaveChanges(Arg.Any(), Arg.Any()).ReturnsAsync(Result.Success()); + + FileProfileVersionMigrationService service = new(profileManager.Instance(), + resolver.Instance(), + repository.Instance()); + + Result firstResult = await service.RunAsync(CancellationToken.None); + Result secondResult = await service.RunAsync(CancellationToken.None); + + firstResult.Data.FilesAssigned.ShouldBe(1); + secondResult.Data.FilesAssigned.ShouldBe(0); + secondResult.Data.FilesSkipped.ShouldBe(1); + secondResult.Data.Failures.ShouldBeEmpty(); + } + + private (EstateManagementContext Context, IDbContextResolverImposter Resolver) CreateContext() + { + DbContextOptions options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString("N")) + .Options; + EstateManagementContext context = new(options); + ServiceCollection services = new(); + services.AddTransient(_ => context); + ServiceProvider provider = services.BuildServiceProvider(); + IServiceScope scope = provider.CreateScope(); + IDbContextResolverImposter resolver = new(); + resolver.Resolve(Arg.Any(), Arg.Any()).Returns(new ResolvedDbContext(scope)); + return (context, resolver); + } +} diff --git a/FileProcessor.BusinessLogic/Services/FileProfileVersionMigrationService.cs b/FileProcessor.BusinessLogic/Services/FileProfileVersionMigrationService.cs new file mode 100644 index 0000000..7a12b66 --- /dev/null +++ b/FileProcessor.BusinessLogic/Services/FileProfileVersionMigrationService.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FileProcessor.BusinessLogic.Managers; +using FileProcessor.FileAggregate; +using FileProcessor.Models; +using Microsoft.EntityFrameworkCore; +using Shared.DomainDrivenDesign.EventSourcing; +using Shared.EntityFramework; +using Shared.EventStore.Aggregate; +using Shared.Results; +using SimpleResults; +using TransactionProcessor.Database.Contexts; +using FileAggregateRoot = FileProcessor.FileAggregate.FileAggregate; +using FileProfileModel = global::FileProcessor.Models.FileProfile; + +namespace FileProcessor.BusinessLogic.Services; + +public interface IFileProfileVersionMigrationService +{ + Task> RunAsync(CancellationToken cancellationToken); +} + +public sealed class FileProfileVersionMigrationService : IFileProfileVersionMigrationService +{ + private const String EstateManagementDatabaseName = "TransactionProcessorReadModel"; + + private readonly IFileProfileManager FileProfileManager; + private readonly IDbContextResolver Resolver; + private readonly IAggregateRepository FileAggregateRepository; + + public FileProfileVersionMigrationService(IFileProfileManager fileProfileManager, + IDbContextResolver resolver, + IAggregateRepository fileAggregateRepository) + { + this.FileProfileManager = fileProfileManager; + this.Resolver = resolver; + this.FileAggregateRepository = fileAggregateRepository; + } + + public async Task> RunAsync(CancellationToken cancellationToken) + { + Result> profileVersionsResult = await this.FileProfileManager.GetAllFileProfileVersions(cancellationToken); + if (profileVersionsResult.IsFailed) + return ResultHelpers.CreateFailure(profileVersionsResult); + + FileProfileVersionMigrationReport report = new(); + Dictionary initialVersions = this.GetInitialVersions(profileVersionsResult.Data, report); + + ResolvedDbContext resolvedContext = this.Resolver.Resolve(EstateManagementDatabaseName, null); + List files = await resolvedContext.Context.Files + .AsNoTracking() + .Select(file => new ExistingFile(file.FileId, file.FileProfileId)) + .ToListAsync(cancellationToken); + + foreach (ExistingFile file in files) + { + cancellationToken.ThrowIfCancellationRequested(); + report.FilesChecked++; + + Result aggregateResult = await this.FileAggregateRepository.GetLatestVersion(file.FileId, cancellationToken); + if (aggregateResult.IsFailed || aggregateResult.Data == null) + { + report.Failures.Add($"File [{file.FileId}] could not be loaded: {aggregateResult.Message}"); + continue; + } + + FileDetails fileDetails = aggregateResult.Data.GetFile(); + if (fileDetails.FileProfileVersionId != Guid.Empty) + { + report.FilesSkipped++; + continue; + } + + if (initialVersions.TryGetValue(file.FileProfileId, out Guid initialVersionId) == false) + { + report.Failures.Add($"File [{file.FileId}] references profile [{file.FileProfileId}], which has no usable initial version"); + continue; + } + + Result assignResult = aggregateResult.Data.AssignFileProfileVersion(initialVersionId); + if (assignResult.IsFailed) + { + report.Failures.Add($"File [{file.FileId}] could not be assigned profile version [{initialVersionId}]: {assignResult.Message}"); + continue; + } + + Result saveResult = await this.FileAggregateRepository.SaveChanges(aggregateResult.Data, cancellationToken); + if (saveResult.IsFailed) + { + report.Failures.Add($"File [{file.FileId}] could not be saved: {saveResult.Message}"); + continue; + } + + report.FilesAssigned++; + } + + return Result.Success(report); + } + + private Dictionary GetInitialVersions(List profileVersions, + FileProfileVersionMigrationReport report) + { + Dictionary initialVersions = new(); + + foreach (IGrouping profileGroup in profileVersions.GroupBy(profile => profile.FileProfileId)) + { + report.ProfilesChecked++; + List versions = profileGroup.Where(profile => profile.FileProfileVersion == 1 && profile.FileProfileVersionId != Guid.Empty).ToList(); + if (versions.Count != 1) + { + report.Failures.Add($"Profile [{profileGroup.Key}] has {versions.Count} usable initial versions"); + continue; + } + + initialVersions[profileGroup.Key] = versions[0].FileProfileVersionId; + report.ProfilesWithInitialVersion++; + } + + return initialVersions; + } + + private sealed record ExistingFile(Guid FileId, Guid FileProfileId); +} + +public sealed class FileProfileVersionMigrationReport +{ + public Int32 ProfilesChecked { get; set; } + + public Int32 ProfilesWithInitialVersion { get; set; } + + public Int32 FilesChecked { get; set; } + + public Int32 FilesAssigned { get; set; } + + public Int32 FilesSkipped { get; set; } + + public List Failures { get; } = new(); +} diff --git a/FileProcessor.File.DomainEvents/FileAggregateDomainEvents.cs.cs b/FileProcessor.File.DomainEvents/FileAggregateDomainEvents.cs.cs index 35e903a..038cf67 100644 --- a/FileProcessor.File.DomainEvents/FileAggregateDomainEvents.cs.cs +++ b/FileProcessor.File.DomainEvents/FileAggregateDomainEvents.cs.cs @@ -30,6 +30,9 @@ public record FileCreatedEvent(Guid FileId, Guid OperatorId, Guid FileProfileVersionId = default) : DomainEvent(FileId, Guid.NewGuid()); + public record FileProfileVersionAssignedEvent(Guid FileId, + Guid FileProfileVersionId) : DomainEvent(FileId, Guid.NewGuid()); + public record FileRejectedEvent(Guid FileId, Guid EstateId, Guid MerchantId, diff --git a/FileProcessor.FileAggregate.Tests/FileAggregateTests.cs b/FileProcessor.FileAggregate.Tests/FileAggregateTests.cs index 4692329..2859c15 100644 --- a/FileProcessor.FileAggregate.Tests/FileAggregateTests.cs +++ b/FileProcessor.FileAggregate.Tests/FileAggregateTests.cs @@ -13,6 +13,44 @@ namespace FileProcessor.FileAggregate.Tests public class FileAggregateTests { + [Fact] + public void FileAggregate_AssignProfileVersion_WhenFileHasNoVersion_AssignsVersion() + { + FileAggregate aggregate = FileAggregate.Create(TestData.FileId); + aggregate.CreateFile(new FileAggregateExtensions.CreateFileRequest(TestData.FileImportLogId, + TestData.EstateId, + TestData.MerchantId, + TestData.UserId, + TestData.FileProfileId, + TestData.FileLocation, + TestData.FileUploadedDateTime, + TestData.SafaricomOperatorId)); + + Result result = aggregate.AssignFileProfileVersion(TestData.FileProfileVersionId); + + result.IsSuccess.ShouldBeTrue(); + aggregate.GetFile().FileProfileVersionId.ShouldBe(TestData.FileProfileVersionId); + } + + [Fact] + public void FileAggregate_ReplayAssignedProfileVersionEvent_RestoresVersion() + { + FileAggregate aggregate = FileAggregate.Create(TestData.FileId); + aggregate.PlayEvent(new FileCreatedEvent(TestData.FileId, + TestData.FileImportLogId, + TestData.EstateId, + TestData.MerchantId, + TestData.UserId, + TestData.FileProfileId, + TestData.FileLocation, + TestData.FileUploadedDateTime, + TestData.SafaricomOperatorId)); + + aggregate.PlayEvent(new FileProfileVersionAssignedEvent(TestData.FileId, TestData.FileProfileVersionId)); + + aggregate.GetFile().FileProfileVersionId.ShouldBe(TestData.FileProfileVersionId); + } + [Fact] public void FileAggregate_CanBeCreated_IsCreated() { diff --git a/FileProcessor.FileAggregate/FileAggregate.cs b/FileProcessor.FileAggregate/FileAggregate.cs index d616e3e..89d620c 100644 --- a/FileProcessor.FileAggregate/FileAggregate.cs +++ b/FileProcessor.FileAggregate/FileAggregate.cs @@ -29,6 +29,11 @@ public static void PlayEvent(this FileAggregate aggregate, FileCreatedEvent doma aggregate.FileReceivedDateTime = domainEvent.FileReceivedDateTime; } + public static void PlayEvent(this FileAggregate aggregate, FileProfileVersionAssignedEvent domainEvent) + { + aggregate.FileProfileVersionId = domainEvent.FileProfileVersionId; + } + public static FileDetails GetFile(this FileAggregate aggregate) { return new FileDetails @@ -151,6 +156,24 @@ public static Result CreateFile(this FileAggregate aggregate, CreateFileRequest return Result.Success(); } + public static Result AssignFileProfileVersion(this FileAggregate aggregate, Guid fileProfileVersionId) + { + if (aggregate.IsCreated == false) + return Result.NotFound($"File Id {aggregate.AggregateId} has not been uploaded yet"); + + if (fileProfileVersionId == Guid.Empty) + return Result.Invalid("File profile version Id cannot be empty"); + + if (aggregate.FileProfileVersionId == fileProfileVersionId) + return Result.Success(); + + if (aggregate.FileProfileVersionId != Guid.Empty) + return Result.Invalid($"File Id {aggregate.AggregateId} already has a different file profile version"); + + aggregate.ApplyAndAppend(new FileProfileVersionAssignedEvent(aggregate.AggregateId, fileProfileVersionId)); + return Result.Success(); + } + public static void PlayEvent(this FileAggregate aggregate, FileRejectedEvent domainEvent) { aggregate.IsRejected = true; diff --git a/FileProcessor/Bootstrapper/MiscRegistry.cs b/FileProcessor/Bootstrapper/MiscRegistry.cs index b9e5d81..87e5818 100644 --- a/FileProcessor/Bootstrapper/MiscRegistry.cs +++ b/FileProcessor/Bootstrapper/MiscRegistry.cs @@ -25,6 +25,7 @@ public MiscRegistry() _ = FileProcessorDomainService.ResolveMaximumDetailRecordCount(); this.AddSingleton(); this.AddSingleton(); + this.AddSingleton(); this.AddSingleton(); this.AddSingleton(); diff --git a/FileProcessor/Endpoints/MigrationEndpoints.cs b/FileProcessor/Endpoints/MigrationEndpoints.cs new file mode 100644 index 0000000..6c4e6ba --- /dev/null +++ b/FileProcessor/Endpoints/MigrationEndpoints.cs @@ -0,0 +1,20 @@ +using System; +using FileProcessor.Handlers; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace FileProcessor.Endpoints; + +public static class MigrationEndpoints +{ + public static readonly String Route = "api/admin/migrations/profile-versioning"; + + public static void MapMigrationEndpoints(this IEndpointRouteBuilder app) + { + app.MapPost(Route, MigrationHandlers.MigrateProfileVersions) + .RequireAuthorization() + .WithTags("Migrations") + .WithName("MigrateProfileVersions"); + } +} diff --git a/FileProcessor/Handlers/MigrationHandlers.cs b/FileProcessor/Handlers/MigrationHandlers.cs new file mode 100644 index 0000000..8f79d81 --- /dev/null +++ b/FileProcessor/Handlers/MigrationHandlers.cs @@ -0,0 +1,15 @@ +using System.Threading; +using System.Threading.Tasks; +using FileProcessor.BusinessLogic.Services; +using Shared.Results.Web; + +namespace FileProcessor.Handlers; + +public static class MigrationHandlers +{ + public static async Task MigrateProfileVersions(IFileProfileVersionMigrationService migrationService, + CancellationToken cancellationToken) + { + return ResponseFactory.FromResult(await migrationService.RunAsync(cancellationToken), report => report); + } +} diff --git a/FileProcessor/Startup.cs b/FileProcessor/Startup.cs index 9735ac5..3b7e263 100644 --- a/FileProcessor/Startup.cs +++ b/FileProcessor/Startup.cs @@ -128,6 +128,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerF endpoints.MapFileEndpoints(); endpoints.MapFileImportLogEndpoints(); endpoints.MapFileProfileEndpoints(); + endpoints.MapMigrationEndpoints(); endpoints.MapHealthChecks("health", new HealthCheckOptions() { From 6504d31a116ae27dd37f2003233e356f1ece6a91 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Sat, 17 Oct 2026 07:25:15 +0100 Subject: [PATCH 2/2] Restrict profile migration to client credentials --- .../MigrationEndpointAuthorizationTests.cs | 31 +++++++++++++++++++ FileProcessor/Endpoints/MigrationEndpoints.cs | 2 ++ 2 files changed, 33 insertions(+) create mode 100644 FileProcessor.Tests/MigrationEndpointAuthorizationTests.cs diff --git a/FileProcessor.Tests/MigrationEndpointAuthorizationTests.cs b/FileProcessor.Tests/MigrationEndpointAuthorizationTests.cs new file mode 100644 index 0000000..81fed57 --- /dev/null +++ b/FileProcessor.Tests/MigrationEndpointAuthorizationTests.cs @@ -0,0 +1,31 @@ +using System.Linq; +using System.Threading.Tasks; +using FileProcessor.Endpoints; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Shared.Authorisation; +using Shouldly; +using Xunit; + +namespace FileProcessor.Tests; + +public class MigrationEndpointAuthorizationTests +{ + [Fact] + public async Task MigrationEndpoint_RequiresClientCredentialsPolicy() + { + WebApplication app = WebApplication.CreateBuilder().Build(); + app.MapMigrationEndpoints(); + await app.StartAsync(); + + RouteEndpoint endpoint = app.Services.GetRequiredService() + .Endpoints + .OfType() + .Single(endpoint => endpoint.Metadata.GetMetadata()?.EndpointName == "MigrateProfileVersions"); + + endpoint.Metadata.GetOrderedMetadata() + .ShouldContain(metadata => metadata.Policy == AuthorizationExtensions.PolicyNames.ClientCredentialsOnlyPolicy); + } +} diff --git a/FileProcessor/Endpoints/MigrationEndpoints.cs b/FileProcessor/Endpoints/MigrationEndpoints.cs index 6c4e6ba..a12ce7e 100644 --- a/FileProcessor/Endpoints/MigrationEndpoints.cs +++ b/FileProcessor/Endpoints/MigrationEndpoints.cs @@ -3,6 +3,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; +using Shared.Authorisation; namespace FileProcessor.Endpoints; @@ -14,6 +15,7 @@ public static void MapMigrationEndpoints(this IEndpointRouteBuilder app) { app.MapPost(Route, MigrationHandlers.MigrateProfileVersions) .RequireAuthorization() + .RequireAuthorization(AuthorizationExtensions.PolicyNames.ClientCredentialsOnlyPolicy) .WithTags("Migrations") .WithName("MigrateProfileVersions"); }