From 78dd6c148eef40f1a7402e8536630e4fd7efd6d2 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Wed, 26 Aug 2026 10:13:42 +0100 Subject: [PATCH 1/2] Update transaction processing support tools --- ScheduledTasks/AutoScavenge/scavenge.ps1 | 9 +- .../DailySettlementProcessingTask.xml | Bin 4098 -> 4000 bytes .../Invoke-DailySettlementProcessing.ps1 | 13 +- .../MerchantProcessingConfigurationStore.cs | 18 +- .../MerchantProcessingOptionsValidator.cs | 116 +++-- .../MerchantFileProcessorDbContext.cs | 380 ++++++++------- .../Program.cs | 183 +++---- ...anagementEndpointRouteBuilderExtensions.cs | 458 ++++++++---------- .../Reporting/FileStatusReportService.cs | 135 +++--- .../Reporting/OperationsDashboardService.cs | 140 +++--- .../Services/MerchantProcessingService.cs | 308 ++++++------ .../Worker.cs | 109 +++-- .../merchant-processing.bootstrap.json | 2 +- .../Pages/Config.cshtml.cs | 2 +- .../Pages/Merchants.cshtml.cs | 2 +- .../Persistence/MerchantPosSettingsStore.cs | 50 +- .../Runtime/ApiClient.cs | 6 +- .../Runtime/MerchantPosSettingsSnapshot.cs | 13 + .../Runtime/MerchantRuntime.cs | 15 +- .../appsettings.staging.json | 6 +- .../DataGenerator/Program.cs | 15 +- .../JobTestDriver/Program.cs | 80 +-- .../TransactionDataGeneratorService.cs | 54 +-- .../EstateSetupFunctions.cs | 137 ++++-- .../EventStoreFunctions.cs | 20 +- .../Program.cs | 48 +- .../identityserverconfig.json | 16 +- .../setupconfig.json | 6 +- .../setupconfig.production.json | 20 +- .../setupconfig.staging.json | 10 +- docker-compose-master.yml | 32 +- docker-compose-prerelease.yml | 38 +- docker-compose.yml | 32 +- 33 files changed, 1302 insertions(+), 1171 deletions(-) diff --git a/ScheduledTasks/AutoScavenge/scavenge.ps1 b/ScheduledTasks/AutoScavenge/scavenge.ps1 index ae5e2db..c7e6c3e 100644 --- a/ScheduledTasks/AutoScavenge/scavenge.ps1 +++ b/ScheduledTasks/AutoScavenge/scavenge.ps1 @@ -1,7 +1,6 @@ param( [string]$BaseUrl = "http://localhost:2113", - [string]$Username, - [string]$Password + [System.Management.Automation.PSCredential]$Credential ) $BaseUrl = $BaseUrl.TrimEnd('/') @@ -12,8 +11,8 @@ $headers = @{ 'Content-Type' = 'application/json' } -if ($Username -and $Password) { - $pair = "$Username`:$Password" +if ($Credential) { + $pair = "$($Credential.UserName)`:$($Credential.GetNetworkCredential().Password)" $bytes = [System.Text.Encoding]::ASCII.GetBytes($pair) $encoded = [Convert]::ToBase64String($bytes) $headers['Authorization'] = "Basic $encoded" @@ -25,4 +24,4 @@ try { catch { Write-Error "POST to $uri failed: $($_.Exception.Message)" exit 1 -} \ No newline at end of file +} diff --git a/ScheduledTasks/SettlementProcessing/DailySettlementProcessingTask.xml b/ScheduledTasks/SettlementProcessing/DailySettlementProcessingTask.xml index 13882de3134c6a77ef072e511fffad536d580ae0..b8bbad3f722929d7d711912ea679dd12d4724214 100644 GIT binary patch delta 11 ScmZotSRlW_fq(KM{saIR!~_KZ delta 107 zcmZ1=-=wg?fj?e>L6^aqA%`K8A(bJIp@bnANGCHC0m%{u1qLOC6b3^EO9msLSP~GM qFqi>FEEtS|tP} TryLoadFromRelationalTablesAsync(MerchantFileProcessorDbContext dbContext, CancellationToken cancellationToken) + { + var snapshot = await LoadRelationalConfigurationAsync(dbContext, cancellationToken); + if (snapshot is null) + { + return null; + } + + var json = JsonSerializer.Serialize(snapshot.Options, jsonSerializerOptions); + return new MerchantProcessingConfigurationSnapshot(snapshot.Options, json, snapshot.UpdatedUtc); + } + + private async Task LoadRelationalConfigurationAsync(MerchantFileProcessorDbContext dbContext, CancellationToken cancellationToken) { var authentication = await dbContext.MerchantProcessingAuthenticationRecords.AsNoTracking().FirstOrDefaultAsync(record => record.Id == 1, cancellationToken); var fileProcessing = await dbContext.MerchantProcessingFileProcessingRecords.AsNoTracking().FirstOrDefaultAsync(record => record.Id == 1, cancellationToken); @@ -147,7 +159,6 @@ private MerchantProcessingConfigurationSnapshot BuildBootstrapSnapshot() return null; } - var profileIds = fileProfiles.ToDictionary(record => record.Id); var fileProfileFields = await LoadFieldRecordsAsync(dbContext.MerchantProcessingFileProfileFieldRecords, cancellationToken); var headerFieldRecords = await LoadFieldRecordsAsync(dbContext.MerchantProcessingFileProfileHeaderFieldRecords, cancellationToken); var trailerFieldRecords = await LoadFieldRecordsAsync(dbContext.MerchantProcessingFileProfileTrailerFieldRecords, cancellationToken); @@ -243,8 +254,7 @@ private MerchantProcessingConfigurationSnapshot BuildBootstrapSnapshot() contractDefinitions.Select(record => record.UpdatedUtc), merchants.Select(record => record.UpdatedUtc)); - var json = JsonSerializer.Serialize(options, jsonSerializerOptions); - return new MerchantProcessingConfigurationSnapshot(options, json, updatedUtc); + return new RelationalConfigurationSnapshot(options, updatedUtc); } private async Task TryLoadFromLegacyConfigurationAsync( @@ -267,6 +277,8 @@ private MerchantProcessingConfigurationSnapshot BuildBootstrapSnapshot() return new MerchantProcessingConfigurationSnapshot(options, json, updatedUtc); } + private sealed record RelationalConfigurationSnapshot(MerchantProcessingOptions Options, DateTimeOffset UpdatedUtc); + private async Task HasNormalizedConfigurationAsync(MerchantFileProcessorDbContext dbContext, CancellationToken cancellationToken) { return await dbContext.MerchantProcessingAuthenticationRecords.AnyAsync(record => record.Id == 1, cancellationToken) || diff --git a/TransactionProcessing.MerchantFileProcessor/Configuration/MerchantProcessingOptionsValidator.cs b/TransactionProcessing.MerchantFileProcessor/Configuration/MerchantProcessingOptionsValidator.cs index 1ef0d1b..609d310 100644 --- a/TransactionProcessing.MerchantFileProcessor/Configuration/MerchantProcessingOptionsValidator.cs +++ b/TransactionProcessing.MerchantFileProcessor/Configuration/MerchantProcessingOptionsValidator.cs @@ -4,65 +4,111 @@ namespace TransactionProcessing.MerchantFileProcessor.Configuration; public static class MerchantProcessingOptionsValidator { public static bool Validate(MerchantProcessingOptions options) { - if (string.IsNullOrWhiteSpace(options.Authentication.ClientId) || string.IsNullOrWhiteSpace(options.Authentication.ClientSecret)) { - return false; - } + return HasValidAuthentication(options) + && HasValidFileProcessing(options) + && HasValidTransactionGeneration(options) + && HasValidPolling(options) + && HasValidMerchantScan(options) + && HasValidFileProfiles(options, out var fileProfileIds) + && HasValidContracts(options, fileProfileIds) + && HasValidMerchants(options); + } - if (string.IsNullOrWhiteSpace(options.FileProcessing.UserId) || !Guid.TryParse(options.FileProcessing.UserId, out _)) { - return false; - } + private static bool HasValidAuthentication(MerchantProcessingOptions options) => + !string.IsNullOrWhiteSpace(options.Authentication.ClientId) && !string.IsNullOrWhiteSpace(options.Authentication.ClientSecret); + + private static bool HasValidFileProcessing(MerchantProcessingOptions options) => + !string.IsNullOrWhiteSpace(options.FileProcessing.UserId) && Guid.TryParse(options.FileProcessing.UserId, out _); + + private static bool HasValidTransactionGeneration(MerchantProcessingOptions options) => + options.TransactionGeneration.MinimumTransactionsPerContract > 1 && + options.TransactionGeneration.MaximumTransactionsPerContract >= options.TransactionGeneration.MinimumTransactionsPerContract; + + private static bool HasValidPolling(MerchantProcessingOptions options) => + options.FileStatusPolling.PollIntervalSeconds > 0; + + private static bool HasValidMerchantScan(MerchantProcessingOptions options) => + options.MerchantScanIntervalSeconds > 0; - if (options.FileProfiles.Count == 0) { + private static bool HasValidFileProfiles(MerchantProcessingOptions options, out HashSet fileProfileIds) + { + fileProfileIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (options.FileProfiles.Count == 0) + { return false; } - if (options.TransactionGeneration.MinimumTransactionsPerContract <= 1 || options.TransactionGeneration.MaximumTransactionsPerContract < options.TransactionGeneration.MinimumTransactionsPerContract) { - return false; + foreach (var fileProfile in options.FileProfiles) + { + if (!IsValidFileProfile(fileProfile, fileProfileIds)) + { + return false; + } } - if (options.FileStatusPolling.PollIntervalSeconds <= 0) { + return true; + } + + private static bool IsValidFileProfile(FileProfileOptions fileProfile, ISet fileProfileIds) + { + if (string.IsNullOrWhiteSpace(fileProfile.FileProfileId) || + string.IsNullOrWhiteSpace(fileProfile.FileProcessorFileProfileId) || + !fileProfileIds.Add(fileProfile.FileProfileId) || + !Guid.TryParse(fileProfile.FileProcessorFileProfileId, out _) || + !FileProfileFormats.All.Contains(fileProfile.Format) || + string.IsNullOrWhiteSpace(fileProfile.FileExtension) || + fileProfile.Fields.Count == 0) + { return false; } - if (options.MerchantScanIntervalSeconds <= 0) { + if (fileProfile.Format.Equals(FileProfileFormats.Delimited, StringComparison.OrdinalIgnoreCase) && + string.IsNullOrWhiteSpace(fileProfile.Delimited.Delimiter)) + { return false; } - if (options.ContractDefinitions.Count == 0) { + return AreFieldsValid(fileProfile.Fields) && + AreFieldsValid(fileProfile.Delimited.HeaderFields) && + AreFieldsValid(fileProfile.Delimited.TrailerFields); + } + + private static bool HasValidContracts(MerchantProcessingOptions options, ISet fileProfileIds) + { + if (options.ContractDefinitions.Count == 0) + { return false; } - var fileProfileIds = new HashSet(StringComparer.OrdinalIgnoreCase); var contractIds = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var fileProfile in options.FileProfiles) { - if (string.IsNullOrWhiteSpace(fileProfile.FileProfileId) || string.IsNullOrWhiteSpace(fileProfile.FileProcessorFileProfileId) || !fileProfileIds.Add(fileProfile.FileProfileId) || !Guid.TryParse(fileProfile.FileProcessorFileProfileId, out _) || !FileProfileFormats.All.Contains(fileProfile.Format) || string.IsNullOrWhiteSpace(fileProfile.FileExtension) || fileProfile.Fields.Count == 0) { - return false; - } - - if (fileProfile.Format.Equals(FileProfileFormats.Delimited, StringComparison.OrdinalIgnoreCase) && string.IsNullOrWhiteSpace(fileProfile.Delimited.Delimiter)) { - return false; - } - - if (!AreFieldsValid(fileProfile.Fields)) { - return false; - } - - if (!AreFieldsValid(fileProfile.Delimited.HeaderFields) || !AreFieldsValid(fileProfile.Delimited.TrailerFields)) { + foreach (var contractDefinition in options.ContractDefinitions) + { + if (string.IsNullOrWhiteSpace(contractDefinition.ContractId) || + string.IsNullOrWhiteSpace(contractDefinition.FileProfileId) || + !contractIds.Add(contractDefinition.ContractId) || + !fileProfileIds.Contains(contractDefinition.FileProfileId)) + { return false; } } - foreach (var contractDefinition in options.ContractDefinitions) { - if (string.IsNullOrWhiteSpace(contractDefinition.ContractId) || string.IsNullOrWhiteSpace(contractDefinition.FileProfileId) || !contractIds.Add(contractDefinition.ContractId) || !fileProfileIds.Contains(contractDefinition.FileProfileId)) { - return false; - } - } + return true; + } - foreach (var merchant in options.Merchants) { + private static bool HasValidMerchants(MerchantProcessingOptions options) + { + foreach (var merchant in options.Merchants) + { var configuredTimes = merchant.RunTimesUtc.Count > 0 ? merchant.RunTimesUtc : [merchant.RunAtUtc]; - if (string.IsNullOrWhiteSpace(merchant.EstateId) || string.IsNullOrWhiteSpace(merchant.MerchantId) || !Guid.TryParse(merchant.EstateId, out _) || !Guid.TryParse(merchant.MerchantId, out _) || configuredTimes.Count == 0 || configuredTimes.Any(runTime => !TimeOnly.TryParseExact(runTime, "HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out _))) { + if (string.IsNullOrWhiteSpace(merchant.EstateId) || + string.IsNullOrWhiteSpace(merchant.MerchantId) || + !Guid.TryParse(merchant.EstateId, out _) || + !Guid.TryParse(merchant.MerchantId, out _) || + configuredTimes.Count == 0 || + configuredTimes.Any(runTime => !TimeOnly.TryParseExact(runTime, "HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out _))) + { return false; } } diff --git a/TransactionProcessing.MerchantFileProcessor/Persistence/MerchantFileProcessorDbContext.cs b/TransactionProcessing.MerchantFileProcessor/Persistence/MerchantFileProcessorDbContext.cs index 3290a46..2ed8ba1 100644 --- a/TransactionProcessing.MerchantFileProcessor/Persistence/MerchantFileProcessorDbContext.cs +++ b/TransactionProcessing.MerchantFileProcessor/Persistence/MerchantFileProcessorDbContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; namespace TransactionProcessing.MerchantFileProcessor.Persistence; @@ -38,186 +39,203 @@ public sealed class MerchantFileProcessorDbContext(DbContextOptions(entity => - { - entity.ToTable("FileSendRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.EstateId).HasMaxLength(64); - entity.Property(record => record.MerchantName).HasMaxLength(256); - entity.Property(record => record.ContractId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.ContractName).HasMaxLength(256); - entity.Property(record => record.FileName).HasMaxLength(260); - entity.Property(record => record.FileProfileId).HasMaxLength(128); - entity.Property(record => record.Format).HasMaxLength(32); - entity.Property(record => record.FileProcessorFileId).HasMaxLength(64); - entity.Property(record => record.ScheduledRunUtc); - entity.Property(record => record.Status).HasMaxLength(32).IsRequired(); - entity.Property(record => record.ErrorMessage).HasMaxLength(2048); - entity.HasIndex(record => new { record.MerchantId, record.ProcessedUtc }); - entity.HasIndex(record => new { record.MerchantId, record.ContractId, record.ScheduledRunUtc }); - entity.HasIndex(record => record.ProcessedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("FileSendRecordLineStatuses"); - entity.HasKey(record => record.Id); - entity.Property(record => record.LineData).HasMaxLength(4096); - entity.Property(record => record.ProcessingStatus).HasMaxLength(32).IsRequired(); - entity.Property(record => record.RejectionReason).HasMaxLength(2048); - entity.HasIndex(record => new { record.FileSendRecordId, record.LineNumber }).IsUnique(); - entity.HasOne() - .WithMany(record => record.LineStatuses) - .HasForeignKey(record => record.FileSendRecordId) - .OnDelete(DeleteBehavior.Cascade); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingAuthenticationRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.ClientId).HasMaxLength(128).IsRequired(); - entity.Property(record => record.ClientSecret).HasMaxLength(256).IsRequired(); - entity.Property(record => record.Scope).HasMaxLength(512); - entity.Property(record => record.Audience).HasMaxLength(512); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileProcessingRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.UserId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingTransactionGenerationRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileStatusPollingRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingMerchantScanRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileProfiles"); - entity.HasKey(record => record.Id); - entity.Property(record => record.FileProfileId).HasMaxLength(128).IsRequired(); - entity.Property(record => record.FileProcessorFileProfileId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.Format).HasMaxLength(32).IsRequired(); - entity.Property(record => record.FileExtension).HasMaxLength(32).IsRequired(); - entity.Property(record => record.FileNamePattern).HasMaxLength(260); - entity.Property(record => record.ContentType).HasMaxLength(128); - entity.Property(record => record.Delimiter).HasMaxLength(8); - entity.Property(record => record.RootPropertyName).HasMaxLength(128); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => record.FileProfileId).IsUnique(); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileProfileFields"); - entity.HasKey(record => record.Id); - entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); - entity.Property(record => record.Source).HasMaxLength(128); - entity.Property(record => record.Format).HasMaxLength(64); - entity.Property(record => record.Value).HasMaxLength(256); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileProfileHeaderFields"); - entity.HasKey(record => record.Id); - entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); - entity.Property(record => record.Source).HasMaxLength(128); - entity.Property(record => record.Format).HasMaxLength(64); - entity.Property(record => record.Value).HasMaxLength(256); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingFileProfileTrailerFields"); - entity.HasKey(record => record.Id); - entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); - entity.Property(record => record.Source).HasMaxLength(128); - entity.Property(record => record.Format).HasMaxLength(64); - entity.Property(record => record.Value).HasMaxLength(256); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingContractDefinitions"); - entity.HasKey(record => record.Id); - entity.Property(record => record.ContractId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.FileProfileId).HasMaxLength(128).IsRequired(); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => record.ContractId).IsUnique(); - entity.HasIndex(record => record.FileProfileId); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingMerchants"); - entity.HasKey(record => record.Id); - entity.Property(record => record.Name).HasMaxLength(256).IsRequired(); - entity.Property(record => record.EstateId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.RunAtUtc).HasMaxLength(16).IsRequired(); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => record.MerchantId).IsUnique(); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingMerchantRunTimes"); - entity.HasKey(record => record.Id); - entity.Property(record => record.RunTimeUtc).HasMaxLength(16).IsRequired(); - entity.Property(record => record.UpdatedUtc); - entity.HasIndex(record => new { record.MerchantRecordId, record.SortOrder }); - entity.HasOne() - .WithMany() - .HasForeignKey(record => record.MerchantRecordId) - .OnDelete(DeleteBehavior.Cascade); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantProcessingConfigurationRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.ConfigurationJson).IsRequired(); - entity.Property(record => record.UpdatedUtc); - }); - - modelBuilder.Entity(entity => - { - entity.ToTable("MerchantRunRecords"); - entity.HasKey(record => record.Id); - entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); - entity.Property(record => record.MerchantName).HasMaxLength(256); - entity.Property(record => record.Status).HasMaxLength(32).IsRequired(); - entity.Property(record => record.ErrorMessage).HasMaxLength(2048); - entity.HasIndex(record => new { record.MerchantId, record.ScheduledRunUtc, record.CompletedUtc }); - }); + ConfigureFileSendRecord(modelBuilder.Entity()); + ConfigureFileSendRecordLineStatus(modelBuilder.Entity()); + ConfigureMerchantProcessingAuthenticationRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileProcessingRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingTransactionGenerationRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileStatusPollingRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingMerchantScanRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileProfileRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileProfileFieldRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileProfileHeaderFieldRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingFileProfileTrailerFieldRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingContractDefinitionRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingMerchantRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingMerchantRunTimeRecord(modelBuilder.Entity()); + ConfigureMerchantProcessingConfigurationRecord(modelBuilder.Entity()); + ConfigureMerchantRunRecord(modelBuilder.Entity()); + } + + private static void ConfigureFileSendRecord(EntityTypeBuilder entity) + { + entity.ToTable("FileSendRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.EstateId).HasMaxLength(64); + entity.Property(record => record.MerchantName).HasMaxLength(256); + entity.Property(record => record.ContractId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.ContractName).HasMaxLength(256); + entity.Property(record => record.FileName).HasMaxLength(260); + entity.Property(record => record.FileProfileId).HasMaxLength(128); + entity.Property(record => record.Format).HasMaxLength(32); + entity.Property(record => record.FileProcessorFileId).HasMaxLength(64); + entity.Property(record => record.ScheduledRunUtc); + entity.Property(record => record.Status).HasMaxLength(32).IsRequired(); + entity.Property(record => record.ErrorMessage).HasMaxLength(2048); + entity.HasIndex(record => new { record.MerchantId, record.ProcessedUtc }); + entity.HasIndex(record => new { record.MerchantId, record.ContractId, record.ScheduledRunUtc }); + entity.HasIndex(record => record.ProcessedUtc); + } + + private static void ConfigureFileSendRecordLineStatus(EntityTypeBuilder entity) + { + entity.ToTable("FileSendRecordLineStatuses"); + entity.HasKey(record => record.Id); + entity.Property(record => record.LineData).HasMaxLength(4096); + entity.Property(record => record.ProcessingStatus).HasMaxLength(32).IsRequired(); + entity.Property(record => record.RejectionReason).HasMaxLength(2048); + entity.HasIndex(record => new { record.FileSendRecordId, record.LineNumber }).IsUnique(); + entity.HasOne() + .WithMany(record => record.LineStatuses) + .HasForeignKey(record => record.FileSendRecordId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureMerchantProcessingAuthenticationRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingAuthenticationRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.ClientId).HasMaxLength(128).IsRequired(); + entity.Property(record => record.ClientSecret).HasMaxLength(256).IsRequired(); + entity.Property(record => record.Scope).HasMaxLength(512); + entity.Property(record => record.Audience).HasMaxLength(512); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantProcessingFileProcessingRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileProcessingRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.UserId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantProcessingTransactionGenerationRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingTransactionGenerationRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantProcessingFileStatusPollingRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileStatusPollingRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantProcessingMerchantScanRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingMerchantScanRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantProcessingFileProfileRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileProfiles"); + entity.HasKey(record => record.Id); + entity.Property(record => record.FileProfileId).HasMaxLength(128).IsRequired(); + entity.Property(record => record.FileProcessorFileProfileId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.Format).HasMaxLength(32).IsRequired(); + entity.Property(record => record.FileExtension).HasMaxLength(32).IsRequired(); + entity.Property(record => record.FileNamePattern).HasMaxLength(260); + entity.Property(record => record.ContentType).HasMaxLength(128); + entity.Property(record => record.Delimiter).HasMaxLength(8); + entity.Property(record => record.RootPropertyName).HasMaxLength(128); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => record.FileProfileId).IsUnique(); + } + + private static void ConfigureMerchantProcessingFileProfileFieldRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileProfileFields"); + entity.HasKey(record => record.Id); + entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); + entity.Property(record => record.Source).HasMaxLength(128); + entity.Property(record => record.Format).HasMaxLength(64); + entity.Property(record => record.Value).HasMaxLength(256); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); + } + + private static void ConfigureMerchantProcessingFileProfileHeaderFieldRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileProfileHeaderFields"); + entity.HasKey(record => record.Id); + entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); + entity.Property(record => record.Source).HasMaxLength(128); + entity.Property(record => record.Format).HasMaxLength(64); + entity.Property(record => record.Value).HasMaxLength(256); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); + } + + private static void ConfigureMerchantProcessingFileProfileTrailerFieldRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingFileProfileTrailerFields"); + entity.HasKey(record => record.Id); + entity.Property(record => record.Name).HasMaxLength(128).IsRequired(); + entity.Property(record => record.Source).HasMaxLength(128); + entity.Property(record => record.Format).HasMaxLength(64); + entity.Property(record => record.Value).HasMaxLength(256); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => new { record.FileProfileRecordId, record.SortOrder }); + } + + private static void ConfigureMerchantProcessingContractDefinitionRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingContractDefinitions"); + entity.HasKey(record => record.Id); + entity.Property(record => record.ContractId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.FileProfileId).HasMaxLength(128).IsRequired(); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => record.ContractId).IsUnique(); + entity.HasIndex(record => record.FileProfileId); + } + + private static void ConfigureMerchantProcessingMerchantRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingMerchants"); + entity.HasKey(record => record.Id); + entity.Property(record => record.Name).HasMaxLength(256).IsRequired(); + entity.Property(record => record.EstateId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.RunAtUtc).HasMaxLength(16).IsRequired(); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => record.MerchantId).IsUnique(); + } + + private static void ConfigureMerchantProcessingMerchantRunTimeRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingMerchantRunTimes"); + entity.HasKey(record => record.Id); + entity.Property(record => record.RunTimeUtc).HasMaxLength(16).IsRequired(); + entity.Property(record => record.UpdatedUtc); + entity.HasIndex(record => new { record.MerchantRecordId, record.SortOrder }); + entity.HasOne() + .WithMany() + .HasForeignKey(record => record.MerchantRecordId) + .OnDelete(DeleteBehavior.Cascade); + } + + private static void ConfigureMerchantProcessingConfigurationRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantProcessingConfigurationRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.ConfigurationJson).IsRequired(); + entity.Property(record => record.UpdatedUtc); + } + + private static void ConfigureMerchantRunRecord(EntityTypeBuilder entity) + { + entity.ToTable("MerchantRunRecords"); + entity.HasKey(record => record.Id); + entity.Property(record => record.MerchantId).HasMaxLength(64).IsRequired(); + entity.Property(record => record.MerchantName).HasMaxLength(256); + entity.Property(record => record.Status).HasMaxLength(32).IsRequired(); + entity.Property(record => record.ErrorMessage).HasMaxLength(2048); + entity.HasIndex(record => new { record.MerchantId, record.ScheduledRunUtc, record.CompletedUtc }); } } diff --git a/TransactionProcessing.MerchantFileProcessor/Program.cs b/TransactionProcessing.MerchantFileProcessor/Program.cs index cf35629..c230b24 100644 --- a/TransactionProcessing.MerchantFileProcessor/Program.cs +++ b/TransactionProcessing.MerchantFileProcessor/Program.cs @@ -225,67 +225,33 @@ static bool ShouldLogCategory( static async Task EnsurePersistenceSchemaAsync(MerchantFileProcessorDbContext dbContext, CancellationToken cancellationToken) { - var existingColumns = await dbContext.Database - .SqlQueryRaw("SELECT name AS Value FROM pragma_table_info('FileSendRecords');") - .ToListAsync(cancellationToken); - - if (!existingColumns.Contains("MerchantName", StringComparer.OrdinalIgnoreCase)) - { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN MerchantName TEXT NULL;", - cancellationToken); - } - - if (!existingColumns.Contains("ContractName", StringComparer.OrdinalIgnoreCase)) - { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN ContractName TEXT NULL;", - cancellationToken); - } - - if (!existingColumns.Contains("FileContent", StringComparer.OrdinalIgnoreCase)) - { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN FileContent TEXT NULL;", - cancellationToken); - } - - if (!existingColumns.Contains("EstateId", StringComparer.OrdinalIgnoreCase)) - { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN EstateId TEXT NULL;", - cancellationToken); - } - - if (!existingColumns.Contains("FileProcessorFileId", StringComparer.OrdinalIgnoreCase)) - { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN FileProcessorFileId TEXT NULL;", - cancellationToken); - } + var existingColumns = await GetExistingColumnsAsync(dbContext, cancellationToken); + await EnsureFileSendSchemaAsync(dbContext, existingColumns, cancellationToken); + await EnsureMerchantProcessingSchemaAsync(dbContext, cancellationToken); +} - if (!existingColumns.Contains("ProcessingCompleted", StringComparer.OrdinalIgnoreCase)) +static async Task EnsureFileSendSchemaAsync( + MerchantFileProcessorDbContext dbContext, + ISet existingColumns, + CancellationToken cancellationToken) +{ + foreach (var (columnName, sql) in new[] { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN ProcessingCompleted INTEGER NOT NULL DEFAULT 0;", - cancellationToken); - } - - if (!existingColumns.Contains("LastStatusCheckUtc", StringComparer.OrdinalIgnoreCase)) + ("MerchantName", "ALTER TABLE FileSendRecords ADD COLUMN MerchantName TEXT NULL;"), + ("ContractName", "ALTER TABLE FileSendRecords ADD COLUMN ContractName TEXT NULL;"), + ("FileContent", "ALTER TABLE FileSendRecords ADD COLUMN FileContent TEXT NULL;"), + ("EstateId", "ALTER TABLE FileSendRecords ADD COLUMN EstateId TEXT NULL;"), + ("FileProcessorFileId", "ALTER TABLE FileSendRecords ADD COLUMN FileProcessorFileId TEXT NULL;"), + ("ProcessingCompleted", "ALTER TABLE FileSendRecords ADD COLUMN ProcessingCompleted INTEGER NOT NULL DEFAULT 0;"), + ("LastStatusCheckUtc", "ALTER TABLE FileSendRecords ADD COLUMN LastStatusCheckUtc TEXT NULL;"), + ("ScheduledRunUtc", "ALTER TABLE FileSendRecords ADD COLUMN ScheduledRunUtc TEXT NOT NULL DEFAULT '0001-01-01T00:00:00+00:00';") + }) { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN LastStatusCheckUtc TEXT NULL;", - cancellationToken); + await EnsureColumnAsync(dbContext, existingColumns, columnName, sql, cancellationToken); } - if (!existingColumns.Contains("ScheduledRunUtc", StringComparer.OrdinalIgnoreCase)) + await ExecuteSqlStatementsAsync(dbContext, new[] { - await dbContext.Database.ExecuteSqlRawAsync( - "ALTER TABLE FileSendRecords ADD COLUMN ScheduledRunUtc TEXT NOT NULL DEFAULT '0001-01-01T00:00:00+00:00';", - cancellationToken); - } - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS FileSendRecordLineStatuses ( Id INTEGER NOT NULL CONSTRAINT PK_FileSendRecordLineStatuses PRIMARY KEY AUTOINCREMENT, @@ -300,13 +266,16 @@ CONSTRAINT FK_FileSendRecordLineStatuses_FileSendRecords_FileSendRecordId FOREIGN KEY (FileSendRecordId) REFERENCES FileSendRecords (Id) ON DELETE CASCADE ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( "CREATE UNIQUE INDEX IF NOT EXISTS IX_FileSendRecordLineStatuses_FileSendRecordId_LineNumber ON FileSendRecordLineStatuses (FileSendRecordId, LineNumber);", - cancellationToken); + }, cancellationToken); +} - await dbContext.Database.ExecuteSqlRawAsync( +static async Task EnsureMerchantProcessingSchemaAsync( + MerchantFileProcessorDbContext dbContext, + CancellationToken cancellationToken) +{ + await ExecuteSqlStatementsAsync(dbContext, new[] + { """ CREATE TABLE IF NOT EXISTS MerchantRunRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantRunRecords PRIMARY KEY AUTOINCREMENT, @@ -319,9 +288,6 @@ CREATE TABLE IF NOT EXISTS MerchantRunRecords ( CompletedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingAuthenticationRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingAuthenticationRecords PRIMARY KEY, @@ -332,9 +298,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingAuthenticationRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProcessingRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileProcessingRecords PRIMARY KEY, @@ -342,9 +305,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProcessingRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingTransactionGenerationRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingTransactionGenerationRecords PRIMARY KEY, @@ -353,9 +313,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingTransactionGenerationRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileStatusPollingRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileStatusPollingRecords PRIMARY KEY, @@ -363,9 +320,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingFileStatusPollingRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingMerchantScanRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingMerchantScanRecords PRIMARY KEY, @@ -373,9 +327,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingMerchantScanRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProfiles ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileProfiles PRIMARY KEY AUTOINCREMENT, @@ -393,9 +344,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProfiles ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProfileFields ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileProfileFields PRIMARY KEY AUTOINCREMENT, @@ -410,9 +358,6 @@ CONSTRAINT FK_MerchantProcessingFileProfileFields_MerchantProcessingFileProfiles FOREIGN KEY (FileProfileRecordId) REFERENCES MerchantProcessingFileProfiles (Id) ON DELETE CASCADE ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProfileHeaderFields ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileProfileHeaderFields PRIMARY KEY AUTOINCREMENT, @@ -427,9 +372,6 @@ CONSTRAINT FK_MerchantProcessingFileProfileHeaderFields_MerchantProcessingFilePr FOREIGN KEY (FileProfileRecordId) REFERENCES MerchantProcessingFileProfiles (Id) ON DELETE CASCADE ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingFileProfileTrailerFields ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingFileProfileTrailerFields PRIMARY KEY AUTOINCREMENT, @@ -444,9 +386,6 @@ CONSTRAINT FK_MerchantProcessingFileProfileTrailerFields_MerchantProcessingFileP FOREIGN KEY (FileProfileRecordId) REFERENCES MerchantProcessingFileProfiles (Id) ON DELETE CASCADE ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingContractDefinitions ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingContractDefinitions PRIMARY KEY AUTOINCREMENT, @@ -456,9 +395,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingContractDefinitions ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingMerchants ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingMerchants PRIMARY KEY AUTOINCREMENT, @@ -471,9 +407,6 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingMerchants ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingMerchantRunTimes ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingMerchantRunTimes PRIMARY KEY AUTOINCREMENT, @@ -485,9 +418,6 @@ CONSTRAINT FK_MerchantProcessingMerchantRunTimes_MerchantProcessingMerchants_Mer FOREIGN KEY (MerchantRecordId) REFERENCES MerchantProcessingMerchants (Id) ON DELETE CASCADE ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( """ CREATE TABLE IF NOT EXISTS MerchantProcessingConfigurationRecords ( Id INTEGER NOT NULL CONSTRAINT PK_MerchantProcessingConfigurationRecords PRIMARY KEY AUTOINCREMENT, @@ -495,31 +425,46 @@ CREATE TABLE IF NOT EXISTS MerchantProcessingConfigurationRecords ( UpdatedUtc TEXT NOT NULL ); """, - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( "CREATE INDEX IF NOT EXISTS IX_MerchantRunRecords_MerchantId_ScheduledRunUtc_CompletedUtc ON MerchantRunRecords (MerchantId, ScheduledRunUtc, CompletedUtc);", - cancellationToken); - - await dbContext.Database.ExecuteSqlRawAsync( "CREATE UNIQUE INDEX IF NOT EXISTS IX_MerchantProcessingFileProfiles_FileProfileId ON MerchantProcessingFileProfiles (FileProfileId);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( "CREATE INDEX IF NOT EXISTS IX_MerchantProcessingFileProfileFields_FileProfileRecordId_SortOrder ON MerchantProcessingFileProfileFields (FileProfileRecordId, SortOrder);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( "CREATE INDEX IF NOT EXISTS IX_MerchantProcessingFileProfileHeaderFields_FileProfileRecordId_SortOrder ON MerchantProcessingFileProfileHeaderFields (FileProfileRecordId, SortOrder);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( "CREATE INDEX IF NOT EXISTS IX_MerchantProcessingFileProfileTrailerFields_FileProfileRecordId_SortOrder ON MerchantProcessingFileProfileTrailerFields (FileProfileRecordId, SortOrder);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( "CREATE UNIQUE INDEX IF NOT EXISTS IX_MerchantProcessingContractDefinitions_ContractId ON MerchantProcessingContractDefinitions (ContractId);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( "CREATE UNIQUE INDEX IF NOT EXISTS IX_MerchantProcessingMerchants_MerchantId ON MerchantProcessingMerchants (MerchantId);", - cancellationToken); - await dbContext.Database.ExecuteSqlRawAsync( - "CREATE INDEX IF NOT EXISTS IX_MerchantProcessingMerchantRunTimes_MerchantRecordId_SortOrder ON MerchantProcessingMerchantRunTimes (MerchantRecordId, SortOrder);", - cancellationToken); + "CREATE INDEX IF NOT EXISTS IX_MerchantProcessingMerchantRunTimes_MerchantRecordId_SortOrder ON MerchantProcessingMerchantRunTimes (MerchantRecordId, SortOrder);" + }, cancellationToken); +} + +static async Task> GetExistingColumnsAsync(MerchantFileProcessorDbContext dbContext, CancellationToken cancellationToken) +{ + var existingColumns = await dbContext.Database + .SqlQueryRaw("SELECT name AS Value FROM pragma_table_info('FileSendRecords');") + .ToListAsync(cancellationToken); + + return new HashSet(existingColumns, StringComparer.OrdinalIgnoreCase); +} + +static async Task EnsureColumnAsync( + MerchantFileProcessorDbContext dbContext, + ISet existingColumns, + string columnName, + string sql, + CancellationToken cancellationToken) +{ + if (!existingColumns.Contains(columnName)) + { + await dbContext.Database.ExecuteSqlRawAsync(sql, cancellationToken); + } +} + +static async Task ExecuteSqlStatementsAsync( + MerchantFileProcessorDbContext dbContext, + IEnumerable sqlStatements, + CancellationToken cancellationToken) +{ + foreach (var sql in sqlStatements) + { + await dbContext.Database.ExecuteSqlRawAsync(sql, cancellationToken); + } } diff --git a/TransactionProcessing.MerchantFileProcessor/Reporting/ConfigurationManagementEndpointRouteBuilderExtensions.cs b/TransactionProcessing.MerchantFileProcessor/Reporting/ConfigurationManagementEndpointRouteBuilderExtensions.cs index a0dc5a3..ab28692 100644 --- a/TransactionProcessing.MerchantFileProcessor/Reporting/ConfigurationManagementEndpointRouteBuilderExtensions.cs +++ b/TransactionProcessing.MerchantFileProcessor/Reporting/ConfigurationManagementEndpointRouteBuilderExtensions.cs @@ -9,290 +9,248 @@ public static class ConfigurationManagementEndpointRouteBuilderExtensions { public static IEndpointRouteBuilder MapConfigurationManagementEndpoints(this IEndpointRouteBuilder endpoints) { - endpoints.MapPost("/api/configuration/runtime", async ( - HttpRequest request, - IMerchantProcessingConfigurationStore configurationStore, - CancellationToken cancellationToken) => + endpoints.MapPost("/api/configuration/runtime", HandleRuntimeConfigurationAsync); + endpoints.MapPost("/api/configuration/merchants", HandleMerchantConfigurationAsync); + endpoints.MapPost("/api/configuration/contracts", HandleContractConfigurationAsync); + endpoints.MapPost("/api/configuration/file-profiles", HandleFileProfileConfigurationAsync); + endpoints.MapPost("/api/configuration/file-profiles/{fileProfileId}/fields/{section}", HandleFileProfileFieldConfigurationAsync); + + return endpoints; + } + + private static async Task HandleRuntimeConfigurationAsync( + HttpRequest request, + IMerchantProcessingConfigurationStore configurationStore, + CancellationToken cancellationToken) + { + var form = await request.ReadFormAsync(cancellationToken); + var returnUrl = GetReturnUrl(form, "/ops/config/runtime"); + var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); + var template = Clone(snapshot.Options); + var options = new MerchantProcessingOptions { - var form = await request.ReadFormAsync(cancellationToken); - var returnUrl = GetReturnUrl(form, "/ops/config/runtime"); - var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); - var options = Clone(snapshot.Options); - options = new MerchantProcessingOptions + Authentication = new AuthenticationOptions { - Authentication = new AuthenticationOptions - { - ClientId = form["ClientId"].ToString(), - ClientSecret = form["ClientSecret"].ToString(), - Scope = GetOptional(form, "Scope"), - Audience = GetOptional(form, "Audience") - }, - FileProcessing = new FileProcessingOptions - { - UserId = form["UserId"].ToString() - }, - TransactionGeneration = new TransactionGenerationOptions - { - MinimumTransactionsPerContract = ParseInt(form["MinimumTransactionsPerContract"], 5), - MaximumTransactionsPerContract = ParseInt(form["MaximumTransactionsPerContract"], 25) - }, - FileStatusPolling = new FileStatusPollingOptions - { - PollIntervalSeconds = ParseInt(form["PollIntervalSeconds"], 30) - }, - MerchantScanIntervalSeconds = ParseInt(form["MerchantScanIntervalSeconds"], 5), - ContractDefinitions = options.ContractDefinitions, - FileProfiles = options.FileProfiles, - Merchants = options.Merchants - }; - - try + ClientId = form["ClientId"].ToString(), + ClientSecret = form["ClientSecret"].ToString(), + Scope = GetOptional(form, "Scope"), + Audience = GetOptional(form, "Audience") + }, + FileProcessing = new FileProcessingOptions { UserId = form["UserId"].ToString() }, + TransactionGeneration = new TransactionGenerationOptions { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect($"{returnUrl}?saved=1"); - } - catch (Exception ex) + MinimumTransactionsPerContract = ParseInt(form["MinimumTransactionsPerContract"], 5), + MaximumTransactionsPerContract = ParseInt(form["MaximumTransactionsPerContract"], 25) + }, + FileStatusPolling = new FileStatusPollingOptions { PollIntervalSeconds = ParseInt(form["PollIntervalSeconds"], 30) }, + MerchantScanIntervalSeconds = ParseInt(form["MerchantScanIntervalSeconds"], 5), + ContractDefinitions = template.ContractDefinitions, + FileProfiles = template.FileProfiles, + Merchants = template.Merchants + }; + + return await SaveWithRedirectAsync(configurationStore, options, returnUrl, cancellationToken); + } + + private static async Task HandleMerchantConfigurationAsync( + HttpRequest request, + IMerchantProcessingConfigurationStore configurationStore, + CancellationToken cancellationToken) + { + var form = await request.ReadFormAsync(cancellationToken); + var returnUrl = GetReturnUrl(form, "/ops/config/merchants"); + var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); + var options = Clone(snapshot.Options); + var originalMerchantId = GetOptional(form, "OriginalMerchantId"); + var merchantId = GetRequired(form, "MerchantId"); + var merchantIndex = options.Merchants.FindIndex(entry => + string.Equals(entry.MerchantId, originalMerchantId ?? merchantId, StringComparison.OrdinalIgnoreCase)); + + if (ParseBool(form["Delete"])) + { + if (merchantIndex < 0) { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); + return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString("Merchant not found.")}"); } - }); - endpoints.MapPost("/api/configuration/merchants", async ( - HttpRequest request, - IMerchantProcessingConfigurationStore configurationStore, - CancellationToken cancellationToken) => + options.Merchants.RemoveAt(merchantIndex); + return await SaveWithRedirectAsync(configurationStore, options, "/ops/config/merchants", cancellationToken, "removed=1"); + } + + var merchant = new MerchantOptions { - var form = await request.ReadFormAsync(cancellationToken); - var returnUrl = GetReturnUrl(form, "/ops/config/merchants"); - var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); - var options = Clone(snapshot.Options); - var originalMerchantId = GetOptional(form, "OriginalMerchantId"); - var merchantId = GetRequired(form, "MerchantId"); - var merchantIndex = options.Merchants.FindIndex(entry => - string.Equals(entry.MerchantId, originalMerchantId ?? merchantId, StringComparison.OrdinalIgnoreCase)); - - if (ParseBool(form["Delete"])) - { - if (merchantIndex < 0) - { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString("Merchant not found.")}"); - } + Name = GetRequired(form, "Name"), + Enabled = ParseBool(form["Enabled"]), + EstateId = GetRequired(form, "EstateId"), + MerchantId = merchantId, + RunAtUtc = GetOptional(form, "RunAtUtc") ?? "02:00:00", + RunTimesUtc = ParseRunTimes(GetOptional(form, "RunTimesUtc")) + }; + + if (merchantIndex >= 0) + { + options.Merchants[merchantIndex] = merchant; + } + else + { + options.Merchants.Add(merchant); + } - options.Merchants.RemoveAt(merchantIndex); + return await SaveWithRedirectAsync(configurationStore, options, returnUrl, cancellationToken); + } - try - { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect("/ops/config/merchants?removed=1"); - } - catch (Exception ex) - { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); - } - } + private static async Task HandleContractConfigurationAsync( + HttpRequest request, + IMerchantProcessingConfigurationStore configurationStore, + CancellationToken cancellationToken) + { + var form = await request.ReadFormAsync(cancellationToken); + var returnUrl = GetReturnUrl(form, "/ops/config/contracts"); + var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); + var options = Clone(snapshot.Options); + var originalContractId = GetOptional(form, "OriginalContractId"); + var contractId = GetRequired(form, "ContractId"); + var contractIndex = options.ContractDefinitions.FindIndex(entry => + string.Equals(entry.ContractId, originalContractId ?? contractId, StringComparison.OrdinalIgnoreCase)); + + var contract = new ContractDefinitionOptions + { + ContractId = contractId, + FileProfileId = GetRequired(form, "FileProfileId") + }; - var merchant = new MerchantOptions - { - Name = GetRequired(form, "Name"), - Enabled = ParseBool(form["Enabled"]), - EstateId = GetRequired(form, "EstateId"), - MerchantId = merchantId, - RunAtUtc = GetOptional(form, "RunAtUtc") ?? "02:00:00", - RunTimesUtc = ParseRunTimes(GetOptional(form, "RunTimesUtc")) - }; + if (contractIndex >= 0) + { + options.ContractDefinitions[contractIndex] = contract; + } + else + { + options.ContractDefinitions.Add(contract); + } - if (merchantIndex >= 0) - { - options.Merchants[merchantIndex] = merchant; - } - else - { - options.Merchants.Add(merchant); - } + return await SaveWithRedirectAsync(configurationStore, options, returnUrl, cancellationToken); + } - try + private static async Task HandleFileProfileConfigurationAsync( + HttpRequest request, + IMerchantProcessingConfigurationStore configurationStore, + CancellationToken cancellationToken) + { + var form = await request.ReadFormAsync(cancellationToken); + var returnUrl = GetReturnUrl(form, "/ops/config/file-profiles"); + var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); + var options = Clone(snapshot.Options); + var originalFileProfileId = GetOptional(form, "OriginalFileProfileId"); + var fileProfileId = GetRequired(form, "FileProfileId"); + var profileIndex = options.FileProfiles.FindIndex(entry => + string.Equals(entry.FileProfileId, originalFileProfileId ?? fileProfileId, StringComparison.OrdinalIgnoreCase)); + + var existingProfile = profileIndex >= 0 ? options.FileProfiles[profileIndex] : new FileProfileOptions(); + var profile = new FileProfileOptions + { + FileProfileId = fileProfileId, + FileProcessorFileProfileId = GetRequired(form, "FileProcessorFileProfileId"), + Format = GetRequired(form, "Format"), + FileExtension = GetRequired(form, "FileExtension"), + FileNamePattern = GetOptional(form, "FileNamePattern"), + ContentType = GetOptional(form, "ContentType"), + Delimited = new DelimitedFileProfileOptions { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect($"{returnUrl}?saved=1"); - } - catch (Exception ex) + Delimiter = GetOptional(form, "Delimiter") ?? ",", + IncludeHeader = ParseBool(form["IncludeHeader"]), + HeaderFields = existingProfile.Delimited.HeaderFields, + TrailerFields = existingProfile.Delimited.TrailerFields + }, + Json = new JsonFileProfileOptions { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); - } - }); + WriteIndented = ParseBool(form["WriteIndented"]), + RootPropertyName = GetOptional(form, "RootPropertyName") + }, + Fields = existingProfile.Fields + }; - endpoints.MapPost("/api/configuration/contracts", async ( - HttpRequest request, - IMerchantProcessingConfigurationStore configurationStore, - CancellationToken cancellationToken) => + if (profileIndex >= 0) { - var form = await request.ReadFormAsync(cancellationToken); - var returnUrl = GetReturnUrl(form, "/ops/config/contracts"); - var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); - var options = Clone(snapshot.Options); - var originalContractId = GetOptional(form, "OriginalContractId"); - var contractId = GetRequired(form, "ContractId"); - var contractIndex = options.ContractDefinitions.FindIndex(entry => - string.Equals(entry.ContractId, originalContractId ?? contractId, StringComparison.OrdinalIgnoreCase)); - - var contract = new ContractDefinitionOptions - { - ContractId = contractId, - FileProfileId = GetRequired(form, "FileProfileId") - }; + options.FileProfiles[profileIndex] = profile; + } + else + { + options.FileProfiles.Add(profile); + } - if (contractIndex >= 0) - { - options.ContractDefinitions[contractIndex] = contract; - } - else - { - options.ContractDefinitions.Add(contract); - } + return await SaveWithRedirectAsync(configurationStore, options, returnUrl, cancellationToken); + } - try - { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect($"{returnUrl}?saved=1"); - } - catch (Exception ex) - { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); - } - }); + private static async Task HandleFileProfileFieldConfigurationAsync( + string fileProfileId, + string section, + HttpRequest request, + IMerchantProcessingConfigurationStore configurationStore, + JsonSerializerOptions jsonSerializerOptions, + CancellationToken cancellationToken) + { + var form = await request.ReadFormAsync(cancellationToken); + var returnUrl = GetReturnUrl(form, $"/ops/config/file-profiles/{Uri.EscapeDataString(fileProfileId)}"); + var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); + var options = Clone(snapshot.Options); + var profileIndex = options.FileProfiles.FindIndex(entry => entry.FileProfileId.Equals(fileProfileId, StringComparison.OrdinalIgnoreCase)); - endpoints.MapPost("/api/configuration/file-profiles", async ( - HttpRequest request, - IMerchantProcessingConfigurationStore configurationStore, - CancellationToken cancellationToken) => + if (profileIndex < 0) { - var form = await request.ReadFormAsync(cancellationToken); - var returnUrl = GetReturnUrl(form, "/ops/config/file-profiles"); - var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); - var options = Clone(snapshot.Options); - var originalFileProfileId = GetOptional(form, "OriginalFileProfileId"); - var fileProfileId = GetRequired(form, "FileProfileId"); - var profileIndex = options.FileProfiles.FindIndex(entry => - string.Equals(entry.FileProfileId, originalFileProfileId ?? fileProfileId, StringComparison.OrdinalIgnoreCase)); - - var existingProfile = profileIndex >= 0 ? options.FileProfiles[profileIndex] : new FileProfileOptions(); - var profile = new FileProfileOptions - { - FileProfileId = fileProfileId, - FileProcessorFileProfileId = GetRequired(form, "FileProcessorFileProfileId"), - Format = GetRequired(form, "Format"), - FileExtension = GetRequired(form, "FileExtension"), - FileNamePattern = GetOptional(form, "FileNamePattern"), - ContentType = GetOptional(form, "ContentType"), - Delimited = new DelimitedFileProfileOptions - { - Delimiter = GetOptional(form, "Delimiter") ?? ",", - IncludeHeader = ParseBool(form["IncludeHeader"]), - HeaderFields = existingProfile.Delimited.HeaderFields, - TrailerFields = existingProfile.Delimited.TrailerFields - }, - Json = new JsonFileProfileOptions - { - WriteIndented = ParseBool(form["WriteIndented"]), - RootPropertyName = GetOptional(form, "RootPropertyName") - }, - Fields = existingProfile.Fields - }; + return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString("File profile not found.")}"); + } - if (profileIndex >= 0) - { - options.FileProfiles[profileIndex] = profile; - } - else - { - options.FileProfiles.Add(profile); - } + var profile = options.FileProfiles[profileIndex]; + var fields = GetFieldList(profile, section); + var originalSortOrder = ParseNullableInt(form["OriginalSortOrder"]); - try - { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect($"{returnUrl}?saved=1"); - } - catch (Exception ex) + if (ParseBool(form["Delete"])) + { + if (originalSortOrder.HasValue && originalSortOrder.Value >= 0 && originalSortOrder.Value < fields.Count) { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); + fields.RemoveAt(originalSortOrder.Value); } - }); - - endpoints.MapPost("/api/configuration/file-profiles/{fileProfileId}/fields/{section}", async ( - string fileProfileId, - string section, - HttpRequest request, - IMerchantProcessingConfigurationStore configurationStore, - JsonSerializerOptions jsonSerializerOptions, - CancellationToken cancellationToken) => + } + else { - var form = await request.ReadFormAsync(cancellationToken); - var returnUrl = GetReturnUrl(form, $"/ops/config/file-profiles/{Uri.EscapeDataString(fileProfileId)}"); - var snapshot = await configurationStore.GetCurrentSnapshotAsync(cancellationToken); - var options = Clone(snapshot.Options); - var profileIndex = options.FileProfiles.FindIndex(entry => entry.FileProfileId.Equals(fileProfileId, StringComparison.OrdinalIgnoreCase)); - - if (profileIndex < 0) + var updatedField = new FileFieldOptions { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString("File profile not found.")}"); - } - - var profile = options.FileProfiles[profileIndex]; - var fields = GetFieldList(profile, section); - var originalSortOrder = ParseNullableInt(form["OriginalSortOrder"]); + Name = GetRequired(form, "Name"), + Source = GetOptional(form, "Source") ?? string.Empty, + Format = GetOptional(form, "Format"), + Value = GetOptional(form, "Value") + }; - if (ParseBool(form["Delete"])) + if (originalSortOrder.HasValue && originalSortOrder.Value >= 0 && originalSortOrder.Value < fields.Count) { - if (originalSortOrder.HasValue) - { - if (originalSortOrder.Value >= 0 && originalSortOrder.Value < fields.Count) - { - fields.RemoveAt(originalSortOrder.Value); - } - } + fields[originalSortOrder.Value] = updatedField; } else { - var field = originalSortOrder.HasValue - ? (originalSortOrder.Value >= 0 && originalSortOrder.Value < fields.Count ? fields[originalSortOrder.Value] : null) - : null; - - if (field is null) - { - field = new FileFieldOptions(); - } - - var updatedField = new FileFieldOptions - { - Name = GetRequired(form, "Name"), - Source = GetOptional(form, "Source") ?? string.Empty, - Format = GetOptional(form, "Format"), - Value = GetOptional(form, "Value") - }; - - if (originalSortOrder.HasValue && originalSortOrder.Value >= 0 && originalSortOrder.Value < fields.Count) - { - fields[originalSortOrder.Value] = updatedField; - } - else - { - fields.Add(updatedField); - } + fields.Add(updatedField); } + } - try - { - await configurationStore.SaveAsync(options, cancellationToken); - return Results.Redirect($"{returnUrl}?saved=1"); - } - catch (Exception ex) - { - return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); - } - }); + return await SaveWithRedirectAsync(configurationStore, options, returnUrl, cancellationToken); + } - return endpoints; + private static async Task SaveWithRedirectAsync( + IMerchantProcessingConfigurationStore configurationStore, + MerchantProcessingOptions options, + string returnUrl, + CancellationToken cancellationToken, + string? queryString = null) + { + try + { + await configurationStore.SaveAsync(options, cancellationToken); + var suffix = string.IsNullOrWhiteSpace(queryString) ? "saved=1" : queryString; + return Results.Redirect($"{returnUrl}?{suffix}"); + } + catch (Exception ex) + { + return Results.Redirect($"{returnUrl}?error={Uri.EscapeDataString(ex.Message)}"); + } } private static List GetFieldList(FileProfileOptions profile, string section) diff --git a/TransactionProcessing.MerchantFileProcessor/Reporting/FileStatusReportService.cs b/TransactionProcessing.MerchantFileProcessor/Reporting/FileStatusReportService.cs index 16af43a..5524034 100644 --- a/TransactionProcessing.MerchantFileProcessor/Reporting/FileStatusReportService.cs +++ b/TransactionProcessing.MerchantFileProcessor/Reporting/FileStatusReportService.cs @@ -73,70 +73,8 @@ public sealed class FileStatusReportService( public async Task GetReportAsync(CancellationToken cancellationToken) { await using var dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); - - var aggregateSource = await dbContext.FileSendRecords - .Select(record => new - { - record.MerchantId, - record.Status, - record.ProcessedUtc - }) - .ToListAsync(cancellationToken); - - var aggregates = aggregateSource - .GroupBy(record => record.MerchantId, StringComparer.OrdinalIgnoreCase) - .Select(group => new - { - MerchantId = group.Key, - SuccessfulFilesSent = group.Count(record => record.Status == FileSendStatuses.Succeeded), - FailedFiles = group.Count(record => record.Status == FileSendStatuses.Failed), - LastProcessedUtc = group.Max(record => (DateTimeOffset?)record.ProcessedUtc) - }) - .ToList(); - - var aggregateLookup = aggregates.ToDictionary(item => item.MerchantId, StringComparer.OrdinalIgnoreCase); - - var options = configurationState.Current; - var merchantSummaries = options.Merchants - .OrderBy(merchant => merchant.MerchantId, StringComparer.OrdinalIgnoreCase) - .Select(merchant => - { - aggregateLookup.TryGetValue(merchant.MerchantId, out var aggregate); - - return new MerchantFileSummary( - merchant.MerchantId, - string.IsNullOrWhiteSpace(merchant.Name) ? merchant.MerchantId : merchant.Name, - merchant.Enabled, - aggregate?.SuccessfulFilesSent ?? 0, - aggregate?.FailedFiles ?? 0, - aggregate?.LastProcessedUtc, - GetNextScheduledUtc(merchant, DateTimeOffset.UtcNow)); - }) - .ToArray(); - - var recentFiles = (await dbContext.FileSendRecords - .Select(record => new FileStatusRow( - record.Id, - record.ProcessedUtc, - record.MerchantId, - record.MerchantName ?? record.MerchantId, - record.ContractId, - record.ContractName ?? record.ContractId, - record.Status, - record.FileName ?? string.Empty, - record.FileProfileId ?? string.Empty, - record.Format ?? string.Empty, - record.FileContent, - record.RecordCount ?? 0, - record.TotalAmount ?? 0m, - record.ErrorMessage, - record.ProcessingCompleted, - record.LastStatusCheckUtc)) - .ToListAsync(cancellationToken)) - .OrderByDescending(record => record.ProcessedUtc) - .Take(100) - .ToList(); - + var merchantSummaries = await BuildMerchantSummariesAsync(dbContext, configurationState, cancellationToken); + var recentFiles = await LoadRecentFilesAsync(dbContext, cancellationToken); return new FileStatusReport(DateTimeOffset.UtcNow, merchantSummaries, recentFiles); } @@ -380,6 +318,75 @@ public async Task RenderHtmlAsync(CancellationToken cancellationToken) TimeSpan.Zero); } + private static async Task> BuildMerchantSummariesAsync( + MerchantFileProcessorDbContext dbContext, + IMerchantProcessingConfigurationState currentState, + CancellationToken cancellationToken) + { + var aggregateLookup = (await dbContext.FileSendRecords + .Select(record => new + { + record.MerchantId, + record.Status, + record.ProcessedUtc + }) + .ToListAsync(cancellationToken)) + .GroupBy(record => record.MerchantId, StringComparer.OrdinalIgnoreCase) + .Select(group => new + { + MerchantId = group.Key, + SuccessfulFilesSent = group.Count(record => record.Status == FileSendStatuses.Succeeded), + FailedFiles = group.Count(record => record.Status == FileSendStatuses.Failed), + LastProcessedUtc = group.Max(record => (DateTimeOffset?)record.ProcessedUtc) + }) + .ToDictionary(item => item.MerchantId, StringComparer.OrdinalIgnoreCase); + + return currentState.Current.Merchants + .OrderBy(merchant => merchant.MerchantId, StringComparer.OrdinalIgnoreCase) + .Select(merchant => + { + aggregateLookup.TryGetValue(merchant.MerchantId, out var aggregate); + + return new MerchantFileSummary( + merchant.MerchantId, + string.IsNullOrWhiteSpace(merchant.Name) ? merchant.MerchantId : merchant.Name, + merchant.Enabled, + aggregate?.SuccessfulFilesSent ?? 0, + aggregate?.FailedFiles ?? 0, + aggregate?.LastProcessedUtc, + GetNextScheduledUtc(merchant, DateTimeOffset.UtcNow)); + }) + .ToArray(); + } + + private static async Task> LoadRecentFilesAsync( + MerchantFileProcessorDbContext dbContext, + CancellationToken cancellationToken) + { + return (await dbContext.FileSendRecords + .Select(record => new FileStatusRow( + record.Id, + record.ProcessedUtc, + record.MerchantId, + record.MerchantName ?? record.MerchantId, + record.ContractId, + record.ContractName ?? record.ContractId, + record.Status, + record.FileName ?? string.Empty, + record.FileProfileId ?? string.Empty, + record.Format ?? string.Empty, + record.FileContent, + record.RecordCount ?? 0, + record.TotalAmount ?? 0m, + record.ErrorMessage, + record.ProcessingCompleted, + record.LastStatusCheckUtc)) + .ToListAsync(cancellationToken)) + .OrderByDescending(record => record.ProcessedUtc) + .Take(100) + .ToList(); + } + private static void AppendDocumentStart(StringBuilder html, string title) { html.AppendLine(""); diff --git a/TransactionProcessing.MerchantFileProcessor/Reporting/OperationsDashboardService.cs b/TransactionProcessing.MerchantFileProcessor/Reporting/OperationsDashboardService.cs index 2fb9748..54f877e 100644 --- a/TransactionProcessing.MerchantFileProcessor/Reporting/OperationsDashboardService.cs +++ b/TransactionProcessing.MerchantFileProcessor/Reporting/OperationsDashboardService.cs @@ -290,19 +290,49 @@ private async Task BuildDashboardModelAsync(Cancellati { await using var dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); var options = configurationState.Current; + var fileSendRows = await LoadFileSendRowsAsync(dbContext, cancellationToken); + var merchantRunRows = await LoadMerchantRunRowsAsync(dbContext, cancellationToken); + var pendingStatusChecks = await CountPendingStatusChecksAsync(dbContext, cancellationToken); - var fileSendRows = await dbContext.FileSendRecords + return new OperationsDashboardModel( + DateTimeOffset.UtcNow, + hostEnvironment.EnvironmentName, + ResolveConnectionStringSummary(), + DescribeAuthentication(), + DescribeFileProcessing(), + DescribeTransactionGeneration(), + DescribePolling(), + DescribeLogging(), + BuildMetrics(options, pendingStatusChecks), + BuildMerchantRows(options, fileSendRows, merchantRunRows), + BuildFileProfiles(options), + BuildContracts(options), + BuildRecentRuns(merchantRunRows)); + } + + private static async Task> LoadFileSendRowsAsync( + MerchantFileProcessorDbContext dbContext, + CancellationToken cancellationToken) + { + return await dbContext.FileSendRecords .Select(record => new { record.MerchantId, MerchantName = record.MerchantName ?? record.MerchantId, record.Status, record.ProcessedUtc, - record.ProcessingCompleted + record.ProcessingCompleted, + record.EstateId, + record.FileProcessorFileId }) .ToListAsync(cancellationToken); + } - var merchantRunRows = await dbContext.MerchantRunRecords + private static async Task> LoadMerchantRunRowsAsync( + MerchantFileProcessorDbContext dbContext, + CancellationToken cancellationToken) + { + return await dbContext.MerchantRunRecords .Select(record => new { record.MerchantId, @@ -313,28 +343,38 @@ private async Task BuildDashboardModelAsync(Cancellati record.ErrorMessage }) .ToListAsync(cancellationToken); + } - var pendingStatusChecks = await dbContext.FileSendRecords.CountAsync(record => + private static async Task CountPendingStatusChecksAsync( + MerchantFileProcessorDbContext dbContext, + CancellationToken cancellationToken) + { + return await dbContext.FileSendRecords.CountAsync(record => record.Status == FileSendStatuses.Succeeded && !record.ProcessingCompleted && !string.IsNullOrWhiteSpace(record.EstateId) && !string.IsNullOrWhiteSpace(record.FileProcessorFileId), cancellationToken); + } - var recentRuns = merchantRunRows - .OrderByDescending(record => record.CompletedUtc) - .Take(50) - .Select(record => new RunHistoryRow( - record.MerchantId, - record.MerchantName, - record.ScheduledRunUtc, - record.CompletedUtc, - record.Status, - record.ErrorMessage)) - .ToArray(); + private static IReadOnlyList BuildMetrics(MerchantProcessingOptions options, int pendingStatusChecks) => + new[] + { + new OperationsMetric("Merchants", options.Merchants.Count.ToString(), $"{options.Merchants.Count(merchant => merchant.Enabled)} enabled"), + new OperationsMetric("Contracts", options.ContractDefinitions.Count.ToString(), "Contract to file-profile mappings"), + new OperationsMetric("File profiles", options.FileProfiles.Count.ToString(), "Delimited and JSON builders"), + new OperationsMetric("Pending checks", pendingStatusChecks.ToString(), "Files awaiting completion polling"), + new OperationsMetric("Run interval", $"{options.FileStatusPolling.PollIntervalSeconds}s", "Status polling cadence"), + new OperationsMetric("Transaction range", $"{options.TransactionGeneration.MinimumTransactionsPerContract}-{options.TransactionGeneration.MaximumTransactionsPerContract}", "Synthetic transaction batch size") + }; + private static IReadOnlyList BuildMerchantRows( + MerchantProcessingOptions options, + IReadOnlyList fileSendRows, + IReadOnlyList merchantRunRows) + { var merchantLookup = fileSendRows - .GroupBy(record => record.MerchantId, StringComparer.OrdinalIgnoreCase) + .GroupBy(record => (string)record.MerchantId, StringComparer.OrdinalIgnoreCase) .ToDictionary( group => group.Key, group => new @@ -346,34 +386,24 @@ private async Task BuildDashboardModelAsync(Cancellati StringComparer.OrdinalIgnoreCase); var runLookup = merchantRunRows - .GroupBy(record => record.MerchantId, StringComparer.OrdinalIgnoreCase) + .GroupBy(record => (string)record.MerchantId, StringComparer.OrdinalIgnoreCase) .ToDictionary( group => group.Key, group => group.OrderByDescending(record => record.CompletedUtc).First(), StringComparer.OrdinalIgnoreCase); - var pendingLookup = await dbContext.FileSendRecords - .Where(record => - record.Status == FileSendStatuses.Succeeded && - !record.ProcessingCompleted && - !string.IsNullOrWhiteSpace(record.MerchantId)) - .GroupBy(record => record.MerchantId) - .Select(group => new - { - MerchantId = group.Key, - PendingCount = group.Count() - }) - .ToListAsync(cancellationToken); - - var pendingLookupMap = pendingLookup.ToDictionary(item => item.MerchantId, item => item.PendingCount, StringComparer.OrdinalIgnoreCase); + var pendingLookup = fileSendRows + .Where(record => (bool)record.ProcessingCompleted == false && !string.IsNullOrWhiteSpace((string)record.MerchantId)) + .GroupBy(record => (string)record.MerchantId) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.OrdinalIgnoreCase); - var merchants = options.Merchants + return options.Merchants .OrderBy(merchant => merchant.MerchantId, StringComparer.OrdinalIgnoreCase) .Select(merchant => { merchantLookup.TryGetValue(merchant.MerchantId, out var fileSummary); runLookup.TryGetValue(merchant.MerchantId, out var runSummary); - pendingLookupMap.TryGetValue(merchant.MerchantId, out var pendingCount); + pendingLookup.TryGetValue(merchant.MerchantId, out var pendingCount); return new MerchantOperationsRow( merchant.MerchantId, @@ -388,8 +418,10 @@ private async Task BuildDashboardModelAsync(Cancellati pendingCount); }) .ToArray(); + } - var fileProfiles = options.FileProfiles + private static IReadOnlyList BuildFileProfiles(MerchantProcessingOptions options) => + options.FileProfiles .OrderBy(profile => profile.FileProfileId, StringComparer.OrdinalIgnoreCase) .Select(profile => new FileProfileRow( profile.FileProfileId, @@ -404,36 +436,24 @@ private async Task BuildDashboardModelAsync(Cancellati DescribeFieldMap(profile.Fields))) .ToArray(); - var contracts = options.ContractDefinitions + private static IReadOnlyList BuildContracts(MerchantProcessingOptions options) => + options.ContractDefinitions .OrderBy(contract => contract.ContractId, StringComparer.OrdinalIgnoreCase) .Select(contract => new ContractMappingRow(contract.ContractId, contract.FileProfileId)) .ToArray(); - var metrics = new[] - { - new OperationsMetric("Merchants", options.Merchants.Count.ToString(), $"{options.Merchants.Count(merchant => merchant.Enabled)} enabled"), - new OperationsMetric("Contracts", options.ContractDefinitions.Count.ToString(), "Contract to file-profile mappings"), - new OperationsMetric("File profiles", options.FileProfiles.Count.ToString(), "Delimited and JSON builders"), - new OperationsMetric("Pending checks", pendingStatusChecks.ToString(), "Files awaiting completion polling"), - new OperationsMetric("Run interval", $"{options.FileStatusPolling.PollIntervalSeconds}s", "Status polling cadence"), - new OperationsMetric("Transaction range", $"{options.TransactionGeneration.MinimumTransactionsPerContract}-{options.TransactionGeneration.MaximumTransactionsPerContract}", "Synthetic transaction batch size") - }; - - return new OperationsDashboardModel( - DateTimeOffset.UtcNow, - hostEnvironment.EnvironmentName, - ResolveConnectionStringSummary(), - DescribeAuthentication(), - DescribeFileProcessing(), - DescribeTransactionGeneration(), - DescribePolling(), - DescribeLogging(), - metrics, - merchants, - fileProfiles, - contracts, - recentRuns); - } + private static IReadOnlyList BuildRecentRuns(IReadOnlyList merchantRunRows) => + merchantRunRows + .OrderByDescending(record => record.CompletedUtc) + .Take(50) + .Select(record => new RunHistoryRow( + record.MerchantId, + record.MerchantName, + record.ScheduledRunUtc, + record.CompletedUtc, + record.Status, + record.ErrorMessage)) + .ToArray(); private string ResolveConnectionStringSummary() { diff --git a/TransactionProcessing.MerchantFileProcessor/Services/MerchantProcessingService.cs b/TransactionProcessing.MerchantFileProcessor/Services/MerchantProcessingService.cs index e965743..aab19dc 100644 --- a/TransactionProcessing.MerchantFileProcessor/Services/MerchantProcessingService.cs +++ b/TransactionProcessing.MerchantFileProcessor/Services/MerchantProcessingService.cs @@ -21,177 +21,209 @@ public async Task ProcessAsync(MerchantOptions merchant, DateTimeOffset schedule { var processingTimestampUtc = DateTimeOffset.UtcNow; var logger = LocalLogger.For(merchant.MerchantId, runId); - var runResultRecorded = false; logger.LogInformation($"Processing merchant for scheduled run {scheduledRunUtc:O}"); try { - logger.LogInformation("Requesting access token"); - var accessTokenResult = await accessTokenProvider.GetAccessToken(cancellationToken); - if (accessTokenResult.IsFailed) + await ProcessMerchantAsync(merchant, scheduledRunUtc, runId, processingTimestampUtc, cancellationToken); + logger.LogInformation($"Completed merchant for scheduled run {scheduledRunUtc:O}"); + } + catch (Exception ex) + { + if (ex is not OperationCanceledException) { - throw new InvalidOperationException(string.Join("; ", accessTokenResult.Errors)); + await RecordMerchantRunResultAsync(runId, merchant, scheduledRunUtc, MerchantRunStatuses.Failed, ex.ToString(), cancellationToken); } - var accessToken = accessTokenResult.Data.AccessToken; - logger.LogInformation("Access token retrieved successfully"); + logger.LogError("Merchant processing failed", ex); - logger.LogInformation("Requesting contract data"); - var contractsResult = await merchantContractDataClient.GetContracts(merchant, accessToken, cancellationToken); - if (contractsResult.IsFailed || contractsResult.Data is null) - { - throw new InvalidOperationException(string.Join("; ", contractsResult.Errors)); - } + throw; + } + } - var contracts = contractsResult.Data; - logger.LogInformation($"Retrieved {contracts.Count} contracts"); + private async Task ProcessMerchantAsync( + MerchantOptions merchant, + DateTimeOffset scheduledRunUtc, + Guid runId, + DateTimeOffset processingTimestampUtc, + CancellationToken cancellationToken) + { + var accessToken = await GetAccessTokenAsync(cancellationToken); + var contracts = await GetConfiguredContractsAsync(merchant, accessToken, cancellationToken); + var contractFailures = await ProcessContractsAsync( + merchant, + scheduledRunUtc, + runId, + processingTimestampUtc, + accessToken, + contracts, + cancellationToken); + + if (contractFailures.Count > 0) + { + await RecordMerchantRunResultAsync( + runId, + merchant, + scheduledRunUtc, + MerchantRunStatuses.Failed, + string.Join(Environment.NewLine + Environment.NewLine, contractFailures.Select(ex => ex.ToString())), + cancellationToken); - var configuredContracts = transactionFileGenerationService.GetConfiguredContracts(contracts); - var skippedUnconfiguredContracts = contracts.Count - configuredContracts.Count; + throw new AggregateException($"Scheduled run {scheduledRunUtc:O} failed for {contractFailures.Count} contract(s).", contractFailures); + } - if (skippedUnconfiguredContracts > 0) - { - logger.LogInformation($"Skipping {skippedUnconfiguredContracts} contracts because they do not have a configured file profile."); - } + await RecordMerchantRunResultAsync( + runId, + merchant, + scheduledRunUtc, + MerchantRunStatuses.Succeeded, + null, + cancellationToken); + } + + private async Task GetAccessTokenAsync(CancellationToken cancellationToken) + { + var logger = LocalLogger.For(); + logger.LogInformation("Requesting access token"); + + var accessTokenResult = await accessTokenProvider.GetAccessToken(cancellationToken); + if (accessTokenResult.IsFailed) + { + throw new InvalidOperationException(string.Join("; ", accessTokenResult.Errors)); + } + + logger.LogInformation("Access token retrieved successfully"); + return accessTokenResult.Data.AccessToken; + } + + private async Task> GetConfiguredContractsAsync( + MerchantOptions merchant, + string accessToken, + CancellationToken cancellationToken) + { + var logger = LocalLogger.For(merchant.MerchantId); + logger.LogInformation("Requesting contract data"); + + var contractsResult = await merchantContractDataClient.GetContracts(merchant, accessToken, cancellationToken); + if (contractsResult.IsFailed || contractsResult.Data is null) + { + throw new InvalidOperationException(string.Join("; ", contractsResult.Errors)); + } + + logger.LogInformation($"Retrieved {contractsResult.Data.Count} contracts"); + + var configuredContracts = transactionFileGenerationService.GetConfiguredContracts(contractsResult.Data); + var skippedUnconfiguredContracts = contractsResult.Data.Count - configuredContracts.Count; + if (skippedUnconfiguredContracts > 0) + { + logger.LogInformation($"Skipping {skippedUnconfiguredContracts} contracts because they do not have a configured file profile."); + } + + return configuredContracts; + } - var contractFailures = new List(); + private async Task> ProcessContractsAsync( + MerchantOptions merchant, + DateTimeOffset scheduledRunUtc, + Guid runId, + DateTimeOffset processingTimestampUtc, + string accessToken, + IReadOnlyList contracts, + CancellationToken cancellationToken) + { + var contractFailures = new List(); - foreach (var contract in configuredContracts) + foreach (var contract in contracts) + { + if (!contract.Products.Any(product => product.IsFixedValue)) { - if (!contract.Products.Any(product => product.IsFixedValue)) - { - logger.LogWarning($"Skipping contract {contract.ContractId} because it does not contain any fixed-value products."); - continue; - } - - var hasSuccessfulUpload = await fileStatusStore.HasSuccessfulUploadAsync( - merchant, - contract, - scheduledRunUtc, - cancellationToken); - - if (hasSuccessfulUpload) - { - logger.LogInformation($"Skipping contract {contract.ContractId} because it was already uploaded for scheduled run {scheduledRunUtc:O}."); - continue; - } - - GeneratedFile? generatedFile = null; - var fileUploaded = false; - - try - { - generatedFile = transactionFileGenerationService.BuildFile(merchant, contract, processingTimestampUtc); - - logger.LogInformation($"Generated {generatedFile.Format} file {generatedFile.FileName} using profile {generatedFile.FileProfileId} for contract {contract.ContractId} with {generatedFile.RecordCount} records totaling {generatedFile.TotalAmount}"); - - var depositAmount = CalculateDepositAmount(generatedFile); - var depositReference = BuildDepositReference(runId, contract, generatedFile); - var depositResult = await merchantDepositClient.MakeDeposit( - merchant, - accessToken, - depositAmount, - depositReference, - processingTimestampUtc, - cancellationToken); - - if (depositResult.IsFailed) - { - throw new InvalidOperationException(string.Join("; ", depositResult.Errors)); - } - - logger.LogInformation($"Made merchant deposit of {depositAmount:0.00} for contract {contract.ContractId}"); - - var uploadResult = await fileProcessingClient.Upload( - merchant, - contract, - accessToken, - generatedFile, - cancellationToken); - if (uploadResult.IsFailed || uploadResult.Data == Guid.Empty) - { - throw new InvalidOperationException(string.Join("; ", uploadResult.Errors)); - } - - var fileId = uploadResult.Data; - fileUploaded = true; - - await fileStatusStore.RecordSuccessAsync( - runId, - merchant, - contract, - scheduledRunUtc, - fileId, - generatedFile, - cancellationToken); - - logger.LogInformation($"Uploaded {generatedFile.Format} file {generatedFile.FileName} for contract {contract.ContractId} with file id {fileId}"); - } - catch (Exception ex) - { - if (!fileUploaded) - { - await fileStatusStore.RecordFailureAsync( - runId, - merchant, - contract, - scheduledRunUtc, - generatedFile, - ex.ToString(), - cancellationToken); - } - - logger.LogError($"Contract processing failed for contract {contract.ContractId}", ex); - contractFailures.Add(ex); - } + LocalLogger.For(merchant.MerchantId).LogWarning($"Skipping contract {contract.ContractId} because it does not contain any fixed-value products."); + continue; } - if (contractFailures.Count > 0) + if (await fileStatusStore.HasSuccessfulUploadAsync(merchant, contract, scheduledRunUtc, cancellationToken)) { - var errorMessage = string.Join(Environment.NewLine + Environment.NewLine, contractFailures.Select(ex => ex.ToString())); - await fileStatusStore.RecordMerchantRunResultAsync( - runId, - merchant, - scheduledRunUtc, - MerchantRunStatuses.Failed, - errorMessage, - cancellationToken); - runResultRecorded = true; - - throw new AggregateException($"Scheduled run {scheduledRunUtc:O} failed for {contractFailures.Count} contract(s).", contractFailures); + LocalLogger.For(merchant.MerchantId).LogInformation($"Skipping contract {contract.ContractId} because it was already uploaded for scheduled run {scheduledRunUtc:O}."); + continue; } - await fileStatusStore.RecordMerchantRunResultAsync( - runId, + await ProcessContractAsync( merchant, + contract, + runId, scheduledRunUtc, - MerchantRunStatuses.Succeeded, - null, + processingTimestampUtc, + accessToken, + contractFailures, cancellationToken); - runResultRecorded = true; + } - logger.LogInformation($"Completed merchant for scheduled run {scheduledRunUtc:O}"); + return contractFailures; + } + + private async Task ProcessContractAsync( + MerchantOptions merchant, + ContractOptions contract, + Guid runId, + DateTimeOffset scheduledRunUtc, + DateTimeOffset processingTimestampUtc, + string accessToken, + List contractFailures, + CancellationToken cancellationToken) + { + GeneratedFile? generatedFile = null; + var fileUploaded = false; + + try + { + var logger = LocalLogger.For(merchant.MerchantId); + generatedFile = transactionFileGenerationService.BuildFile(merchant, contract, processingTimestampUtc); + logger.LogInformation($"Generated {generatedFile.Format} file {generatedFile.FileName} using profile {generatedFile.FileProfileId} for contract {contract.ContractId} with {generatedFile.RecordCount} records totaling {generatedFile.TotalAmount}"); + + var depositAmount = CalculateDepositAmount(generatedFile); + var depositReference = BuildDepositReference(runId, contract, generatedFile); + var depositResult = await merchantDepositClient.MakeDeposit(merchant, accessToken, depositAmount, depositReference, processingTimestampUtc, cancellationToken); + if (depositResult.IsFailed) + { + throw new InvalidOperationException(string.Join("; ", depositResult.Errors)); + } + + logger.LogInformation($"Made merchant deposit of {depositAmount:0.00} for contract {contract.ContractId}"); + + var uploadResult = await fileProcessingClient.Upload(merchant, contract, accessToken, generatedFile, cancellationToken); + if (uploadResult.IsFailed || uploadResult.Data == Guid.Empty) + { + throw new InvalidOperationException(string.Join("; ", uploadResult.Errors)); + } + + var fileId = uploadResult.Data; + fileUploaded = true; + + await fileStatusStore.RecordSuccessAsync(runId, merchant, contract, scheduledRunUtc, fileId, generatedFile, cancellationToken); + logger.LogInformation($"Uploaded {generatedFile.Format} file {generatedFile.FileName} for contract {contract.ContractId} with file id {fileId}"); } catch (Exception ex) { - if (!runResultRecorded && ex is not OperationCanceledException) + if (!fileUploaded) { - await fileStatusStore.RecordMerchantRunResultAsync( - runId, - merchant, - scheduledRunUtc, - MerchantRunStatuses.Failed, - ex.ToString(), - cancellationToken); + await fileStatusStore.RecordFailureAsync(runId, merchant, contract, scheduledRunUtc, generatedFile, ex.ToString(), cancellationToken); } - logger.LogError("Merchant processing failed", ex); - - throw; + LocalLogger.For(merchant.MerchantId).LogError($"Contract processing failed for contract {contract.ContractId}", ex); + contractFailures.Add(ex); } } + private Task RecordMerchantRunResultAsync( + Guid runId, + MerchantOptions merchant, + DateTimeOffset scheduledRunUtc, + string status, + string? errorMessage, + CancellationToken cancellationToken) => + fileStatusStore.RecordMerchantRunResultAsync(runId, merchant, scheduledRunUtc, status, errorMessage, cancellationToken); + private static decimal CalculateDepositAmount(GeneratedFile generatedFile) { if (generatedFile.Transactions.Count < 2) diff --git a/TransactionProcessing.MerchantFileProcessor/Worker.cs b/TransactionProcessing.MerchantFileProcessor/Worker.cs index e07e6d1..6e76998 100644 --- a/TransactionProcessing.MerchantFileProcessor/Worker.cs +++ b/TransactionProcessing.MerchantFileProcessor/Worker.cs @@ -11,71 +11,70 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { LocalLogger.Context logger = LocalLogger.For(); logger.LogInformation("Merchant processor started"); - while (!stoppingToken.IsCancellationRequested) { - MerchantProcessingOptions options = configurationState.Current; - MerchantOptions[] enabledMerchants = options.Merchants.Where(merchant => merchant.Enabled).ToArray(); - TimeSpan scanInterval = TimeSpan.FromSeconds(Math.Max(1, options.MerchantScanIntervalSeconds)); - - if (enabledMerchants.Length == 0) { - logger.LogWarning("No enabled merchants are configured for processing. Waiting before checking again."); - try { - await Task.Delay(scanInterval, stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - break; - } - continue; - } + while (await RunIterationAsync(stoppingToken)) { + } + } - DateTimeOffset now = DateTimeOffset.UtcNow; - ScheduledMerchant scheduledMerchant = await this.GetNextScheduledMerchantAsync(now, enabledMerchants, stoppingToken); - DateTimeOffset nextRun = scheduledMerchant.TriggerUtc; - TimeSpan delay = nextRun - now; + private async Task RunIterationAsync(CancellationToken stoppingToken) { + LocalLogger.Context logger = LocalLogger.For(); + MerchantProcessingOptions options = configurationState.Current; + MerchantOptions[] enabledMerchants = options.Merchants.Where(merchant => merchant.Enabled).ToArray(); + TimeSpan scanInterval = TimeSpan.FromSeconds(Math.Max(1, options.MerchantScanIntervalSeconds)); - LocalLogger.Context merchantLogger = LocalLogger.For(scheduledMerchant.Merchant.MerchantId); - merchantLogger.LogInformation($"Next merchant processing run scheduled at {nextRun:O} for slot {scheduledMerchant.ScheduledRunUtc:O}"); + if (enabledMerchants.Length == 0) { + logger.LogWarning("No enabled merchants are configured for processing. Waiting before checking again."); + return await DelayAsync(scanInterval, stoppingToken); + } - if (delay > scanInterval) { - try { - await Task.Delay(scanInterval, stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - break; - } - continue; - } + DateTimeOffset now = DateTimeOffset.UtcNow; + ScheduledMerchant scheduledMerchant = await this.GetNextScheduledMerchantAsync(now, enabledMerchants, stoppingToken); + DateTimeOffset nextRun = scheduledMerchant.TriggerUtc; + TimeSpan delay = nextRun - now; - if (delay > TimeSpan.Zero) { - try { - await Task.Delay(delay, stoppingToken); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - break; - } - } + LocalLogger.Context merchantLogger = LocalLogger.For(scheduledMerchant.Merchant.MerchantId); + merchantLogger.LogInformation($"Next merchant processing run scheduled at {nextRun:O} for slot {scheduledMerchant.ScheduledRunUtc:O}"); - if (stoppingToken.IsCancellationRequested) { - break; - } + if (delay > scanInterval) { + return await DelayAsync(scanInterval, stoppingToken); + } - Guid? runId = null; + if (delay > TimeSpan.Zero && !await DelayAsync(delay, stoppingToken)) { + return false; + } - try { - runId = Guid.NewGuid(); - Guid currentRunId = runId.Value; + if (stoppingToken.IsCancellationRequested) { + return false; + } - merchantLogger.WithRun(currentRunId).LogInformation("Starting merchant processing run"); + Guid? runId = null; - await merchantProcessingService.ProcessAsync(scheduledMerchant.Merchant, scheduledMerchant.ScheduledRunUtc, currentRunId, stoppingToken); + try { + runId = Guid.NewGuid(); + Guid currentRunId = runId.Value; - merchantLogger.WithRun(currentRunId).LogInformation("Merchant processing run completed successfully"); - } - catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - break; - } - catch (Exception ex) { - merchantLogger.WithRun(runId).LogError(runId is null ? "Merchant processing run failed before a run identifier was assigned." : "Merchant processing run failed", ex); - } + merchantLogger.WithRun(currentRunId).LogInformation("Starting merchant processing run"); + + await merchantProcessingService.ProcessAsync(scheduledMerchant.Merchant, scheduledMerchant.ScheduledRunUtc, currentRunId, stoppingToken); + + merchantLogger.WithRun(currentRunId).LogInformation("Merchant processing run completed successfully"); + return true; + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { + return false; + } + catch (Exception ex) { + merchantLogger.WithRun(runId).LogError(runId is null ? "Merchant processing run failed before a run identifier was assigned." : "Merchant processing run failed", ex); + return true; + } + } + + private static async Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) { + try { + await Task.Delay(delay, cancellationToken); + return true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + return false; } } diff --git a/TransactionProcessing.MerchantFileProcessor/merchant-processing.bootstrap.json b/TransactionProcessing.MerchantFileProcessor/merchant-processing.bootstrap.json index 5a32966..f025412 100644 --- a/TransactionProcessing.MerchantFileProcessor/merchant-processing.bootstrap.json +++ b/TransactionProcessing.MerchantFileProcessor/merchant-processing.bootstrap.json @@ -2,7 +2,7 @@ "MerchantProcessing": { "Authentication": { "ClientId": "serviceClient", - "ClientSecret": "d192cbc46d834d0da90e8a9d50ded543" + "ClientSecret": "" }, "FileProcessing": { "UserId": "a51791a8-2731-490c-a8ac-639461f2a41f" diff --git a/TransactionProcessing.MerchantPos/Pages/Config.cshtml.cs b/TransactionProcessing.MerchantPos/Pages/Config.cshtml.cs index bfcbf5d..e99aabb 100644 --- a/TransactionProcessing.MerchantPos/Pages/Config.cshtml.cs +++ b/TransactionProcessing.MerchantPos/Pages/Config.cshtml.cs @@ -7,7 +7,7 @@ namespace TransactionProcessing.MerchantPos.Pages; public sealed class ConfigModel : PageModel { - private const string SecretMask = "*****"; + private static readonly string SecretMask = new('*', 5); private readonly MerchantPosSettingsStore _settingsStore; public ConfigModel(MerchantPosSettingsStore settingsStore) diff --git a/TransactionProcessing.MerchantPos/Pages/Merchants.cshtml.cs b/TransactionProcessing.MerchantPos/Pages/Merchants.cshtml.cs index 9542627..4f52bbc 100644 --- a/TransactionProcessing.MerchantPos/Pages/Merchants.cshtml.cs +++ b/TransactionProcessing.MerchantPos/Pages/Merchants.cshtml.cs @@ -7,7 +7,7 @@ namespace TransactionProcessing.MerchantPos.Pages; public sealed class MerchantsModel : PageModel { - private const string SecretMask = "*****"; + private static readonly string SecretMask = new('*', 5); private readonly MerchantPosSettingsStore _settingsStore; public MerchantsModel(MerchantPosSettingsStore settingsStore) diff --git a/TransactionProcessing.MerchantPos/Persistence/MerchantPosSettingsStore.cs b/TransactionProcessing.MerchantPos/Persistence/MerchantPosSettingsStore.cs index a4bc789..058da6a 100644 --- a/TransactionProcessing.MerchantPos/Persistence/MerchantPosSettingsStore.cs +++ b/TransactionProcessing.MerchantPos/Persistence/MerchantPosSettingsStore.cs @@ -109,30 +109,40 @@ public static string NormalizePath(string? path) private static MerchantPosSettingsSnapshot MergeWithDefaults(MerchantPosSettingsSnapshot snapshot, MerchantPosSettingsSnapshot defaults) { - snapshot.WorkerSettings ??= new WorkerSettings(); - snapshot.WorkerSettings.ClientId = string.IsNullOrWhiteSpace(snapshot.WorkerSettings.ClientId) ? defaults.WorkerSettings.ClientId : snapshot.WorkerSettings.ClientId; - snapshot.WorkerSettings.ClientSecret = string.IsNullOrWhiteSpace(snapshot.WorkerSettings.ClientSecret) ? defaults.WorkerSettings.ClientSecret : snapshot.WorkerSettings.ClientSecret; - snapshot.WorkerSettings.ServiceClientId = string.IsNullOrWhiteSpace(snapshot.WorkerSettings.ServiceClientId) ? defaults.WorkerSettings.ServiceClientId : snapshot.WorkerSettings.ServiceClientId; - snapshot.WorkerSettings.ServiceClientSecret = string.IsNullOrWhiteSpace(snapshot.WorkerSettings.ServiceClientSecret) ? defaults.WorkerSettings.ServiceClientSecret : snapshot.WorkerSettings.ServiceClientSecret; - snapshot.WorkerSettings.MerchantScanIntervalSeconds = snapshot.WorkerSettings.MerchantScanIntervalSeconds <= 0 - ? defaults.WorkerSettings.MerchantScanIntervalSeconds - : snapshot.WorkerSettings.MerchantScanIntervalSeconds; - if (snapshot.WorkerSettings.Merchants is null || snapshot.WorkerSettings.Merchants.Count == 0) + ApplyWorkerDefaults(snapshot.WorkerSettings ??= new WorkerSettings(), defaults.WorkerSettings); + ApplyApiDefaults(snapshot.ApiConfiguration ??= new ApiConfigurationSettings(), defaults.ApiConfiguration); + ApplyConnectionStringDefaults(snapshot.ConnectionStrings ??= new ConnectionStringsSettings(), defaults.ConnectionStrings); + snapshot.EnsureDefaults(); + return snapshot; + } + + private static void ApplyWorkerDefaults(WorkerSettings current, WorkerSettings defaults) + { + current.ClientId = string.IsNullOrWhiteSpace(current.ClientId) ? defaults.ClientId : current.ClientId; + current.ClientSecret = string.IsNullOrWhiteSpace(current.ClientSecret) ? defaults.ClientSecret : current.ClientSecret; + current.ServiceClientId = string.IsNullOrWhiteSpace(current.ServiceClientId) ? defaults.ServiceClientId : current.ServiceClientId; + current.ServiceClientSecret = string.IsNullOrWhiteSpace(current.ServiceClientSecret) ? defaults.ServiceClientSecret : current.ServiceClientSecret; + current.MerchantScanIntervalSeconds = current.MerchantScanIntervalSeconds <= 0 + ? defaults.MerchantScanIntervalSeconds + : current.MerchantScanIntervalSeconds; + if (current.Merchants is null || current.Merchants.Count == 0) { - snapshot.WorkerSettings.Merchants = defaults.WorkerSettings.Merchants; + current.Merchants = defaults.Merchants; } + } - snapshot.ApiConfiguration ??= new ApiConfigurationSettings(); - snapshot.ApiConfiguration.SecurityService = string.IsNullOrWhiteSpace(snapshot.ApiConfiguration.SecurityService) ? defaults.ApiConfiguration.SecurityService : snapshot.ApiConfiguration.SecurityService; - snapshot.ApiConfiguration.TransactionProcessorACL = string.IsNullOrWhiteSpace(snapshot.ApiConfiguration.TransactionProcessorACL) ? defaults.ApiConfiguration.TransactionProcessorACL : snapshot.ApiConfiguration.TransactionProcessorACL; - snapshot.ApiConfiguration.TransactionProcessorApi = string.IsNullOrWhiteSpace(snapshot.ApiConfiguration.TransactionProcessorApi) ? defaults.ApiConfiguration.TransactionProcessorApi : snapshot.ApiConfiguration.TransactionProcessorApi; - snapshot.ApiConfiguration.TestHost = string.IsNullOrWhiteSpace(snapshot.ApiConfiguration.TestHost) ? defaults.ApiConfiguration.TestHost : snapshot.ApiConfiguration.TestHost; + private static void ApplyApiDefaults(ApiConfigurationSettings current, ApiConfigurationSettings defaults) + { + current.SecurityService = string.IsNullOrWhiteSpace(current.SecurityService) ? defaults.SecurityService : current.SecurityService; + current.TransactionProcessorACL = string.IsNullOrWhiteSpace(current.TransactionProcessorACL) ? defaults.TransactionProcessorACL : current.TransactionProcessorACL; + current.TransactionProcessorApi = string.IsNullOrWhiteSpace(current.TransactionProcessorApi) ? defaults.TransactionProcessorApi : current.TransactionProcessorApi; + current.TestHost = string.IsNullOrWhiteSpace(current.TestHost) ? defaults.TestHost : current.TestHost; + } - snapshot.ConnectionStrings ??= new ConnectionStringsSettings(); - snapshot.ConnectionStrings.MerchantDb = string.IsNullOrWhiteSpace(snapshot.ConnectionStrings.MerchantDb) ? defaults.ConnectionStrings.MerchantDb : snapshot.ConnectionStrings.MerchantDb; - snapshot.ConnectionStrings.SettingsDb = string.IsNullOrWhiteSpace(snapshot.ConnectionStrings.SettingsDb) ? defaults.ConnectionStrings.SettingsDb : snapshot.ConnectionStrings.SettingsDb; - snapshot.EnsureDefaults(); - return snapshot; + private static void ApplyConnectionStringDefaults(ConnectionStringsSettings current, ConnectionStringsSettings defaults) + { + current.MerchantDb = string.IsNullOrWhiteSpace(current.MerchantDb) ? defaults.MerchantDb : current.MerchantDb; + current.SettingsDb = string.IsNullOrWhiteSpace(current.SettingsDb) ? defaults.SettingsDb : current.SettingsDb; } private MerchantPosSettingsDbContext CreateContext(string settingsDbPath) diff --git a/TransactionProcessing.MerchantPos/Runtime/ApiClient.cs b/TransactionProcessing.MerchantPos/Runtime/ApiClient.cs index 7015635..716a697 100644 --- a/TransactionProcessing.MerchantPos/Runtime/ApiClient.cs +++ b/TransactionProcessing.MerchantPos/Runtime/ApiClient.cs @@ -8,6 +8,7 @@ using SecurityService.DataTransferObjects; using Shared.Results; using Shared.Serialisation; +using System.Security.Cryptography; using TransactionProcessor.Client; using TransactionProcessor.DataTransferObjects; using TransactionProcessor.DataTransferObjects.Requests.Merchant; @@ -396,10 +397,9 @@ private String BuildRequestUrl(String route) { return requestUri; } - private readonly Random _rng = new(); private async Task> CreateBillPaymentBill(Decimal billAmount, CancellationToken cancellationToken) { - Int32 accountNumber = this._rng.Next(1, 100000); + Int32 accountNumber = RandomNumberGenerator.GetInt32(1, 100000); String baseAddress = this.SettingsStore.Current.ApiConfiguration.TestHost; var body = new @@ -423,7 +423,7 @@ private String BuildRequestUrl(String route) { private async Task> CreateBillPaymentMeter(CancellationToken cancellationToken) { - Int32 meterNumber = this._rng.Next(1, 100000); + Int32 meterNumber = RandomNumberGenerator.GetInt32(1, 100000); String baseAddress = this.SettingsStore.Current.ApiConfiguration.TestHost; var body = new { diff --git a/TransactionProcessing.MerchantPos/Runtime/MerchantPosSettingsSnapshot.cs b/TransactionProcessing.MerchantPos/Runtime/MerchantPosSettingsSnapshot.cs index 117a6db..e589610 100644 --- a/TransactionProcessing.MerchantPos/Runtime/MerchantPosSettingsSnapshot.cs +++ b/TransactionProcessing.MerchantPos/Runtime/MerchantPosSettingsSnapshot.cs @@ -17,6 +17,13 @@ public static MerchantPosSettingsSnapshot FromConfiguration(IConfiguration confi } public void EnsureDefaults() + { + EnsureWorkerDefaults(); + EnsureApiConfigurationDefaults(); + EnsureConnectionStringDefaults(); + } + + private void EnsureWorkerDefaults() { WorkerSettings ??= new WorkerSettings(); WorkerSettings.ClientId ??= string.Empty; @@ -25,13 +32,19 @@ public void EnsureDefaults() WorkerSettings.ServiceClientSecret ??= string.Empty; WorkerSettings.MerchantScanIntervalSeconds = WorkerSettings.MerchantScanIntervalSeconds <= 0 ? 5 : WorkerSettings.MerchantScanIntervalSeconds; WorkerSettings.Merchants ??= new List(); + } + private void EnsureApiConfigurationDefaults() + { ApiConfiguration ??= new ApiConfigurationSettings(); ApiConfiguration.SecurityService ??= string.Empty; ApiConfiguration.TransactionProcessorACL ??= string.Empty; ApiConfiguration.TransactionProcessorApi ??= string.Empty; ApiConfiguration.TestHost ??= string.Empty; + } + private void EnsureConnectionStringDefaults() + { ConnectionStrings ??= new ConnectionStringsSettings(); ConnectionStrings.MerchantDb ??= string.Empty; ConnectionStrings.SettingsDb ??= string.Empty; diff --git a/TransactionProcessing.MerchantPos/Runtime/MerchantRuntime.cs b/TransactionProcessing.MerchantPos/Runtime/MerchantRuntime.cs index 854713e..afa6fbc 100644 --- a/TransactionProcessing.MerchantPos/Runtime/MerchantRuntime.cs +++ b/TransactionProcessing.MerchantPos/Runtime/MerchantRuntime.cs @@ -6,6 +6,7 @@ using MerchantPos.EF.Models; using SecurityService.DataTransferObjects; using TransactionProcessing.MerchantPos.Runtime; +using System.Security.Cryptography; public class MerchantRuntime { @@ -13,8 +14,6 @@ public class MerchantRuntime private readonly ISecurityServiceClient SecurityServiceClient; private readonly IEfRepository Repository; private readonly MerchantMetrics Metrics; - private readonly Random _rng = new(); - public MerchantRuntime(IApiClient apiClient, ISecurityServiceClient securityServiceClient, IEfRepository repository, @@ -243,16 +242,16 @@ private async Task DoSaleCycle(MerchantConfig cfg,Int32 transactionNumber, Cance decimal balance = await Repository.GetBalance(cfg.MerchantId); // Random product - Product product = cfg.Products[_rng.Next(cfg.Products.Count)]; + Product product = cfg.Products[RandomNumberGenerator.GetInt32(cfg.Products.Count)]; Decimal value = product.Value switch { - 0 => this._rng.Next(9, 250), + 0 => RandomNumberGenerator.GetInt32(9, 250), _ => product.Value }; // Possible intentional fail - bool induceFail = _rng.NextDouble() < cfg.FailureInjectionProbability; + bool induceFail = NextDouble() < cfg.FailureInjectionProbability; decimal saleValue = induceFail ? balance + 10 : value; Result result = await ApiClient.SendSale(cfg, this.CurrentUserToken, product, saleValue, transactionNumber, cancellationToken); @@ -286,6 +285,12 @@ private async Task DoSaleCycle(MerchantConfig cfg,Int32 transactionNumber, Cance } } + private static double NextDouble() + { + var sample = RandomNumberGenerator.GetInt32(int.MaxValue); + return sample / (double)int.MaxValue; + } + private async Task DoReconciliation(MerchantConfig cfg, CancellationToken cancellationToken) { Result tokenResult = await this.GetUserToken(cfg, cancellationToken); diff --git a/TransactionProcessing.MerchantPos/appsettings.staging.json b/TransactionProcessing.MerchantPos/appsettings.staging.json index 72b03e5..42984a0 100644 --- a/TransactionProcessing.MerchantPos/appsettings.staging.json +++ b/TransactionProcessing.MerchantPos/appsettings.staging.json @@ -1,9 +1,9 @@ { "WorkerSettings": { "ClientId": "mobileAppClient", - "ClientSecret": "d192cbc46d834d0da90e8a9d50ded543", + "ClientSecret": "", "ServiceClientId": "serviceClient", - "ServiceClientSecret": "d192cbc46d834d0da90e8a9d50ded543", + "ServiceClientSecret": "", "MerchantScanIntervalSeconds": 5, "Merchants": [ { @@ -14,7 +14,7 @@ "ApplicationVersion": "1.0.5", "DeviceIdentifier": "stagingmerchant1device", "Username": "merchantuser@stagingmerchant1.co.uk", - "Password": "123456", + "Password": "", "SaleIntervalSeconds": 30, "FailureInjectionProbability": 0.05, "DepositThreshold": 100, diff --git a/TransactionProcessing.SchedulerService/DataGenerator/Program.cs b/TransactionProcessing.SchedulerService/DataGenerator/Program.cs index a24230d..7042475 100644 --- a/TransactionProcessing.SchedulerService/DataGenerator/Program.cs +++ b/TransactionProcessing.SchedulerService/DataGenerator/Program.cs @@ -72,7 +72,7 @@ static async Task Main(string[] args){ // Get a token to talk to the estate service CancellationToken cancellationToken = new(); String clientId = "serviceClient"; - String clientSecret = "d192cbc46d834d0da90e8a9d50ded543"; + String clientSecret = ResolveRequiredSecret("TRANSACTIONPROCESSOR_SERVICECLIENT_SECRET"); ITransactionDataGeneratorService g = new TransactionDataGeneratorService(Program.SecurityServiceClient, Program.TransactionProcessorClient, Program.baseAddressFunc("TransactionProcessorApi"), @@ -92,6 +92,17 @@ static async Task Main(string[] args){ Console.WriteLine("Process Complete"); } + private static string ResolveRequiredSecret(string environmentVariableName) + { + string? secret = Environment.GetEnvironmentVariable(environmentVariableName); + if (!string.IsNullOrWhiteSpace(secret)) + { + return secret; + } + + throw new InvalidOperationException($"Missing required secret. Set {environmentVariableName}."); + } + private static async Task GenerateStatements(ITransactionDataGeneratorService g, Guid estateId, CancellationToken cancellationToken){ Result> getMerchantsResult = await g.GetMerchants(estateId, cancellationToken); if (getMerchantsResult.IsFailed) @@ -281,4 +292,4 @@ public void LogWarning(String message) { public Boolean IsInitialised { get; set; } } -} \ No newline at end of file +} diff --git a/TransactionProcessing.SchedulerService/JobTestDriver/Program.cs b/TransactionProcessing.SchedulerService/JobTestDriver/Program.cs index cd93fde..77424bb 100644 --- a/TransactionProcessing.SchedulerService/JobTestDriver/Program.cs +++ b/TransactionProcessing.SchedulerService/JobTestDriver/Program.cs @@ -1,74 +1,12 @@ -namespace JobTestDriver -{ - //using TransactionProcessing.SchedulerService.DataGenerator; - - internal class Program{ - static async Task Main(string[] args){ - - //List<(String groupName, String streamName, Int64 parkedMessageCount)>? info = await Jobs.GetParkedQueueInformation("esdb://admin:changeit@192.168.0.133:2113?tls=false&tlsVerifyCert=false", CancellationToken.None); - //foreach ((String groupName, String streamName, Int64 parkedMessageCount) infoItem in info){ - // Console.WriteLine($"Group: {infoItem.groupName} Stream: {infoItem.streamName} Parked Count: {infoItem.parkedMessageCount}"); - //} - - //List<(Guid, String, DateTime, String)>? incompleteFiles = await Jobs.GetIncompleteFileList("server=192.168.0.133;user id=sa;password=Sc0tland;database=EstateReportingReadModel;Encrypt=True;TrustServerCertificate=True", CancellationToken.None); - //foreach ((Guid, String, DateTime, String) incompleteFile in incompleteFiles){ - // Console.WriteLine($"FileId: {incompleteFile.Item1} Location: {incompleteFile.Item2} Rcvd: {incompleteFile.Item3} Merchant: {incompleteFile.Item4}"); - //} - //HttpClient client = new HttpClient(); - //IMessagingServiceClient messagingServiceClient = new MessagingServiceClient(delegate(String s){ return "http://127.0.0.1:5006";}, client); - //String accessToken = "eyJhbGciOiJSUzI1NiIsImtpZCI6IjQzOUMxRDk5MDUwQTYyMDhEM0U5M0JFMjlBQUJBNzI5IiwidHlwIjoiYXQrand0In0.eyJpc3MiOiJodHRwczovLzEyNy4wLjAuMTo1MDAxIiwibmJmIjoxNjgzMTA2NDE2LCJpYXQiOjE2ODMxMDY0MTYsImV4cCI6MTY4MzExMDAxNiwiYXVkIjpbImVzdGF0ZU1hbmFnZW1lbnQiLCJlc3RhdGVSZXBvcnRpbmciLCJmaWxlUHJvY2Vzc29yIiwibWVzc2FnaW5nU2VydmljZSIsInRyYW5zYWN0aW9uUHJvY2Vzc29yIiwidHJhbnNhY3Rpb25Qcm9jZXNzb3JBQ0wiLCJ2b3VjaGVyTWFuYWdlbWVudCIsImh0dHBzOi8vMTI3LjAuMC4xOjUwMDEvcmVzb3VyY2VzIl0sInNjb3BlIjpbImVzdGF0ZU1hbmFnZW1lbnQiLCJlc3RhdGVSZXBvcnRpbmciLCJmaWxlUHJvY2Vzc29yIiwibWVzc2FnaW5nU2VydmljZSIsInRyYW5zYWN0aW9uUHJvY2Vzc29yIiwidHJhbnNhY3Rpb25Qcm9jZXNzb3JBQ0wiLCJ2b3VjaGVyTWFuYWdlbWVudCJdLCJjbGllbnRfaWQiOiJzZXJ2aWNlQ2xpZW50IiwianRpIjoiMDZBMUI4NzYyRjFGNDJGNkIwMzM5RTYwRTk2MkVDQkUifQ.G5pFWRJF430ZZxnGO_yIxEC6Zj81LRr3HNq6d8V9EV4Pswp5YO7hZ867Ln4mjnrYag4lGI4cpT5S6646r9KNdZMiLOsdQs2LEJPuUjEdVADIwm8rcdqT8OX-sC6uGA6VL0bMmYWQXw1E8d4kax444I6jeNquLjpoWVD1BDp9L1zzC6e_k9W7Fc9MQOogOqO82TXrBl9nkpBbmJQ0HDiub2yVUTUKLwkCeRfBDlyeU8tyNE7kH6IGdHIL_WYUtiiRYjBJ2PNLzTtrXQk4rqw6GB-25K2qcgP5FO0MI675tAkuPKI0DaySXHnAjYssW8wZYy0tkaJL0OIlmOUe-9jM6g"; - //await Jobs.SendSupportEmail(DateTime.Now, - // accessToken, - // "esdb://admin:changeit@192.168.0.133:2113?tls=false&tlsVerifyCert=false", - // "server=192.168.0.133;user id=sa;password=Sc0tland;database=EstateReportingReadModel;Encrypt=True;TrustServerCertificate=True", - // new List{ - // "435613ac-a468-47a3-ac4f-649d89764c22" - // }, - // messagingServiceClient, - // CancellationToken.None); - - //HttpClientHandler handler = new HttpClientHandler - //{ - // ServerCertificateCustomValidationCallback = (message, - // cert, - // chain, - // errors) => - // { - // return true; - // } - //}; - //HttpClient client = new HttpClient(handler); - //ISecurityServiceClient securityServiceClient = new SecurityServiceClient(delegate (String s) { return "https://192.168.1.167:5001"; }, client); - //ITransactionProcessorClient transactionProcessorClient = new TransactionProcessorClient(delegate (String s) { return "https://eojrtqfzvyheu0l.m.pipedream.net"; }, client); - //String transactionProcessorApi = "http://192.168.1.167:5002"; - //String fileProcessorApi = "http://192.168.1.167:5009"; - //String testHostApi = "http://192.168.1.167:9000"; - //String clientId = "serviceClient"; - //String clientSecret = "d192cbc46d834d0da90e8a9d50ded543"; - - //ITransactionDataGeneratorService t = new TransactionDataGeneratorService(securityServiceClient, - // transactionProcessorClient, - // transactionProcessorApi, - // fileProcessorApi, - // testHostApi, - // clientId, - // clientSecret, - // RunningMode.WhatIf); - //Guid estateId = Guid.Parse("435613ac-a468-47a3-ac4f-649d89764c22"); - - //MakeFloatCreditsJobConfig c = new MakeFloatCreditsJobConfig(clientId,clientSecret, fileProcessorApi,"","", transactionProcessorApi, estateId, - // new List { new DepositAmount("" ,"", 100) } - //); - - //await Jobs.GenerateFloatCredits(t, c, CancellationToken.None); - //Guid merchantId = Guid.Parse("ab1c99fb-1c6c-4694-9a32-b71be5d1da33"); - //await Jobs.GenerateTransactions(t, estateId, merchantId, false, CancellationToken.None); - ////var d = TransactionDataGenerator.GetTransactionDateTime(new Random(), DateTime.Now); - ////Console.WriteLine(d); - //await Jobs.PerformSettlement(t, DateTime.Now,estateId, CancellationToken.None); - - +using System.Threading.Tasks; +namespace JobTestDriver +{ + internal class Program + { + static Task Main(string[] args) + { + return Task.CompletedTask; } } -} \ No newline at end of file +} diff --git a/TransactionProcessing.SchedulerService/TransactionProcessing.SchedulerService.DataGenerator/TransactionDataGeneratorService.cs b/TransactionProcessing.SchedulerService/TransactionProcessing.SchedulerService.DataGenerator/TransactionDataGeneratorService.cs index 49d719e..faf0a7b 100644 --- a/TransactionProcessing.SchedulerService/TransactionProcessing.SchedulerService.DataGenerator/TransactionDataGeneratorService.cs +++ b/TransactionProcessing.SchedulerService/TransactionProcessing.SchedulerService.DataGenerator/TransactionDataGeneratorService.cs @@ -6,6 +6,7 @@ using SimpleResults; using System; using System.Net.Http.Headers; +using System.Security.Cryptography; using System.Text; using TransactionProcessor.Client; using TransactionProcessor.DataTransferObjects; @@ -210,13 +211,13 @@ public async Task SendSales(DateTime dateTime, (Int32 meterNumber, String customerName, Decimal amount) meterDetails = default; // Step 2: Decide how many total transactions to generate in this run - numberOfSales = this.r.Next(1, 6); // e.g. 1–5 transactions per run + numberOfSales = RandomNumberGenerator.GetInt32(1, 6); // e.g. 1–5 transactions per run List<(SaleTransactionRequest request, Decimal amount)> saleRequests = null; for (int i = 0; i < numberOfSales; i++) { - ContractProduct? contractProduct = contract.Products[this.r.Next(contract.Products.Count)]; + ContractProduct? contractProduct = contract.Products[RandomNumberGenerator.GetInt32(contract.Products.Count)]; // Spread transactions randomly across 5 minutes - int delayMs = this.r.Next(0, timeDelay * 60 * 1000); + int delayMs = RandomNumberGenerator.GetInt32(0, timeDelay * 60 * 1000); await Task.Delay(delayMs); ProductSubType productSubType = this.GetProductSubType(contract.OperatorName); @@ -236,7 +237,7 @@ public async Task SendSales(DateTime dateTime, ProductSubType.Voucher => this.BuildVoucherSaleRequests(dateTime, merchant, contract, contractProduct), ProductSubType.BillPaymentPostPay => this.BuildPataPawaPostPayBillPaymentSaleRequests(dateTime, merchant, contract, contractProduct, billDetails), ProductSubType.BillPaymentPrePay => this.BuildPataPawaPrePayBillPaymentSaleRequests(dateTime, merchant, contract, contractProduct, meterDetails), - _ => throw new Exception($"Product Sub Type [{productSubType}] not yet supported") + _ => throw new InvalidOperationException($"Product Sub Type [{productSubType}] not yet supported") }; // Add the value of the sale to the deposit amount @@ -290,7 +291,7 @@ public async Task SendUploadFile(DateTime dateTime, MerchantResponse merchant, Guid userId, CancellationToken cancellationToken) { - Int32 numberOfSales = this.r.Next(5, 15); + Int32 numberOfSales = RandomNumberGenerator.GetInt32(5, 15); var builtFileResult = await this.BuildUploadFile(dateTime, merchant, contract, numberOfSales, cancellationToken); if (builtFileResult.IsFailed) { @@ -406,7 +407,7 @@ private static async Task GetFileProfileIdFromOperator(String operatorName for (Int32 i = 1; i <= numberOfLines; i++) { - Decimal amount = GetAmount(this.r); + Decimal amount = GetAmount(); String mobileNumber = String.Format($"077777777{i.ToString().PadLeft(2, '0')}"); mobileTopupUploadFile.AddLine(amount, mobileNumber); @@ -434,7 +435,7 @@ private static async Task GetFileProfileIdFromOperator(String operatorName for (Int32 i = 1; i <= numberOfLines; i++) { - Decimal amount = GetAmount(this.r); + Decimal amount = GetAmount(); String mobileNumber = String.Format($"077777777{i.ToString().PadLeft(2, '0')}"); String emailAddress = String.Format($"testrecipient{i.ToString().PadLeft(2, '0')}@testing.com"); String recipient = mobileNumber; @@ -471,23 +472,22 @@ private static async Task GetFileProfileIdFromOperator(String operatorName private String TestHostApi; - public static Decimal GetAmount(Random r, ContractProduct product = null) + public static Decimal GetAmount(ContractProduct product = null) { return product switch { - null => r.Next(9, 250), - _ when product.Value.HasValue == false => r.Next(9, 250), + null => RandomNumberGenerator.GetInt32(9, 250), + _ when product.Value.HasValue == false => RandomNumberGenerator.GetInt32(9, 250), _ => product.Value.Value }; } - private Random r = new Random(); private async Task<(Int32 accountNumber, String accountName, Decimal balance)> CreateBillPaymentBill(String contractOperatorName, ContractProduct contractProduct, CancellationToken cancellationToken) { if (contractOperatorName == "PataPawa PostPay") { - Int32 accountNumber = this.r.Next(1, 100000); - Decimal amount = GetAmount(this.r, contractProduct); + Int32 accountNumber = RandomNumberGenerator.GetInt32(1, 100000); + Decimal amount = GetAmount(contractProduct); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, $"{this.TestHostApi}/api/developer/patapawapostpay/createbill"); var body = new @@ -520,8 +520,8 @@ public static Decimal GetAmount(Random r, ContractProduct product = null) { if (contractOperatorName == "PataPawa PrePay") { - Int32 meterNumber = this.r.Next(1, 100000); - Decimal amount = GetAmount(this.r, contractProduct); + Int32 meterNumber = RandomNumberGenerator.GetInt32(1, 100000); + Decimal amount = GetAmount(contractProduct); HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, $"{this.TestHostApi}/api/developer/patapawaprepay/createmeter"); var body = new @@ -834,7 +834,7 @@ private void WriteMessageX(String message, private List<(SaleTransactionRequest request, Decimal amount)> BuildVoucherSaleRequests(DateTime dateTime, MerchantResponse merchant, ContractResponse contract, ContractProduct contractProduct) { - Decimal amount = GetAmount(this.r, contractProduct); + Decimal amount = GetAmount(contractProduct); Dictionary requestMetaData = new Dictionary{ { "Amount", amount.ToString() }, @@ -854,7 +854,7 @@ private void WriteMessageX(String message, MerchantId = merchant.MerchantId, EstateId = merchant.EstateId, TransactionType = "Sale", - TransactionDateTime = GetTransactionDateTime(this.r, dateTime), + TransactionDateTime = GetTransactionDateTime(dateTime), OperatorId = contract.OperatorId, ProductId = contractProduct.ProductId }; @@ -863,21 +863,21 @@ private void WriteMessageX(String message, return requests; } - public static DateTime GetTransactionDateTime(Random r, DateTime dateTime) + public static DateTime GetTransactionDateTime(DateTime dateTime) { if (dateTime.Hour != 0) { // Already have a time only change the seconds - Int32 seconds = r.Next(0, 59); + Int32 seconds = RandomNumberGenerator.GetInt32(0, 59); return new DateTime(dateTime.Year, dateTime.Month, dateTime.Day, dateTime.Hour, dateTime.Minute, seconds); } else { // Generate the time - Int32 hours = r.Next(9, 22); - Int32 minutes = r.Next(0, 59); - Int32 seconds = r.Next(0, 59); + Int32 hours = RandomNumberGenerator.GetInt32(9, 22); + Int32 minutes = RandomNumberGenerator.GetInt32(0, 59); + Int32 seconds = RandomNumberGenerator.GetInt32(0, 59); return dateTime.AddHours(hours).AddMinutes(minutes).AddSeconds(seconds); } @@ -885,7 +885,7 @@ public static DateTime GetTransactionDateTime(Random r, DateTime dateTime) private List<(SaleTransactionRequest request, Decimal amount)> BuildMobileTopupSaleRequests(DateTime dateTime, MerchantResponse merchant, ContractResponse contract, ContractProduct contractProduct) { - Decimal amount = GetAmount(this.r, contractProduct); + Decimal amount = GetAmount(contractProduct); Dictionary requestMetaData = new Dictionary{ { "Amount", amount.ToString() }, @@ -905,7 +905,7 @@ public static DateTime GetTransactionDateTime(Random r, DateTime dateTime) MerchantId = merchant.MerchantId, EstateId = merchant.EstateId, TransactionType = "Sale", - TransactionDateTime = GetTransactionDateTime(this.r, dateTime), + TransactionDateTime = GetTransactionDateTime(dateTime), OperatorId = contract.OperatorId, ProductId = contractProduct.ProductId }; @@ -927,7 +927,7 @@ public static DateTime GetTransactionDateTime(Random r, DateTime dateTime) { "PataPawaPostPaidMessageType", "VerifyAccount" } }; - DateTime transactionDateTime = GetTransactionDateTime(this.r, dateTime); + DateTime transactionDateTime = GetTransactionDateTime(dateTime); SaleTransactionRequest getAccountRequest = new SaleTransactionRequest { @@ -986,7 +986,7 @@ public static DateTime GetTransactionDateTime(Random r, DateTime dateTime) { "PataPawaPrePayMessageType", "meter" } }; - DateTime transactionDateTime = GetTransactionDateTime(this.r, dateTime); + DateTime transactionDateTime = GetTransactionDateTime(dateTime); SaleTransactionRequest getAccountRequest = new SaleTransactionRequest { @@ -1072,4 +1072,4 @@ private ProductSubType GetProductSubType(String operatorName) return productType; } -} \ No newline at end of file +} diff --git a/TransactionProcessor.SystemSetupTool/EstateSetupFunctions.cs b/TransactionProcessor.SystemSetupTool/EstateSetupFunctions.cs index 2a551fa..1b37d1a 100644 --- a/TransactionProcessor.SystemSetupTool/EstateSetupFunctions.cs +++ b/TransactionProcessor.SystemSetupTool/EstateSetupFunctions.cs @@ -45,7 +45,7 @@ public EstateSetupFunctions(ISecurityServiceClient securityServiceClient, } public async Task SetupEstate(CancellationToken cancellationToken) { - var result = await this.SecurityServiceClient.GetToken("serviceClient", "d192cbc46d834d0da90e8a9d50ded543", CancellationToken.None); + var result = await this.SecurityServiceClient.GetToken("serviceClient", ResolveServiceClientSecret(), CancellationToken.None); this.TokenResponse = result.Data; Result createEstateResult = await this.CreateEstate(cancellationToken); @@ -77,6 +77,17 @@ public async Task SetupEstate(CancellationToken cancellationToken) { return Result.Success(); } + private static string ResolveServiceClientSecret() + { + string? secret = Environment.GetEnvironmentVariable("TRANSACTIONPROCESSOR_SERVICECLIENT_SECRET"); + if (!string.IsNullOrWhiteSpace(secret)) + { + return secret; + } + + throw new InvalidOperationException("Missing required secret for service client. Set TRANSACTIONPROCESSOR_SERVICECLIENT_SECRET."); + } + private async Task> GetEstate(Guid estateId, CancellationToken cancellationToken) { Result estateResponse = await this.TransactionProcessorClient.GetEstate(this.TokenResponse.AccessToken, estateId, cancellationToken); @@ -165,7 +176,7 @@ private async Task> GetContractId(String contractDescription, await Retry.For(async () => { var existingContractsResult = await this.TransactionProcessorClient.GetContracts(this.TokenResponse.AccessToken, this.EstateId, cancellationToken); if (existingContractsResult.IsFailed) - throw new Exception("GetContracts failed"); + throw new InvalidOperationException("GetContracts failed"); var c = existingContractsResult.Data.SingleOrDefault(c => c.Description == contractDescription); if (c == null) { @@ -190,7 +201,7 @@ await Retry.For(async () => { var getContractResult = await this.TransactionProcessorClient.GetContract(this.TokenResponse.AccessToken, this.EstateId, contractId, cancellationToken); if (getContractResult.IsFailed) - throw new Exception("GetContract failed"); + throw new InvalidOperationException("GetContract failed"); var cp = getContractResult.Data.Products.SingleOrDefault(p => p.Name == productName); if (cp == null) { @@ -396,73 +407,127 @@ public async Task CreateMerchant(Merchant merchant, private async Task UpdateMerchant(Merchant merchant, MerchantResponse existingMerchant, CancellationToken cancellationToken) { - // check the merchants device - if (existingMerchant.Devices.ContainsValue(merchant.Device.DeviceIdentifier) == false) { - AddMerchantDeviceRequest addMerchantDeviceRequest = new AddMerchantDeviceRequest { DeviceIdentifier = merchant.Device.DeviceIdentifier }; - var addDeviceToMerchantResult = await this.TransactionProcessorClient.AddDeviceToMerchant(this.TokenResponse.AccessToken, this.EstateId, existingMerchant.MerchantId, addMerchantDeviceRequest, cancellationToken); - if (addDeviceToMerchantResult.IsFailed) - return ResultHelpers.CreateFailure(addDeviceToMerchantResult); + var ensureDeviceResult = await EnsureMerchantDeviceAsync(merchant, existingMerchant, cancellationToken); + if (ensureDeviceResult.IsFailed) + { + return ResultHelpers.CreateFailure(ensureDeviceResult); + } + + var ensureUserResult = await EnsureMerchantUserAsync(merchant, cancellationToken); + if (ensureUserResult.IsFailed) + { + return ResultHelpers.CreateFailure(ensureUserResult); + } + + var ensureOperatorsResult = await EnsureMerchantOperatorsAsync(merchant, existingMerchant, cancellationToken); + if (ensureOperatorsResult.IsFailed) + { + return ResultHelpers.CreateFailure(ensureOperatorsResult); + } + + var ensureContractsResult = await EnsureMerchantContractsAsync(existingMerchant, cancellationToken); + if (ensureContractsResult.IsFailed) + { + return ResultHelpers.CreateFailure(ensureContractsResult); + } + + return Result.Success(); + } + + private async Task EnsureMerchantDeviceAsync(Merchant merchant, MerchantResponse existingMerchant, CancellationToken cancellationToken) + { + if (existingMerchant.Devices.ContainsValue(merchant.Device.DeviceIdentifier)) + { + return Result.Success(); } - // Check the users + AddMerchantDeviceRequest addMerchantDeviceRequest = new() { DeviceIdentifier = merchant.Device.DeviceIdentifier }; + var addDeviceToMerchantResult = await this.TransactionProcessorClient.AddDeviceToMerchant(this.TokenResponse.AccessToken, this.EstateId, existingMerchant.MerchantId, addMerchantDeviceRequest, cancellationToken); + return addDeviceToMerchantResult.IsFailed ? ResultHelpers.CreateFailure(addDeviceToMerchantResult) : Result.Success(); + } + + private async Task EnsureMerchantUserAsync(Merchant merchant, CancellationToken cancellationToken) + { var userResult = await this.SecurityServiceClient.GetUsers(merchant.User.EmailAddress, cancellationToken); if (userResult.IsFailed) + { return ResultHelpers.CreateFailure(userResult); - if (userResult.Data == null || userResult.Data.Count == 0) { - CreateMerchantUserRequest createMerchantUserRequest = new CreateMerchantUserRequest { - EmailAddress = merchant.User.EmailAddress, - FamilyName = merchant.User.FamilyName, - GivenName = merchant.User.GivenName, - MiddleName = merchant.User.MiddleName, - Password = merchant.User.Password - }; - var createMerchantUserResult = await this.TransactionProcessorClient.CreateMerchantUser(this.TokenResponse.AccessToken, this.EstateId, merchant.MerchantId, createMerchantUserRequest, cancellationToken); - if (createMerchantUserResult.IsFailed) - return ResultHelpers.CreateFailure(createMerchantUserResult); } + if (userResult.Data is { Count: > 0 }) + { + return Result.Success(); + } + + CreateMerchantUserRequest createMerchantUserRequest = new() { + EmailAddress = merchant.User.EmailAddress, + FamilyName = merchant.User.FamilyName, + GivenName = merchant.User.GivenName, + MiddleName = merchant.User.MiddleName, + Password = merchant.User.Password + }; + var createMerchantUserResult = await this.TransactionProcessorClient.CreateMerchantUser(this.TokenResponse.AccessToken, this.EstateId, merchant.MerchantId, createMerchantUserRequest, cancellationToken); + return createMerchantUserResult.IsFailed ? ResultHelpers.CreateFailure(createMerchantUserResult) : Result.Success(); + } + + private async Task EnsureMerchantOperatorsAsync(Merchant merchant, MerchantResponse existingMerchant, CancellationToken cancellationToken) + { var getEstateResult = await this.TransactionProcessorClient.GetEstate(this.TokenResponse.AccessToken, this.EstateId, cancellationToken); if (getEstateResult.IsFailed) + { return ResultHelpers.CreateFailure(getEstateResult); + } - foreach (var @operator in getEstateResult.Data.Operators) { - if (existingMerchant.Operators == null) { - existingMerchant.Operators = new List(); - } - - var merchantOperator = existingMerchant.Operators.SingleOrDefault(o => o.OperatorId == @operator.OperatorId); - if (merchantOperator != null) + existingMerchant.Operators ??= new List(); + foreach (var @operator in getEstateResult.Data.Operators) + { + if (existingMerchant.Operators.Any(o => o.OperatorId == @operator.OperatorId)) + { continue; + } DataTransferObjects.Requests.Merchant.AssignOperatorRequest assignOperatorRequest = new() { OperatorId = @operator.OperatorId, MerchantNumber = null, TerminalNumber = null }; - var assignOperatorToMerchantResult = await this.TransactionProcessorClient.AssignOperatorToMerchant(this.TokenResponse.AccessToken, this.EstateId, existingMerchant.MerchantId, assignOperatorRequest, cancellationToken); if (assignOperatorToMerchantResult.IsFailed) + { return ResultHelpers.CreateFailure(assignOperatorToMerchantResult); + } } + return Result.Success(); + } + + private async Task EnsureMerchantContractsAsync(MerchantResponse existingMerchant, CancellationToken cancellationToken) + { var getContractsResult = await this.TransactionProcessorClient.GetContracts(this.TokenResponse.AccessToken, this.EstateId, cancellationToken); if (getContractsResult.IsFailed) + { return ResultHelpers.CreateFailure(getContractsResult); + } var merchantContractsResult = await this.TransactionProcessorClient.GetMerchantContracts(this.TokenResponse.AccessToken, this.EstateId, existingMerchant.MerchantId, cancellationToken); if (merchantContractsResult.IsFailed && merchantContractsResult.Status != ResultStatus.NotFound) + { return ResultHelpers.CreateFailure(merchantContractsResult); - List merchantContracts = merchantContractsResult.Data; - if (merchantContractsResult.Status == ResultStatus.NotFound) { - merchantContracts = new List(); } - // Now contracts - foreach (ContractResponse contractResponse in getContractsResult.Data) { - if (merchantContracts.SingleOrDefault(c => c.ContractId == contractResponse.ContractId) != null) + var merchantContracts = merchantContractsResult.Status == ResultStatus.NotFound || merchantContractsResult.Data is null + ? new List() + : merchantContractsResult.Data; + + foreach (ContractResponse contractResponse in getContractsResult.Data) + { + if (merchantContracts.Any(c => c.ContractId == contractResponse.ContractId)) + { continue; + } AddMerchantContractRequest addMerchantContractRequest = new() { ContractId = contractResponse.ContractId }; var addContractToMerchantResult = await this.TransactionProcessorClient.AddContractToMerchant(this.TokenResponse.AccessToken, this.EstateId, existingMerchant.MerchantId, addMerchantContractRequest, cancellationToken); - if (addContractToMerchantResult.IsFailed) + { return ResultHelpers.CreateFailure(addContractToMerchantResult); + } } return Result.Success(); diff --git a/TransactionProcessor.SystemSetupTool/EventStoreFunctions.cs b/TransactionProcessor.SystemSetupTool/EventStoreFunctions.cs index 5ee0b5d..b811657 100644 --- a/TransactionProcessor.SystemSetupTool/EventStoreFunctions.cs +++ b/TransactionProcessor.SystemSetupTool/EventStoreFunctions.cs @@ -96,27 +96,27 @@ private async Task DeployProjections(CancellationToken cancellationToken foreach (var projection in projectionsToDeploy) { - if (projection.Contains("EstateManagementSubscriptionStreamBuilder") || - projection.Contains("FileProcessorSubscriptionStreamBuilder") || - projection.Contains("TransactionProcessorSubscriptionStreamBuilder") || - projection.Contains("EstateAggregator") || - projection.Contains("MerchantAggregator") || - projection.Contains("CallbackHandlerEnricher")) + if (projection.Contains("EstateManagementSubscriptionStreamBuilder", StringComparison.Ordinal) || + projection.Contains("FileProcessorSubscriptionStreamBuilder", StringComparison.Ordinal) || + projection.Contains("TransactionProcessorSubscriptionStreamBuilder", StringComparison.Ordinal) || + projection.Contains("EstateAggregator", StringComparison.Ordinal) || + projection.Contains("MerchantAggregator", StringComparison.Ordinal) || + projection.Contains("CallbackHandlerEnricher", StringComparison.Ordinal)) { continue; } FileInfo f = new FileInfo(projection); - String name = f.Name.Substring(0, f.Name.Length - (f.Name.Length - f.Name.LastIndexOf("."))); + String name = Path.GetFileNameWithoutExtension(f.Name); var body = File.ReadAllText(f.FullName); - var x = body.IndexOf("//endtestsetup"); + var x = body.IndexOf("//endtestsetup", StringComparison.Ordinal); x = x + "//endtestsetup".Length; body = body.Substring(x); // Is this already deployed (in the master list) - if (currentProjections.Any(p => p.Name == name) == false) + if (currentProjections.Any(p => string.Equals(p.Name, name, StringComparison.Ordinal)) == false) { // Projection does not exist so create await this.ProjectionClient.CreateContinuousAsync(name, body, true, cancellationToken: cancellationToken); @@ -132,4 +132,4 @@ private async Task DeployProjections(CancellationToken cancellationToken return Result.Success(); } -} \ No newline at end of file +} diff --git a/TransactionProcessor.SystemSetupTool/Program.cs b/TransactionProcessor.SystemSetupTool/Program.cs index e532c68..e0acc06 100644 --- a/TransactionProcessor.SystemSetupTool/Program.cs +++ b/TransactionProcessor.SystemSetupTool/Program.cs @@ -95,7 +95,7 @@ static async Task Main(string[] args) { private static async Task SetupFileProcessors(FileProcessingOptions fileProcessingOptions, CancellationToken cancellationToken) { - Result tokenResult = await SecurityServiceClient.GetToken("serviceClient", "d192cbc46d834d0da90e8a9d50ded543", CancellationToken.None); + Result tokenResult = await SecurityServiceClient.GetToken("serviceClient", ResolveServiceClientSecret(), CancellationToken.None); if (tokenResult.IsFailed) return ResultHelpers.CreateFailure(tokenResult); foreach (FileProfile fileProfile in fileProcessingOptions.FileProfiles) { @@ -153,10 +153,56 @@ private static async Task GetIdentityServerConfig(C IdentityServerConfiguration identityServerConfiguration = StringSerialiser.Deserialise(identityServerJsonData); + ApplyIdentityServerSecrets(identityServerConfiguration); return identityServerConfiguration; } + private static void ApplyIdentityServerSecrets(IdentityServerConfiguration identityServerConfiguration) + { + foreach (var apiResource in identityServerConfiguration.apiresources) + { + apiResource.secret = ResolveRequiredSecret(GetIdentityServerSecretEnvironmentVariable(apiResource.name), $"api resource '{apiResource.name}'"); + } + + foreach (var client in identityServerConfiguration.clients) + { + client.secret = ResolveRequiredSecret(GetIdentityServerSecretEnvironmentVariable(client.client_id), $"client '{client.client_id}'"); + } + } + + private static string ResolveServiceClientSecret() + { + return ResolveRequiredSecret("TRANSACTIONPROCESSOR_SERVICECLIENT_SECRET", "service client"); + } + + private static string ResolveRequiredSecret(string environmentVariableName, string description) + { + string? secret = Environment.GetEnvironmentVariable(environmentVariableName); + if (!string.IsNullOrWhiteSpace(secret)) + { + return secret; + } + + throw new InvalidOperationException($"Missing required secret for {description}. Set the {environmentVariableName} environment variable."); + } + + private static string GetIdentityServerSecretEnvironmentVariable(string identifier) + { + return identifier switch + { + "estateReporting" => "TRANSACTIONPROCESSOR_ESTATEREPORTING_SECRET", + "messagingService" => "TRANSACTIONPROCESSOR_MESSAGINGSERVICE_SECRET", + "transactionProcessor" => "TRANSACTIONPROCESSOR_TRANSACTIONPROCESSOR_SECRET", + "transactionProcessorACL" => "TRANSACTIONPROCESSOR_TRANSACTIONPROCESSORACL_SECRET", + "fileProcessor" => "TRANSACTIONPROCESSOR_FILEPROCESSOR_SECRET", + "serviceClient" => "TRANSACTIONPROCESSOR_SERVICECLIENT_SECRET", + "managementUIClient" => "TRANSACTIONPROCESSOR_MANAGEMENTUICLIENT_SECRET", + "mobileAppClient" => "TRANSACTIONPROCESSOR_MOBILEAPPCLIENT_SECRET", + _ => $"TRANSACTIONPROCESSOR_{identifier.ToUpperInvariant()}_SECRET" + }; + } + private static async Task GetFileProfileConfig(CancellationToken cancellationToken) { // Read the file profile config json string diff --git a/TransactionProcessor.SystemSetupTool/identityserverconfig.json b/TransactionProcessor.SystemSetupTool/identityserverconfig.json index bdccdba..c9ddba8 100644 --- a/TransactionProcessor.SystemSetupTool/identityserverconfig.json +++ b/TransactionProcessor.SystemSetupTool/identityserverconfig.json @@ -49,7 +49,7 @@ ], "apiresources": [ { - "secret": "3241913328414df4b8d031ed8f59d1bb", + "secret": "", "name": "estateReporting", "display_name": "Estate Reporting REST", "description": "API Resource representing Estate Reporting REST", @@ -57,7 +57,7 @@ "user_claims": null }, { - "secret": "32c677c3cf2a4b8f8ccfb27a37c6f27e", + "secret": "", "name": "messagingService", "display_name": "Messaging REST", "description": "API Resource representing Messaging Service REST", @@ -65,7 +65,7 @@ "user_claims": null }, { - "secret": "14363045ab8649909ec812e587720333", + "secret": "", "name": "transactionProcessor", "display_name": "Transaction Processor REST", "description": "API Resource representing Transaction Processor REST", @@ -73,7 +73,7 @@ "user_claims": [ "merchantId", "estateId", "role" ] }, { - "secret": "bc7c25e9bf1649cc9d41abebfe98955e", + "secret": "", "name": "transactionProcessorACL", "display_name": "Transaction Processor ACL REST", "description": "API Resource representing Transaction Processor ACL REST", @@ -81,7 +81,7 @@ "user_claims": [ "merchantId", "estateId", "role" ] }, { - "secret": "04f91a4057a041b9a65765dfc4efac7d", + "secret": "", "name": "fileProcessor", "display_name": "File Processor REST", "description": "API Resource representing File Processor REST", @@ -92,7 +92,7 @@ "clients": [ { "client_id": "serviceClient", - "secret": "d192cbc46d834d0da90e8a9d50ded543", + "secret": "", "client_name": "Service Client", "client_description": "Client for use in inter service communications", "allowed_scopes": [ "transactionProcessorACL", "transactionProcessor", "estateReporting", "messagingService", "fileProcessor" ], @@ -100,7 +100,7 @@ }, { "client_id": "managementUIClient", - "secret": "d192cbc46d834d0da90e8a9d50ded543", + "secret": "", "client_name": "Management UI App Client", "client_description": "Client for use by management app", "allowed_scopes": [ "fileProcessor", "transactionProcessor", "openid", "email", "profile" ], @@ -123,7 +123,7 @@ }, { "client_id": "mobileAppClient", - "secret": "d192cbc46d834d0da90e8a9d50ded543", + "secret": "", "client_name": "Mobile App Client", "client_description": "Client for use by mobile app to process transactions", "allowed_scopes": [ "transactionProcessorACL" ], diff --git a/TransactionProcessor.SystemSetupTool/setupconfig.json b/TransactionProcessor.SystemSetupTool/setupconfig.json index 5247308..2d97f13 100644 --- a/TransactionProcessor.SystemSetupTool/setupconfig.json +++ b/TransactionProcessor.SystemSetupTool/setupconfig.json @@ -6,7 +6,7 @@ "email_address": "estateuser@devestate1.co.uk", "user": { "email_address": "estateuser@devestate1.co.uk", - "password": "123456", + "password": "", "given_name": "Development Estate", "middle_name": "", "family_name": "User" @@ -35,7 +35,7 @@ ], "user": { "email_address": "merchantuser@testmerchant1.co.uk", - "password": "123456", + "password": "", "given_name": "Test Merchant 1", "middle_name": "", "family_name": "User" @@ -187,4 +187,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/TransactionProcessor.SystemSetupTool/setupconfig.production.json b/TransactionProcessor.SystemSetupTool/setupconfig.production.json index c310110..75af752 100644 --- a/TransactionProcessor.SystemSetupTool/setupconfig.production.json +++ b/TransactionProcessor.SystemSetupTool/setupconfig.production.json @@ -6,7 +6,7 @@ "email_address": "estateuser@productionestate1.co.uk", "user": { "email_address": "estateuser@productionestate1.co.uk", - "password": "123456", + "password": "", "given_name": "Production Estate", "middle_name": "", "family_name": "User" @@ -35,7 +35,7 @@ ], "user": { "email_address": "merchantuser@livemerchant1.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 1", "middle_name": "", "family_name": "User" @@ -68,7 +68,7 @@ ], "user": { "email_address": "merchantuser@livemerchant2.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 2", "middle_name": "", "family_name": "User" @@ -101,7 +101,7 @@ ], "user": { "email_address": "merchantuser@livemerchant3.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 3", "middle_name": "", "family_name": "User" @@ -134,7 +134,7 @@ ], "user": { "email_address": "merchantuser@livemerchant4.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 4", "middle_name": "", "family_name": "User" @@ -167,7 +167,7 @@ ], "user": { "email_address": "merchantuser@livemerchant5.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 5", "middle_name": "", "family_name": "User" @@ -200,7 +200,7 @@ ], "user": { "email_address": "merchantuser@livemerchant6.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 6", "middle_name": "", "family_name": "User" @@ -233,7 +233,7 @@ ], "user": { "email_address": "merchantuser@livemerchant7.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 7", "middle_name": "", "family_name": "User" @@ -266,7 +266,7 @@ ], "user": { "email_address": "merchantuser@livemerchant8.co.uk", - "password": "123456", + "password": "", "given_name": "Live Merchant 8", "middle_name": "", "family_name": "User" @@ -419,4 +419,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/TransactionProcessor.SystemSetupTool/setupconfig.staging.json b/TransactionProcessor.SystemSetupTool/setupconfig.staging.json index ccbd7dd..ddd9ce3 100644 --- a/TransactionProcessor.SystemSetupTool/setupconfig.staging.json +++ b/TransactionProcessor.SystemSetupTool/setupconfig.staging.json @@ -6,7 +6,7 @@ "email_address": "estateuser@stagingestate1.co.uk", "user": { "email_address": "estateuser@stagingestate1.co.uk", - "password": "123456", + "password": "", "given_name": "Staging Estate", "middle_name": "", "family_name": "User" @@ -35,7 +35,7 @@ ], "user": { "email_address": "merchantuser@stagingmerchant1.co.uk", - "password": "123456", + "password": "", "given_name": "Staging Merchant 1", "middle_name": "", "family_name": "User" @@ -68,7 +68,7 @@ ], "user": { "email_address": "merchantuser@stagingmerchant2.co.uk", - "password": "123456", + "password": "", "given_name": "Staging Merchant 2", "middle_name": "", "family_name": "User" @@ -101,7 +101,7 @@ ], "user": { "email_address": "merchantuser@stagingmerchant3.co.uk", - "password": "123456", + "password": "", "given_name": "Staging Merchant 3", "middle_name": "", "family_name": "User" @@ -253,4 +253,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/docker-compose-master.yml b/docker-compose-master.yml index fe32d67..ded576b 100644 --- a/docker-compose-master.yml +++ b/docker-compose-master.yml @@ -9,7 +9,7 @@ services: - E:/home/txnproc/docker/master/db/log:/var/opt/mssql/log environment: - ACCEPT_EULA=Y - - SA_PASSWORD=thisisalongpassword123! + - SA_PASSWORD=${SQLSERVER_SA_PASSWORD} kurrentdb: image: kurrentplatform/kurrentdb:26.1 @@ -36,10 +36,10 @@ services: environment: - ServiceOptions:PublicOrigin=https://securityservice:5001 - ServiceOptions:IssuerUrl=https://securityservice:5001 - - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=PersistedGrantStore;Encrypt=false - - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=Configuration;Encrypt=false - - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=AuthenticationDbContext;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false + - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=PersistedGrantStore;Encrypt=false + - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=Configuration;Encrypt=false + - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=AuthenticationDbContext;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false - urls=https://*:5001 restart: on-failure depends_on: @@ -73,7 +73,7 @@ services: ports: - "5002:5002" environment: - - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TransactionProcessorReadModel;Encrypt=false + - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TransactionProcessorReadModel;Encrypt=false - EventStoreSettings:ConnectionString=esdb://admin:changeit@kurrentdb:2113?tls=false&tlsVerifyCert=false - EventStoreSettings:Insecure=true - AppSettings:SecurityService=https://securityservice:5001 @@ -81,7 +81,7 @@ services: - urls=http://*:5002 - AppSettings:EstateManagementApi=http://estatemanagement:5000 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} - OperatorConfiguration:Safaricom:Url=http://testhosts:9000/api/safaricom - OperatorConfiguration:PataPawaPrePay:Url=http://testhosts:9000/api/patapawaprepay - OperatorConfiguration:PataPawaPrePay:ApiLogonRequired=true @@ -106,7 +106,7 @@ services: - urls=http://*:5003 - AppSettings:TransactionProcessorApi=http://transactionprocessor:5002 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} depends_on: - securityservice - transactionprocessor @@ -117,9 +117,9 @@ services: volumes: - E:/home/txnproc/docker/master/trace:/home/txnproc/trace environment: - - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TestBankReadModel;Encrypt=false - - ConnectionStrings:PataPawaReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=PataPawaReadModel;Encrypt=false - - ConnectionStrings:AgencyBankingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=AgencyBankingReadModel;Encrypt=false + - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TestBankReadModel;Encrypt=false + - ConnectionStrings:PataPawaReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=PataPawaReadModel;Encrypt=false + - ConnectionStrings:AgencyBankingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=AgencyBankingReadModel;Encrypt=false ports: - "9000:9000" restart: on-failure @@ -143,9 +143,9 @@ services: - AppSettings:TransactionProcessorApi=http://transactionprocessor:5002 - AppSettings:EstateManagementApi=http://estatemanagement:5000 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 - - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TransactionProcessorReadModel;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} + - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TransactionProcessorReadModel;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false - AppSettings:TemporaryFileLocation=/home/txnproc/bulkfiles/temporary - AppSettings:FileProfiles__0__ListeningDirectory=/home/txnproc/bulkfiles/safaricom - AppSettings:FileProfiles__1__ListeningDirectory=/home/txnproc/bulkfiles/voucher @@ -180,8 +180,8 @@ services: - AppSettings:SecurityService=https://securityservice:5001 - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5011 - - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TransactionProcessorReadModel;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false + - ConnectionStrings:TransactionProcessorReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TransactionProcessorReadModel;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false depends_on: - sqlserver - kurrentdb diff --git a/docker-compose-prerelease.yml b/docker-compose-prerelease.yml index 6730282..830142e 100644 --- a/docker-compose-prerelease.yml +++ b/docker-compose-prerelease.yml @@ -10,7 +10,7 @@ services: - F:\home\txnproc\docker\dev\db\log:/var/opt/mssql/log environment: - ACCEPT_EULA=Y - - SA_PASSWORD=thisisalongpassword123! + - SA_PASSWORD=${SQLSERVER_SA_PASSWORD} eventstore: image: eventstore/eventstore:21.2.0-buster-slim @@ -37,10 +37,10 @@ services: environment: - ServiceOptions:PublicOrigin=https://securityservice:5001 - ServiceOptions:IssuerUrl=https://securityservice:5001 - - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=PersistedGrantStore - - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=Configuration - - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=AuthenticationDbContext - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123! | + - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=PersistedGrantStore + - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=Configuration + - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=AuthenticationDbContext + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD} | - urls=https://*:5001 restart: on-failure:5 depends_on: @@ -59,8 +59,8 @@ services: - AppSettings:SecurityService=https://securityservice:5001 - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5000 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123! | + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD} | depends_on: - sqlserver_dev - eventstore_dev @@ -79,8 +79,8 @@ services: - AppSettings:SecurityService=https://securityservice:5001 - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5005 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123! | + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD} | depends_on: - sqlserver_dev - eventstore_dev @@ -121,7 +121,7 @@ services: - AppSettings:EstateManagementApi=http://estatemanagement:5000 - AppSettings:VoucherManagementApi=http://vouchermanagement:5007 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} - OperatorConfiguration:Safaricom:Url=http://testhosts:9000/api/safaricom depends_on: - eventstore_dev @@ -143,7 +143,7 @@ services: - urls=http://*:5003 - AppSettings:TransactionProcessorApi=http://transactionprocessor:5002 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} depends_on: - securityservice_dev - transactionprocessor_dev @@ -161,10 +161,10 @@ services: - AppSettings:SecurityService=https://securityservice:5001 - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5007 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123! | + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD} | - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} - AppSettings:EstateManagementApi=http://estatemanagement:5000 depends_on: - sqlserver_dev @@ -184,7 +184,7 @@ services: - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5008 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} - AppSettings:VoucherManagementApi=http://vouchermanagement:5007 depends_on: - securityservice_dev @@ -196,7 +196,7 @@ services: volumes: - F:\home\txnproc\docker\dev\trace:/home/txnproc/trace environment: - - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TestBankReadModel + - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TestBankReadModel ports: - "9000:9000" restart: on-failure:5 @@ -220,9 +220,9 @@ services: - AppSettings:TransactionProcessorApi=http://transactionProcessor:5002 - AppSettings:EstateManagementApi=http://estateManamgement:5000 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123! | + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD} | - AppSettings:TemporaryFileLocation=/home/txnproc/bulkfiles/temporary - AppSettings:FileProfiles:0:ListeningDirectory=/home/txnproc/bulkfiles/safaricom - AppSettings:FileProfiles:1:ListeningDirectory=/home/txnproc/bulkfiles/voucher diff --git a/docker-compose.yml b/docker-compose.yml index fa3361d..d662c57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - F:\home\txnproc\docker\db\log:/var/opt/mssql/log environment: - ACCEPT_EULA=Y - - SA_PASSWORD=thisisalongpassword123! + - SA_PASSWORD=${SQLSERVER_SA_PASSWORD} eventstore: image: eventstore/eventstore:21.2.0-buster-slim @@ -37,10 +37,10 @@ services: environment: - ServiceOptions:PublicOrigin=https://securityservice:5001 - ServiceOptions:IssuerUrl=https://securityservice:5001 - - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=PersistedGrantStore;Encrypt=false - - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=Configuration;Encrypt=false - - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=AuthenticationDbContext;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false | + - ConnectionStrings:PersistedGrantDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=PersistedGrantStore;Encrypt=false + - ConnectionStrings:ConfigurationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=Configuration;Encrypt=false + - ConnectionStrings:AuthenticationDbContext=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=AuthenticationDbContext;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false | - urls=https://*:5001 restart: on-failure:5 depends_on: @@ -60,8 +60,8 @@ services: - AppSettings:Authority=https://securityservice:5001 - SecurityConfiguration:Authority=https://securityservice:5001 - urls=http://*:5000 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false | + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false | depends_on: - sqlserver - eventstore @@ -80,8 +80,8 @@ services: # - AppSettings:SecurityService=https://securityservice:5001 # - SecurityConfiguration:Authority=https://securityservice:5001 # - urls=http://*:5005 - # - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel;Encrypt=false - # - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false | + # - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel;Encrypt=false + # - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false | # depends_on: # - sqlserver # - eventstore @@ -122,7 +122,7 @@ services: - AppSettings:EstateManagementApi=http://estatemanagement:5000 - AppSettings:VoucherManagementApi=http://vouchermanagement:5007 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} - OperatorConfiguration:Safaricom:Url=http://testhosts:9000/api/safaricom - OperatorConfiguration:PataPawaPostPay:Url=http://testhosts:9000/PataPawaPostPayService/basichttp - OperatorConfiguration:PataPawaPrePay:Url=http://testhosts:9000/api/patapawaprepay @@ -145,7 +145,7 @@ services: - urls=http://*:5003 - AppSettings:TransactionProcessorApi=http://transactionprocessor:5002 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} depends_on: - securityservice - transactionprocessor @@ -156,8 +156,8 @@ services: volumes: - F:\home\txnproc\docker\trace:/home/txnproc/trace environment: - - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=TestBankReadModel;Encrypt=false - - ConnectionStrings:PataPawaReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=PataPawaReadModel;Encrypt=false + - ConnectionStrings:TestBankReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=TestBankReadModel;Encrypt=false + - ConnectionStrings:PataPawaReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=PataPawaReadModel;Encrypt=false ports: - "9000:9000" restart: on-failure:5 @@ -181,9 +181,9 @@ services: - AppSettings:TransactionProcessorApi=http://transactionProcessor:5002 - AppSettings:EstateManagementApi=http://estateManamgement:5000 - AppSettings:ClientId=serviceClient - - AppSettings:ClientSecret=d192cbc46d834d0da90e8a9d50ded543 - - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=thisisalongpassword123!;database=EstateReportingReadModel;Encrypt=false - - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=thisisalongpassword123!;Encrypt=false | + - AppSettings:ClientSecret=${TRANSACTIONPROCESSOR_CLIENTSECRET} + - ConnectionStrings:EstateReportingReadModel=server=sqlserver;user id=sa;password=${SQLSERVER_SA_PASSWORD};database=EstateReportingReadModel;Encrypt=false + - ConnectionStrings:HealthCheck=server=sqlserver;database=master;user id=sa;password=${SQLSERVER_SA_PASSWORD};Encrypt=false | - AppSettings:TemporaryFileLocation=/home/txnproc/bulkfiles/temporary - AppSettings:FileProfiles:0:ListeningDirectory=/home/txnproc/bulkfiles/safaricom - AppSettings:FileProfiles:1:ListeningDirectory=/home/txnproc/bulkfiles/voucher From 29d424a6d04671c785e8d4c33d3ceb80140dd7ca Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Wed, 26 Aug 2026 10:30:41 +0100 Subject: [PATCH 2/2] Support username and password authentication in scavenge script --- ScheduledTasks/AutoScavenge/scavenge.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ScheduledTasks/AutoScavenge/scavenge.ps1 b/ScheduledTasks/AutoScavenge/scavenge.ps1 index c7e6c3e..c048bca 100644 --- a/ScheduledTasks/AutoScavenge/scavenge.ps1 +++ b/ScheduledTasks/AutoScavenge/scavenge.ps1 @@ -1,5 +1,7 @@ param( [string]$BaseUrl = "http://localhost:2113", + [string]$Username, + [string]$Password, [System.Management.Automation.PSCredential]$Credential ) @@ -17,6 +19,12 @@ if ($Credential) { $encoded = [Convert]::ToBase64String($bytes) $headers['Authorization'] = "Basic $encoded" } +elseif (-not [string]::IsNullOrWhiteSpace($Username) -and -not [string]::IsNullOrWhiteSpace($Password)) { + $pair = "$Username`:$Password" + $bytes = [System.Text.Encoding]::ASCII.GetBytes($pair) + $encoded = [Convert]::ToBase64String($bytes) + $headers['Authorization'] = "Basic $encoded" +} try { Invoke-RestMethod -Method Post -Uri $uri -Headers $headers