Skip to content

Reject oversized files with an audit event #831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
StuartFerguson merged 7 commits into from
Oct 16, 2026
Merged

Reject oversized files with an audit event #831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
50b2ec3
docs: design file rejection event
StuartFerguson Oct 16, 2026
6efcd20
feat: reject oversized files with audit event
StuartFerguson Oct 16, 2026
810c59b
Merge branch 'main' into task/#796_upload-temp-file-processing
StuartFerguson Oct 16, 2026
3873fb5
package updates :|
StuartFerguson Oct 16, 2026
01f5701
Merge branch 'task/#796_upload-temp-file-processing' of https://githu…
StuartFerguson Oct 16, 2026
1f1bd55
fix: address oversized file review feedback
StuartFerguson Oct 16, 2026
d351998
fix: keep uploaded file ids separate from import log files
StuartFerguson Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 40 additions & 40 deletions Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,61 +4,61 @@
<PackageVersion Include="AspNetCore.HealthChecks.UI.Client" Version="9.0.0" />
<PackageVersion Include="AspNetCore.HealthChecks.UI.InMemory.Storage" Version="9.0.0" />
<PackageVersion Include="AspNetCore.HealthChecks.Uris" Version="9.0.0" />
<PackageVersion Include="ClientProxyBase" Version="2026.10.2" />
<PackageVersion Include="HealthMonitoring.Client" Version="2026.9.4.48" />
<PackageVersion Include="OpenIddict.Validation.AspNetCore" Version="7.7.1" />
<PackageVersion Include="OpenIddict.Validation.SystemNetHttp" Version="7.7.1" />
<PackageVersion Include="coverlet.collector" Version="10.0.1" />
<PackageVersion Include="coverlet.msbuild" Version="10.0.1" />
<PackageVersion Include="ClientProxyBase" Version="2026.10.3" />
<PackageVersion Include="HealthMonitoring.Client" Version="2026.9.4.53" />
<PackageVersion Include="OpenIddict.Validation.AspNetCore" Version="7.7.2" />
<PackageVersion Include="OpenIddict.Validation.SystemNetHttp" Version="7.7.2" />
<PackageVersion Include="coverlet.collector" Version="10.1.0" />
<PackageVersion Include="coverlet.msbuild" Version="10.1.0" />
<PackageVersion Include="Ductus.FluentDocker" Version="2.85.0" />
<PackageVersion Include="EventStoreProjections" Version="2023.12.3" />
<PackageVersion Include="Grpc.Net.Client" Version="2.84.0" />
<PackageVersion Include="Imposter" Version="0.1.11" />
<PackageVersion Include="Imposter" Version="1.0.1" />
<PackageVersion Include="Lamar" Version="16.0.0" />
<PackageVersion Include="Lamar.Microsoft.DependencyInjection" Version="16.0.0" />
<PackageVersion Include="MessagingService.IntegrationTesting.Helpers" Version="2026.9.3.154" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.12" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="10.0.12" />
<PackageVersion Include="Microsoft.EntityFrameworkCore" Version="10.0.12" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.12" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.12" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.12" />
<PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="10.0.12" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.12" />
<PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.11.2" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.10.1" />
<PackageVersion Include="MessagingService.IntegrationTesting.Helpers" Version="2026.9.3.155" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.13" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="10.0.13" />
<PackageVersion Include="Microsoft.EntityFrameworkCore" Version="10.0.13" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.InMemory" Version="10.0.13" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.13" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.13" />
<PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="10.0.13" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.13" />
<PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.12.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.11.0" />
<PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.23.0" />
<PackageVersion Include="MediatR" Version="14.2.0" />
<PackageVersion Include="NLog.Extensions.Logging" Version="6.2.1" />
<PackageVersion Include="NUnit" Version="4.6.1" />
<PackageVersion Include="NUnit3TestAdapter" Version="6.3.0" />
<PackageVersion Include="NLog.Extensions.Logging" Version="6.2.2" />
<PackageVersion Include="NUnit" Version="5.0.0" />
<PackageVersion Include="NUnit3TestAdapter" Version="6.3.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
<PackageVersion Include="Reqnroll" Version="3.3.4" />
<PackageVersion Include="Reqnroll.NUnit" Version="3.3.4" />
<PackageVersion Include="Reqnroll.Tools.MsBuild.Generation" Version="3.3.4" />
<PackageVersion Include="Sentry.AspNetCore" Version="6.11.1" />
<PackageVersion Include="SharpCompress" Version="0.50.4" />
<PackageVersion Include="Sentry.AspNetCore" Version="6.12.1" />
<PackageVersion Include="SharpCompress" Version="0.50.5" />
<PackageVersion Include="Shouldly" Version="4.3.0" />
<PackageVersion Include="Shared" Version="2026.10.2" />
<PackageVersion Include="Shared.DomainDrivenDesign" Version="2026.10.2" />
<PackageVersion Include="Shared.EventStore" Version="2026.10.2" />
<PackageVersion Include="Shared.IntegrationTesting" Version="2026.10.2" />
<PackageVersion Include="Shared.Results" Version="2026.10.2" />
<PackageVersion Include="Shared.Results.Web" Version="2026.10.2" />
<PackageVersion Include="SecurityService.Client" Version="2026.9.3.198" />
<PackageVersion Include="SecurityService.IntegrationTesting.Helpers" Version="2026.9.3.198" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="10.2.3" />
<PackageVersion Include="Swashbuckle.AspNetCore.Annotations" Version="10.2.3" />
<PackageVersion Include="Shared" Version="2026.10.3" />
<PackageVersion Include="Shared.DomainDrivenDesign" Version="2026.10.3" />
<PackageVersion Include="Shared.EventStore" Version="2026.10.3" />
<PackageVersion Include="Shared.IntegrationTesting" Version="2026.10.3" />
<PackageVersion Include="Shared.Results" Version="2026.10.3" />
<PackageVersion Include="Shared.Results.Web" Version="2026.10.3" />
<PackageVersion Include="SecurityService.Client" Version="2026.9.3.199" />
<PackageVersion Include="SecurityService.IntegrationTesting.Helpers" Version="2026.9.3.199" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="10.3.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.Annotations" Version="10.3.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.Filters" Version="10.0.1" />
<PackageVersion Include="Swashbuckle.AspNetCore.Swagger" Version="10.2.3" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerGen" Version="10.2.3" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.2.3" />
<PackageVersion Include="System.IO.Abstractions" Version="22.2.0" />
<PackageVersion Include="System.IO.Abstractions.TestingHelpers" Version="22.2.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.Swagger" Version="10.3.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerGen" Version="10.3.0" />
<PackageVersion Include="Swashbuckle.AspNetCore.SwaggerUI" Version="10.3.0" />
<PackageVersion Include="System.IO.Abstractions" Version="22.3.0" />
<PackageVersion Include="System.IO.Abstractions.TestingHelpers" Version="22.3.0" />
<PackageVersion Include="TransactionProcessor.Client" Version="2026.9.3.409" />
<PackageVersion Include="TransactionProcessor.Database" Version="2026.9.3.409" />
<PackageVersion Include="TransactionProcessor.IntegrationTesting.Helpers" Version="2026.9.3.409" />
<PackageVersion Include="xunit.v3" Version="4.0.1" />
<PackageVersion Include="xunit.runner.visualstudio" Version="4.0.0" />
<PackageVersion Include="xunit.v3" Version="4.0.2" />
<PackageVersion Include="xunit.runner.visualstudio" Version="4.0.1" />
</ItemGroup>
</Project>
12 changes: 11 additions & 1 deletion FIleProcessor.Models/FileDetails.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,16 @@ public class FileDetails
/// </value>
public Boolean ProcessingCompleted { get; set; }

public Boolean IsRejected { get; set; }

public String RejectionCode { get; set; }

public String RejectionReason { get; set; }

public Int32 ObservedDetailRecordCount { get; set; }

public Int32 MaximumDetailRecordCount { get; set; }

/// <summary>
/// Gets or sets the processing summary.
/// </summary>
Expand All @@ -100,4 +110,4 @@ public class FileDetails

#endregion
}
}
}
48 changes: 48 additions & 0 deletions FileProcessor.BusinessLogic.Tests/FileProcessorDomainServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,54 @@ public async Task FileRequestHandler_ProcessUploadedFileRequest_RequestIsHandled
result.IsSuccess.ShouldBeTrue();
}

[Fact]
public async Task ProcessUploadedFile_WhenFileExceedsMaximumDetailRecords_RejectsWithoutAddingLines()
{
this.FileProcessorManager.GetFileProfile(Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.FileProfileSafaricom);
this.FileFormatHandler.FileLineCanBeIgnored(Arg<String>.Any()).Returns((String line) => String.IsNullOrWhiteSpace(line) || line.StartsWith("H") || line.StartsWith("T"));
FileAggregate createdAggregate = TestData.GetCreatedFileAggregate();
this.FileAggregateRepository.GetLatestVersion(Arg<Guid>.Any(), Arg<CancellationToken>.Any())
.ReturnsAsync(Result.Success(TestData.GetEmptyFileAggregate()))
.Then().ReturnsAsync(Result.Success(createdAggregate));
this.FileAggregateRepository.SaveChanges(Arg<FileAggregate>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success());
this.SecurityServiceClient.GetToken(Arg<String>.Any(), Arg<String>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(Result.Success(TestData.TokenResponse()));
this.TransactionProcessorClient.GetOperators(Arg<String>.Any(), Arg<Guid>.Any(), Arg<CancellationToken>.Any()).ReturnsAsync(TestData.OperatorList);

StringBuilder content = new("H,20210508,\n");
for (Int32 row = 1; row <= 1001; row++)
content.Append($"D,0777777{row:0000},{10 + ((row - 1) % 240)}\n");
content.Append("T,1001,");
this.FileSystem.AddFile(TestData.FilePathWithName, new MockFileData(content.ToString()));

Result result = await this.FileProcessorDomainService.ProcessUploadedFile(TestData.ProcessUploadedFileCommand, CancellationToken.None);

result.IsSuccess.ShouldBeTrue();
createdAggregate.GetFile().IsRejected.ShouldBeTrue();
createdAggregate.GetFile().ObservedDetailRecordCount.ShouldBe(1001);
createdAggregate.GetFile().MaximumDetailRecordCount.ShouldBe(1000);
createdAggregate.GetFile().FileLines.ShouldBeEmpty();
this.FileSystem.File.Exists($"{TestData.FileProfileSafaricom.FailedDirectory}/{Path.GetFileName(TestData.FilePathWithName)}").ShouldBeTrue();
}

[Theory]
[InlineData("0")]
[InlineData("-1")]
[InlineData("not-a-number")]
public void ResolveMaximumDetailRecordCount_WhenConfiguredValueIsInvalid_Throws(String configuredValue)
{
IConfigurationRoot invalidConfiguration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<String, String>
{
["AppSettings:MaximumDetailRecordCount"] = configuredValue
})
.Build();
ConfigurationReader.Initialise(invalidConfiguration);

Should.Throw<InvalidOperationException>(() => FileProcessorDomainService.ResolveMaximumDetailRecordCount());

ConfigurationReader.Initialise(new ConfigurationBuilder().AddInMemoryCollection(TestData.DefaultAppSettings).Build());
}

[Fact]
public async Task ProcessUploadedFile_WhenOperatorLookupFails_MovesAcceptedFileToFailedDirectory()
{
Expand Down
52 changes: 52 additions & 0 deletions FileProcessor.BusinessLogic/Services/FileProcessorDomainService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ Task<Result> ProcessTransactionForFileLine(FileCommands.ProcessTransactionForFil

public class FileProcessorDomainService : IFileProcessorDomainService
{
private const Int32 DefaultMaximumDetailRecordCount = 1000;
private const String MaximumDetailRecordsExceededCode = "MaximumDetailRecordsExceeded";

private readonly IFileProcessorManager FileProcessorManager;

private readonly IAggregateRepository<FileImportLogAggregate, DomainEvent> FileImportLogAggregateRepository;
Expand All @@ -67,6 +70,8 @@ public class FileProcessorDomainService : IFileProcessorDomainService

private readonly IFileSystem FileSystem;

private readonly Int32 MaximumDetailRecordCount;

public FileProcessorDomainService(IFileProcessorManager fileProcessorManager,
IAggregateRepository<FileImportLogAggregate, DomainEvent> fileImportLogAggregateRepository,
IAggregateRepository<FileAggregate, DomainEvent> fileAggregateRepository,
Expand All @@ -81,6 +86,19 @@ public FileProcessorDomainService(IFileProcessorManager fileProcessorManager,
this.SecurityServiceClient = securityServiceClient;
this.FileFormatHandlerResolver = fileFormatHandlerResolver;
this.FileSystem = fileSystem;
this.MaximumDetailRecordCount = ResolveMaximumDetailRecordCount();
}

public static Int32 ResolveMaximumDetailRecordCount()
{
String configuredMaximumDetailRecordCount = ConfigurationReader.GetValue("AppSettings", "MaximumDetailRecordCount");
if (String.IsNullOrWhiteSpace(configuredMaximumDetailRecordCount))
return DefaultMaximumDetailRecordCount;

if (Int32.TryParse(configuredMaximumDetailRecordCount, out Int32 maximumDetailRecordCount) == false || maximumDetailRecordCount <= 0)
throw new InvalidOperationException("AppSettings:MaximumDetailRecordCount must be a positive integer when configured");

return maximumDetailRecordCount;
}

private async Task<Result> ApplyFileUpdates(Func<FileAggregate, Task<Result>> action,
Expand Down Expand Up @@ -754,8 +772,15 @@ private async Task<Result> ProcessFile(Guid fileId,
inProgressFile = fileInfo;
}

Boolean fileWasRejected = false;
Result result = await ApplyFileUpdates(async (FileAggregate fileAggregate) => {

if (fileAggregate.GetFile().IsRejected)
{
fileWasRejected = true;
return Result.Success();
}

String fileContent = null;
//Open file for Read\Write
using (Stream fs = inProgressFile.Open(FileMode.OpenOrCreate, FileAccess.Read, FileShare.Read)) {
Expand All @@ -769,6 +794,26 @@ private async Task<Result> ProcessFile(Guid fileId,
if (String.IsNullOrEmpty(fileContent) == false) {
String[] fileLines = fileContent.Split(fileProfile.LineTerminator);

Int32 detailRecordCount = fileLines.Count(fileLine =>
{
String trimmedLine = fileLine.Trim();
return String.IsNullOrEmpty(trimmedLine) == false && this.FileLineCanBeIgnored(trimmedLine, fileProfile.FileFormatHandler) == false;
});

if (detailRecordCount > this.MaximumDetailRecordCount)
{
Result rejectionResult = fileAggregate.RejectFile(detailRecordCount,
this.MaximumDetailRecordCount,
MaximumDetailRecordsExceededCode,
$"File contains {detailRecordCount} detail records; the maximum permitted is {this.MaximumDetailRecordCount}",
DateTime.UtcNow);
if (rejectionResult.IsFailed)
return rejectionResult;

fileWasRejected = true;
return Result.Success();
}

foreach (String fileLine in fileLines) {
Result stateResult = fileAggregate.AddFileLine(fileLine.Trim());
if (stateResult.IsFailed)
Expand All @@ -780,6 +825,13 @@ private async Task<Result> ProcessFile(Guid fileId,
}, fileId, cancellationToken);

if (result.IsSuccess) {
if (fileWasRejected)
{
Logger.LogWarning($"About to move rejected file {inProgressFile.Name} to [{fileProfile.FailedDirectory}]");
inProgressFile.MoveTo($"{fileProfile.FailedDirectory}/{inProgressFile.Name}", true);
return Result.Success();
}

Logger.LogInformation($"About to move file {inProgressFile.Name} to [{fileProfile.ProcessedDirectory}]");

// Move file now
Expand Down
12 changes: 11 additions & 1 deletion FileProcessor.DataTransferObjects/Responses/FileDetails.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,16 @@ public class FileDetails

public Boolean ProcessingCompleted { get; set; }

public Boolean IsRejected { get; set; }

public String RejectionCode { get; set; }

public String RejectionReason { get; set; }

public Int32 ObservedDetailRecordCount { get; set; }

public Int32 MaximumDetailRecordCount { get; set; }

public Guid EstateId { get; set; }

public Guid UserId { get; set; }
Expand All @@ -31,4 +41,4 @@ public class FileDetails

public FileProcessingSummary ProcessingSummary { get; set; }
}
}
}
9 changes: 9 additions & 0 deletions FileProcessor.File.DomainEvents/FileAggregateDomainEvents.cs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,13 @@ public record FileCreatedEvent(Guid FileId,
String FileLocation,
DateTime FileReceivedDateTime,
Guid OperatorId) : DomainEvent(FileId, Guid.NewGuid());

public record FileRejectedEvent(Guid FileId,
Guid EstateId,
Guid MerchantId,
String RejectionCode,
String RejectionReason,
Int32 ObservedDetailRecordCount,
Int32 MaximumDetailRecordCount,
DateTime RejectedDateTime) : DomainEvent(FileId, Guid.NewGuid());
}
Loading
Loading