From 3c12c90ae29e61a2058ed48324fc82d36c126d93 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Mon, 9 Nov 2026 19:45:11 +0000 Subject: [PATCH 1/2] Bound log ingestion and upload sizes --- CentralLogging.Api/Endpoints/LogEndpoints.cs | 79 +++++++++++++++---- .../Endpoints/LogIngestionOptions.cs | 10 +++ CentralLogging.Api/Program.cs | 24 ++++++ CentralLogging.Api/README.md | 2 + CentralLogging.Api/appsettings.json | 5 +- 5 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 CentralLogging.Api/Endpoints/LogIngestionOptions.cs diff --git a/CentralLogging.Api/Endpoints/LogEndpoints.cs b/CentralLogging.Api/Endpoints/LogEndpoints.cs index 4592f2f..11ce972 100644 --- a/CentralLogging.Api/Endpoints/LogEndpoints.cs +++ b/CentralLogging.Api/Endpoints/LogEndpoints.cs @@ -35,10 +35,14 @@ public static async Task IngestAsync(IReadOnlyList event public static async Task UploadAsync(IFormFile? file, CentralLogsDbContext database, + Microsoft.Extensions.Options.IOptions options, CancellationToken cancellationToken) { if (file is null) return Results.Problem("Choose a log file to upload.", statusCode: StatusCodes.Status400BadRequest); + if (file.Length > options.Value.MaximumUploadBytes) + return Results.Problem($"The uploaded file exceeds the {options.Value.MaximumUploadBytes} byte limit.", + statusCode: StatusCodes.Status413PayloadTooLarge); Int32 accepted = 0; Int32 duplicates = 0; @@ -46,26 +50,33 @@ public static async Task UploadAsync(IFormFile? file, List batch = new(MaximumBatchSize); await using Stream stream = file.OpenReadStream(); - using StreamReader reader = new(stream); String? line; Int32 lineNumber = 0; - while ((line = await reader.ReadLineAsync(cancellationToken)) is not null) + BoundedLineReader lineReader = new(stream, options.Value.MaximumLineBytes); + try { - lineNumber++; - if (String.IsNullOrWhiteSpace(line)) - continue; - - LogEventInput input = ParseUploadLine(line, lineNumber, uploadTimestamp); - batch.Add(input); - if (batch.Count == MaximumBatchSize) + while ((line = await lineReader.ReadLineAsync(cancellationToken)) is not null) { - IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken); - database.ChangeTracker.Clear(); - accepted += counts.Accepted; - duplicates += counts.Duplicates; - batch.Clear(); + lineNumber++; + if (String.IsNullOrWhiteSpace(line)) + continue; + + LogEventInput input = ParseUploadLine(line, lineNumber, uploadTimestamp); + batch.Add(input); + if (batch.Count == MaximumBatchSize) + { + IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken); + database.ChangeTracker.Clear(); + accepted += counts.Accepted; + duplicates += counts.Duplicates; + batch.Clear(); + } } } + catch (UploadLimitExceededException exception) + { + return Results.Problem(exception.Message, statusCode: StatusCodes.Status413PayloadTooLarge); + } if (batch.Count > 0) { @@ -78,6 +89,46 @@ public static async Task UploadAsync(IFormFile? file, return Results.Ok(new { accepted, duplicates, invalidLines = 0 }); } + private sealed class BoundedLineReader(Stream stream, Int32 maximumLineBytes) + { + private readonly Byte[] Buffer = new Byte[8192]; + private Int32 Offset; + private Int32 Count; + + public async Task ReadLineAsync(CancellationToken cancellationToken) + { + using MemoryStream line = new(Math.Min(maximumLineBytes, 8192)); + while (true) + { + if (this.Offset == this.Count) + { + this.Count = await stream.ReadAsync(this.Buffer, cancellationToken); + this.Offset = 0; + if (this.Count == 0) + { + if (line.Length == 0) + return null; + break; + } + } + + Byte value = this.Buffer[this.Offset++]; + if (value == (Byte)'\n') + break; + if (line.Length == maximumLineBytes) + throw new UploadLimitExceededException(maximumLineBytes); + line.WriteByte(value); + } + + Byte[] bytes = line.ToArray(); + Int32 length = bytes.Length > 0 && bytes[^1] == (Byte)'\r' ? bytes.Length - 1 : bytes.Length; + return Encoding.UTF8.GetString(bytes, 0, length); + } + } + + private sealed class UploadLimitExceededException(Int32 maximumLineBytes) + : Exception($"An uploaded line exceeds the {maximumLineBytes} byte limit."); + private static LogEventInput ParseUploadLine(String line, Int32 lineNumber, DateTimeOffset uploadTimestamp) { try diff --git a/CentralLogging.Api/Endpoints/LogIngestionOptions.cs b/CentralLogging.Api/Endpoints/LogIngestionOptions.cs new file mode 100644 index 0000000..4e0a057 --- /dev/null +++ b/CentralLogging.Api/Endpoints/LogIngestionOptions.cs @@ -0,0 +1,10 @@ +namespace CentralLogging.Api.Endpoints; + +public sealed class LogIngestionOptions +{ + public const String SectionName = "LogIngestion"; + + public Int64 MaximumRequestBytes { get; set; } = 10 * 1024 * 1024; + public Int64 MaximumUploadBytes { get; set; } = 100 * 1024 * 1024; + public Int32 MaximumLineBytes { get; set; } = 1024 * 1024; +} diff --git a/CentralLogging.Api/Program.cs b/CentralLogging.Api/Program.cs index ea18c0c..2f1218d 100644 --- a/CentralLogging.Api/Program.cs +++ b/CentralLogging.Api/Program.cs @@ -4,6 +4,8 @@ using CentralLogging.Api.Endpoints; using Microsoft.Extensions.DependencyInjection; using Microsoft.EntityFrameworkCore; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.Extensions.Options; WebApplicationBuilder builder = WebApplication.CreateBuilder(args); @@ -17,7 +19,17 @@ String apiKey = configuredApiKey; builder.Services.AddDbContext(options => options.UseSqlServer(connectionString)); +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(LogIngestionOptions.SectionName)) + .Validate(options => options.MaximumRequestBytes > 0 && options.MaximumUploadBytes > 0 && options.MaximumLineBytes > 0, + "Log ingestion size limits must be greater than zero.") + .ValidateOnStart(); +builder.Services.Configure(options => +{ + options.MultipartBodyLengthLimit = (builder.Configuration.GetValue("LogIngestion:MaximumUploadBytes") ?? 100 * 1024 * 1024) + 64 * 1024; +}); WebApplication app = builder.Build(); +LogIngestionOptions ingestionOptions = app.Services.GetRequiredService>().Value; await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) { @@ -36,6 +48,18 @@ return; } + if (HttpMethods.IsPost(context.Request.Method) && context.Request.Path.StartsWithSegments("/api/logs")) + { + IHttpMaxRequestBodySizeFeature? sizeFeature = context.Features.Get(); + if (sizeFeature is { IsReadOnly: false }) + { + Int64 maximumBytes = context.Request.Path.StartsWithSegments("/api/logs/upload") + ? ingestionOptions.MaximumUploadBytes + 64 * 1024 + : ingestionOptions.MaximumRequestBytes; + sizeFeature.MaxRequestBodySize = maximumBytes; + } + } + await next(context); }); diff --git a/CentralLogging.Api/README.md b/CentralLogging.Api/README.md index e7a7c2c..835d455 100644 --- a/CentralLogging.Api/README.md +++ b/CentralLogging.Api/README.md @@ -15,6 +15,8 @@ This API is the first receiving and query component for the central logging desi 3. Requests to `/api/logs` must send the configured secret in the `X-Log-Api-Key` header. The browser UI prompts for this key at runtime, keeps it in memory only, and sends it with API requests. +Ingestion size limits are configured under `LogIngestion` and can be overridden per deployment with environment variables. Defaults are `MaximumRequestBytes` = 10 MiB for JSON ingestion requests, `MaximumUploadBytes` = 100 MiB for the uploaded file, and `MaximumLineBytes` = 1 MiB for an individual uploaded NDJSON line. Multipart framing is allowed an additional 64 KiB. For example, set `LogIngestion__MaximumUploadBytes=209715200` to allow up to 200 MiB files. Upload lines and files beyond their limits receive HTTP 413; JSON requests beyond the request body limit are rejected by the server before JSON binding. + Open `http://localhost:/` to use the search UI. It provides time, service, tenant, correlation ID, and service-specific structured-field filters. `GET /health/ready` checks database connectivity. `POST /api/logs` accepts a JSON array of 1 to 500 events and returns the accepted and duplicate counts. Reposting an `eventId` is safe. `GET /api/logs/services` lists service names seen in stored events. The UI can upload an existing newline-delimited JSON (`.json` or `.jsonl`) log file. Every nonblank line is retained, including malformed JSON and records missing fields. The API stores the original line in `rawLog`, fills unavailable required fields with safe fallbacks, and ingests in batches of up to 500. The response includes `accepted`, `duplicates`, and `invalidLines` (zero when persistence succeeds). Uploads use `POST /api/logs/upload` as `multipart/form-data` with a `file` field and the same `X-Log-Api-Key` header. If a later batch fails, earlier batches may already be stored; retrying the file is safe because event IDs are deduplicated. Search results are database-wide and can include previously ingested events; after an upload, run a new search to reload the results. diff --git a/CentralLogging.Api/appsettings.json b/CentralLogging.Api/appsettings.json index fc1680d..718b069 100644 --- a/CentralLogging.Api/appsettings.json +++ b/CentralLogging.Api/appsettings.json @@ -3,7 +3,10 @@ "CentralLogs": "Server=localhost;Database=CentralLogs;Integrated Security=True;Encrypt=True;TrustServerCertificate=True" }, "LogIngestion": { - "ApiKey": "TestApiKey" + "ApiKey": "TestApiKey", + "MaximumRequestBytes": 10485760, + "MaximumUploadBytes": 104857600, + "MaximumLineBytes": 1048576 }, "Logging": { "LogLevel": { From 1d40f747564524322b2c3cbd20289afcc5935c87 Mon Sep 17 00:00:00 2001 From: StuartFerguson Date: Mon, 9 Nov 2026 20:12:08 +0000 Subject: [PATCH 2/2] Fix upload BOM handling and CI compile error --- CentralLogging.Api/Endpoints/LogEndpoints.cs | 7 ++++- .../Endpoints/LogIngestionOptions.cs | 2 +- .../Api/UploadIntegrationTests.cs | 27 ++++++++++++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/CentralLogging.Api/Endpoints/LogEndpoints.cs b/CentralLogging.Api/Endpoints/LogEndpoints.cs index 11ce972..bf44950 100644 --- a/CentralLogging.Api/Endpoints/LogEndpoints.cs +++ b/CentralLogging.Api/Endpoints/LogEndpoints.cs @@ -94,6 +94,7 @@ private sealed class BoundedLineReader(Stream stream, Int32 maximumLineBytes) private readonly Byte[] Buffer = new Byte[8192]; private Int32 Offset; private Int32 Count; + private Boolean IsFirstLine = true; public async Task ReadLineAsync(CancellationToken cancellationToken) { @@ -122,7 +123,11 @@ private sealed class BoundedLineReader(Stream stream, Int32 maximumLineBytes) Byte[] bytes = line.ToArray(); Int32 length = bytes.Length > 0 && bytes[^1] == (Byte)'\r' ? bytes.Length - 1 : bytes.Length; - return Encoding.UTF8.GetString(bytes, 0, length); + String result = Encoding.UTF8.GetString(bytes, 0, length); + if (this.IsFirstLine && result.StartsWith('\uFEFF')) + result = result[1..]; + this.IsFirstLine = false; + return result; } } diff --git a/CentralLogging.Api/Endpoints/LogIngestionOptions.cs b/CentralLogging.Api/Endpoints/LogIngestionOptions.cs index 4e0a057..f6d67ec 100644 --- a/CentralLogging.Api/Endpoints/LogIngestionOptions.cs +++ b/CentralLogging.Api/Endpoints/LogIngestionOptions.cs @@ -2,7 +2,7 @@ namespace CentralLogging.Api.Endpoints; public sealed class LogIngestionOptions { - public const String SectionName = "LogIngestion"; + public static String SectionName => "LogIngestion"; public Int64 MaximumRequestBytes { get; set; } = 10 * 1024 * 1024; public Int64 MaximumUploadBytes { get; set; } = 100 * 1024 * 1024; diff --git a/CentralLogging.Tests/Api/UploadIntegrationTests.cs b/CentralLogging.Tests/Api/UploadIntegrationTests.cs index 7284736..0367485 100644 --- a/CentralLogging.Tests/Api/UploadIntegrationTests.cs +++ b/CentralLogging.Tests/Api/UploadIntegrationTests.cs @@ -54,6 +54,30 @@ public async Task Upload_retains_malformed_and_incomplete_lines_and_deduplicates (await fixture.ReadDatabaseAsync(database => database.LogEvents.CountAsync())).ShouldBe(2); } + [SqlServerFact] + public async Task Upload_strips_a_UTF8_bom_before_parsing_and_preserves_event_identity() + { + await fixture.ResetDatabaseAsync(); + Guid eventId = Guid.NewGuid(); + String contents = $"\uFEFF{ApiTestData.LogLine(eventId, "Payments", "bom event")}\n"; + + using HttpResponseMessage response = await SendUploadAsync(contents); + response.StatusCode.ShouldBe(System.Net.HttpStatusCode.OK); + using JsonDocument body = await ApiTestData.ReadJsonAsync(response); + body.RootElement.GetProperty("accepted").GetInt32().ShouldBe(1); + + var stored = await fixture.ReadDatabaseAsync(database => database.LogEvents + .Select(log => new { log.EventId, log.ServiceName, log.Message }) + .SingleAsync()); + stored.EventId.ShouldBe(eventId); + stored.ServiceName.ShouldBe("Payments"); + stored.Message.ShouldBe("bom event"); + + using HttpResponseMessage replayResponse = await SendUploadAsync(contents); + using JsonDocument replayBody = await ApiTestData.ReadJsonAsync(replayResponse); + replayBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(1); + } + [SqlServerFact] public async Task Upload_keeps_completed_batches_when_a_later_batch_fails() { @@ -84,7 +108,8 @@ await fixture.ReadDatabaseAsync(async database => { await fixture.ReadDatabaseAsync(async database => { - await LogEndpoints.UploadAsync(file, database, CancellationToken.None); + await LogEndpoints.UploadAsync(file, database, + Microsoft.Extensions.Options.Options.Create(new LogIngestionOptions()), CancellationToken.None); return true; }); });