Bound log ingestion and upload request sizes

CentralLogging.Api/Endpoints/LogEndpoints.cs

@@ -35,37 +35,48 @@ public static async Task<IResult> IngestAsync(IReadOnlyList<LogEventInput> event
 
     public static async Task<IResult> UploadAsync(IFormFile? file,
                                                    CentralLogsDbContext database,
+                                                   Microsoft.Extensions.Options.IOptions<LogIngestionOptions> options,
                                                    CancellationToken cancellationToken)
     {
         if (file is null)
             return Results.Problem("Choose a log file to upload.", statusCode: StatusCodes.Status400BadRequest);
+        if (file.Length > options.Value.MaximumUploadBytes)
+            return Results.Problem($"The uploaded file exceeds the {options.Value.MaximumUploadBytes} byte limit.",
+                statusCode: StatusCodes.Status413PayloadTooLarge);
 
         Int32 accepted = 0;
         Int32 duplicates = 0;
         DateTimeOffset uploadTimestamp = DateTimeOffset.UtcNow;
         List<LogEventInput> batch = new(MaximumBatchSize);
 
         await using Stream stream = file.OpenReadStream();
-        using StreamReader reader = new(stream);
         String? line;
         Int32 lineNumber = 0;
-        while ((line = await reader.ReadLineAsync(cancellationToken)) is not null)
+        BoundedLineReader lineReader = new(stream, options.Value.MaximumLineBytes);
+        try
         {
-            lineNumber++;
-            if (String.IsNullOrWhiteSpace(line))
-                continue;
-
-            LogEventInput input = ParseUploadLine(line, lineNumber, uploadTimestamp);
-            batch.Add(input);
-            if (batch.Count == MaximumBatchSize)
+            while ((line = await lineReader.ReadLineAsync(cancellationToken)) is not null)
             {
-                IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken);
-                database.ChangeTracker.Clear();
-                accepted += counts.Accepted;
-                duplicates += counts.Duplicates;
-                batch.Clear();
+                lineNumber++;
+                if (String.IsNullOrWhiteSpace(line))
+                    continue;
+
+                LogEventInput input = ParseUploadLine(line, lineNumber, uploadTimestamp);
+                batch.Add(input);
+                if (batch.Count == MaximumBatchSize)
+                {
+                    IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken);
+                    database.ChangeTracker.Clear();
+                    accepted += counts.Accepted;
+                    duplicates += counts.Duplicates;
+                    batch.Clear();
+                }
             }
         }
+        catch (UploadLimitExceededException exception)
+        {
+            return Results.Problem(exception.Message, statusCode: StatusCodes.Status413PayloadTooLarge);
+        }
 
         if (batch.Count > 0)
         {
@@ -78,6 +89,51 @@ public static async Task<IResult> UploadAsync(IFormFile? file,
         return Results.Ok(new { accepted, duplicates, invalidLines = 0 });
     }
 
+    private sealed class BoundedLineReader(Stream stream, Int32 maximumLineBytes)
+    {
+        private readonly Byte[] Buffer = new Byte[8192];
+        private Int32 Offset;
+        private Int32 Count;
+        private Boolean IsFirstLine = true;
+
+        public async Task<String?> ReadLineAsync(CancellationToken cancellationToken)
+        {
+            using MemoryStream line = new(Math.Min(maximumLineBytes, 8192));
+            while (true)
+            {
+                if (this.Offset == this.Count)
+                {
+                    this.Count = await stream.ReadAsync(this.Buffer, cancellationToken);
+                    this.Offset = 0;
+                    if (this.Count == 0)
+                    {
+                        if (line.Length == 0)
+                            return null;
+                        break;
+                    }
+                }
+
+                Byte value = this.Buffer[this.Offset++];
+                if (value == (Byte)'\n')
+                    break;
+                if (line.Length == maximumLineBytes)
+                    throw new UploadLimitExceededException(maximumLineBytes);
+                line.WriteByte(value);
+            }
+
+            Byte[] bytes = line.ToArray();
+            Int32 length = bytes.Length > 0 && bytes[^1] == (Byte)'\r' ? bytes.Length - 1 : bytes.Length;
+            String result = Encoding.UTF8.GetString(bytes, 0, length);
+            if (this.IsFirstLine && result.StartsWith('\uFEFF'))
+                result = result[1..];
+            this.IsFirstLine = false;
+            return result;
+        }
+    }
+
+    private sealed class UploadLimitExceededException(Int32 maximumLineBytes)
+        : Exception($"An uploaded line exceeds the {maximumLineBytes} byte limit.");
+
     private static LogEventInput ParseUploadLine(String line, Int32 lineNumber, DateTimeOffset uploadTimestamp)
     {
         try

CentralLogging.Api/Endpoints/LogIngestionOptions.cs

@@ -0,0 +1,10 @@
+namespace CentralLogging.Api.Endpoints;
+
+public sealed class LogIngestionOptions
+{
+    public static String SectionName => "LogIngestion";
+
+    public Int64 MaximumRequestBytes { get; set; } = 10 * 1024 * 1024;
+    public Int64 MaximumUploadBytes { get; set; } = 100 * 1024 * 1024;
+    public Int32 MaximumLineBytes { get; set; } = 1024 * 1024;
+}

CentralLogging.Api/Program.cs

@@ -4,6 +4,8 @@
 using CentralLogging.Api.Endpoints;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.AspNetCore.Http.Features;
+using Microsoft.Extensions.Options;
 
 WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
 
@@ -17,7 +19,17 @@
 String apiKey = configuredApiKey;
 
 builder.Services.AddDbContext<CentralLogsDbContext>(options => options.UseSqlServer(connectionString));
+builder.Services.AddOptions<LogIngestionOptions>()
+    .Bind(builder.Configuration.GetSection(LogIngestionOptions.SectionName))
+    .Validate(options => options.MaximumRequestBytes > 0 && options.MaximumUploadBytes > 0 && options.MaximumLineBytes > 0,
+        "Log ingestion size limits must be greater than zero.")
+    .ValidateOnStart();
+builder.Services.Configure<FormOptions>(options =>
+{
+    options.MultipartBodyLengthLimit = (builder.Configuration.GetValue<Int64?>("LogIngestion:MaximumUploadBytes") ?? 100 * 1024 * 1024) + 64 * 1024;
+});
 WebApplication app = builder.Build();
+LogIngestionOptions ingestionOptions = app.Services.GetRequiredService<IOptions<LogIngestionOptions>>().Value;
 
 await using (AsyncServiceScope scope = app.Services.CreateAsyncScope())
 {
@@ -36,6 +48,18 @@
         return;
     }
 
+    if (HttpMethods.IsPost(context.Request.Method) && context.Request.Path.StartsWithSegments("/api/logs"))
+    {
+        IHttpMaxRequestBodySizeFeature? sizeFeature = context.Features.Get<IHttpMaxRequestBodySizeFeature>();
+        if (sizeFeature is { IsReadOnly: false })
+        {
+            Int64 maximumBytes = context.Request.Path.StartsWithSegments("/api/logs/upload")
+                ? ingestionOptions.MaximumUploadBytes + 64 * 1024
+                : ingestionOptions.MaximumRequestBytes;
+            sizeFeature.MaxRequestBodySize = maximumBytes;
+        }
+    }
+
     await next(context);
 });
 

CentralLogging.Api/README.md

@@ -15,6 +15,8 @@ This API is the first receiving and query component for the central logging desi
 
 3. Requests to `/api/logs` must send the configured secret in the `X-Log-Api-Key` header. The browser UI prompts for this key at runtime, keeps it in memory only, and sends it with API requests.
 
+Ingestion size limits are configured under `LogIngestion` and can be overridden per deployment with environment variables. Defaults are `MaximumRequestBytes` = 10 MiB for JSON ingestion requests, `MaximumUploadBytes` = 100 MiB for the uploaded file, and `MaximumLineBytes` = 1 MiB for an individual uploaded NDJSON line. Multipart framing is allowed an additional 64 KiB. For example, set `LogIngestion__MaximumUploadBytes=209715200` to allow up to 200 MiB files. Upload lines and files beyond their limits receive HTTP 413; JSON requests beyond the request body limit are rejected by the server before JSON binding.
+
 Open `http://localhost:<port>/` to use the search UI. It provides time, service, tenant, correlation ID, and service-specific structured-field filters. `GET /health/ready` checks database connectivity. `POST /api/logs` accepts a JSON array of 1 to 500 events and returns the accepted and duplicate counts. Reposting an `eventId` is safe. `GET /api/logs/services` lists service names seen in stored events.
 
 The UI can upload an existing newline-delimited JSON (`.json` or `.jsonl`) log file. Every nonblank line is retained, including malformed JSON and records missing fields. The API stores the original line in `rawLog`, fills unavailable required fields with safe fallbacks, and ingests in batches of up to 500. The response includes `accepted`, `duplicates`, and `invalidLines` (zero when persistence succeeds). Uploads use `POST /api/logs/upload` as `multipart/form-data` with a `file` field and the same `X-Log-Api-Key` header. If a later batch fails, earlier batches may already be stored; retrying the file is safe because event IDs are deduplicated. Search results are database-wide and can include previously ingested events; after an upload, run a new search to reload the results.

CentralLogging.Api/appsettings.json

@@ -3,7 +3,10 @@
     "CentralLogs": "Server=localhost;Database=CentralLogs;Integrated Security=True;Encrypt=True;TrustServerCertificate=True"
   },
   "LogIngestion": {
-    "ApiKey": "TestApiKey"
+    "ApiKey": "TestApiKey",
+    "MaximumRequestBytes": 10485760,
+    "MaximumUploadBytes": 104857600,
+    "MaximumLineBytes": 1048576
   },
   "Logging": {
     "LogLevel": {

CentralLogging.Tests/Api/UploadIntegrationTests.cs

@@ -54,6 +54,30 @@ public async Task Upload_retains_malformed_and_incomplete_lines_and_deduplicates
         (await fixture.ReadDatabaseAsync(database => database.LogEvents.CountAsync())).ShouldBe(2);
     }
 
+    [SqlServerFact]
+    public async Task Upload_strips_a_UTF8_bom_before_parsing_and_preserves_event_identity()
+    {
+        await fixture.ResetDatabaseAsync();
+        Guid eventId = Guid.NewGuid();
+        String contents = $"\uFEFF{ApiTestData.LogLine(eventId, "Payments", "bom event")}\n";
+
+        using HttpResponseMessage response = await SendUploadAsync(contents);
+        response.StatusCode.ShouldBe(System.Net.HttpStatusCode.OK);
+        using JsonDocument body = await ApiTestData.ReadJsonAsync(response);
+        body.RootElement.GetProperty("accepted").GetInt32().ShouldBe(1);
+
+        var stored = await fixture.ReadDatabaseAsync(database => database.LogEvents
+            .Select(log => new { log.EventId, log.ServiceName, log.Message })
+            .SingleAsync());
+        stored.EventId.ShouldBe(eventId);
+        stored.ServiceName.ShouldBe("Payments");
+        stored.Message.ShouldBe("bom event");
+
+        using HttpResponseMessage replayResponse = await SendUploadAsync(contents);
+        using JsonDocument replayBody = await ApiTestData.ReadJsonAsync(replayResponse);
+        replayBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(1);
+    }
+
     [SqlServerFact]
     public async Task Upload_keeps_completed_batches_when_a_later_batch_fails()
     {
@@ -84,7 +108,8 @@ await fixture.ReadDatabaseAsync(async database =>
             {
                 await fixture.ReadDatabaseAsync(async database =>
                 {
-                    await LogEndpoints.UploadAsync(file, database, CancellationToken.None);
+                    await LogEndpoints.UploadAsync(file, database,
+                        Microsoft.Extensions.Options.Options.Create(new LogIngestionOptions()), CancellationToken.None);
                     return true;
                 });
             });