diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 050dfb1..602a58d 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -76,3 +76,25 @@ jobs: - name: Run Health Monitoring unit tests run: dotnet test HealthMonitoring/HealthMonitoring.Tests/HealthMonitoring.Tests.csproj --configuration Release --no-restore --filter "FullyQualifiedName!~ConfigurationTests" + + central-logging-tests: + name: "Run Central Logging Tests" + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Install NET 10 + uses: actions/setup-dotnet@v4.0.1 + with: + dotnet-version: '10.0.x' + + - name: Restore Central Logging solution + run: dotnet restore CentralLogging.sln --source https://api.nuget.org/v3/index.json + + - name: Build Central Logging solution + run: dotnet build CentralLogging.sln --configuration Release --no-restore + + - name: Test Central Logging solution + run: dotnet test CentralLogging.sln --configuration Release --no-build --no-restore diff --git a/CentralLogging.Api/CentralLogging.Api.csproj b/CentralLogging.Api/CentralLogging.Api.csproj new file mode 100644 index 0000000..ad16689 --- /dev/null +++ b/CentralLogging.Api/CentralLogging.Api.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + diff --git a/CentralLogging.Api/Data/CentralLogsDbContext.cs b/CentralLogging.Api/Data/CentralLogsDbContext.cs new file mode 100644 index 0000000..17dcb47 --- /dev/null +++ b/CentralLogging.Api/Data/CentralLogsDbContext.cs @@ -0,0 +1,59 @@ +using CentralLogging.Api.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace CentralLogging.Api.Data; + +public sealed class CentralLogsDbContext(DbContextOptions options) : DbContext(options) +{ + private const String MaxNvarcharColumnType = "nvarchar(max)"; + + public DbSet LogEvents => this.Set(); + public DbSet ServiceStructuredFields => this.Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + EntityTypeBuilder entity = modelBuilder.Entity(); + entity.ToTable("LogEvents", "dbo", table => + table.HasCheckConstraint("CK_LogEvents_PropertiesJson_IsJson", "ISJSON([PropertiesJson]) = 1")); + entity.HasKey(log => log.EventId); + entity.Property(log => log.TimestampUtc).HasColumnType("datetimeoffset(7)").IsRequired(); + entity.Property(log => log.LogDateUtc) + .HasColumnType("date") + .HasComputedColumnSql("CONVERT(date, [TimestampUtc])", stored: true); + entity.Property(log => log.Severity).HasMaxLength(16).IsRequired(); + entity.Property(log => log.ServiceName).HasMaxLength(200).IsRequired(); + entity.Property(log => log.TraceId).HasMaxLength(32); + entity.Property(log => log.SpanId).HasMaxLength(16); + entity.Property(log => log.RequestId).HasMaxLength(200); + entity.Property(log => log.Message).HasColumnType(MaxNvarcharColumnType).IsRequired(); + entity.Property(log => log.MessageTemplate).HasColumnType(MaxNvarcharColumnType); + entity.Property(log => log.Exception).HasColumnType(MaxNvarcharColumnType); + entity.Property(log => log.PropertiesJson).HasColumnType(MaxNvarcharColumnType).IsRequired(); + entity.HasIndex(log => log.TimestampUtc).IsDescending(); + entity.HasIndex(log => new { log.LogDateUtc, log.TimestampUtc }) + .IsDescending(false, true) + .HasDatabaseName("IX_LogEvents_LogDateUtc_TimestampUtc"); + entity.HasIndex(log => new { log.TenantId, log.TimestampUtc }).IsDescending(false, true); + entity.HasIndex(log => new { log.ServiceName, log.TimestampUtc }).IsDescending(false, true); + entity.HasIndex(log => log.CorrelationId).HasFilter("[CorrelationId] IS NOT NULL"); + + EntityTypeBuilder structuredField = modelBuilder.Entity(); + structuredField.ToTable("ServiceStructuredFields", "dbo"); + structuredField.HasKey(field => new { field.ServiceName, field.FieldName, field.JsonType }); + structuredField.Property(field => field.ServiceName) + .HasMaxLength(200) + .UseCollation("Latin1_General_100_BIN2") + .IsRequired() + .Metadata.SetValueComparer(new ValueComparer( + (left, right) => StringComparer.Ordinal.Equals(left, right), + value => StringComparer.Ordinal.GetHashCode(value), + value => value)); + structuredField.Property(field => field.FieldName).HasMaxLength(200).UseCollation("Latin1_General_100_BIN2").IsRequired(); + structuredField.Property(field => field.JsonType).HasMaxLength(16).IsRequired(); + structuredField.Property(field => field.FirstSeenUtc).HasColumnType("datetimeoffset(7)").IsRequired(); + structuredField.Property(field => field.LastSeenUtc).HasColumnType("datetimeoffset(7)").IsRequired(); + structuredField.Property(field => field.EventCount).IsRequired(); + } +} diff --git a/CentralLogging.Api/Data/CentralLogsDbContextFactory.cs b/CentralLogging.Api/Data/CentralLogsDbContextFactory.cs new file mode 100644 index 0000000..fb4532d --- /dev/null +++ b/CentralLogging.Api/Data/CentralLogsDbContextFactory.cs @@ -0,0 +1,26 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Configuration; + +namespace CentralLogging.Api.Data; + +public sealed class CentralLogsDbContextFactory : IDesignTimeDbContextFactory +{ + public CentralLogsDbContext CreateDbContext(String[] args) + { + IConfiguration configuration = new ConfigurationBuilder() + .SetBasePath(Path.GetDirectoryName(typeof(CentralLogsDbContextFactory).Assembly.Location)!) + .AddJsonFile("appsettings.json", optional: false) + .AddEnvironmentVariables() + .Build(); + + String connectionString = configuration.GetConnectionString("CentralLogs") + ?? throw new InvalidOperationException("Configure ConnectionStrings:CentralLogs for EF Core tooling."); + + DbContextOptions options = new DbContextOptionsBuilder() + .UseSqlServer(connectionString) + .Options; + + return new CentralLogsDbContext(options); + } +} diff --git a/CentralLogging.Api/Endpoints/LogEndpoints.cs b/CentralLogging.Api/Endpoints/LogEndpoints.cs new file mode 100644 index 0000000..4592f2f --- /dev/null +++ b/CentralLogging.Api/Endpoints/LogEndpoints.cs @@ -0,0 +1,633 @@ +using System.Data; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using CentralLogging.Api.Data; +using CentralLogging.Api.Models; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; + +namespace CentralLogging.Api.Endpoints; + +public static class LogEndpoints +{ + private const Int32 MaximumBatchSize = 500; + private const Int32 MaximumPageSize = 500; + private const String UnknownValue = "Unknown"; + private const String JsonNumberType = "number"; + private const String JsonBooleanType = "boolean"; + + public static async Task IngestAsync(IReadOnlyList events, + CentralLogsDbContext database, + CancellationToken cancellationToken) + { + if (events.Count is < 1 or > MaximumBatchSize) + return Results.Problem($"A batch must contain between 1 and {MaximumBatchSize} events.", statusCode: StatusCodes.Status400BadRequest); + + String? validationError = ValidateEvents(events); + if (validationError is not null) + return Results.Problem(validationError, statusCode: StatusCodes.Status400BadRequest); + + IngestionCounts counts = await IngestBatchAsync(events, database, cancellationToken); + return Results.Accepted(value: new { accepted = counts.Accepted, duplicates = counts.Duplicates }); + } + + public static async Task UploadAsync(IFormFile? file, + CentralLogsDbContext database, + CancellationToken cancellationToken) + { + if (file is null) + return Results.Problem("Choose a log file to upload.", statusCode: StatusCodes.Status400BadRequest); + + Int32 accepted = 0; + Int32 duplicates = 0; + DateTimeOffset uploadTimestamp = DateTimeOffset.UtcNow; + List batch = new(MaximumBatchSize); + + await using Stream stream = file.OpenReadStream(); + using StreamReader reader = new(stream); + String? line; + Int32 lineNumber = 0; + while ((line = await reader.ReadLineAsync(cancellationToken)) is not null) + { + lineNumber++; + if (String.IsNullOrWhiteSpace(line)) + continue; + + LogEventInput input = ParseUploadLine(line, lineNumber, uploadTimestamp); + batch.Add(input); + if (batch.Count == MaximumBatchSize) + { + IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken); + database.ChangeTracker.Clear(); + accepted += counts.Accepted; + duplicates += counts.Duplicates; + batch.Clear(); + } + } + + if (batch.Count > 0) + { + IngestionCounts counts = await IngestBatchAsync(batch, database, cancellationToken); + database.ChangeTracker.Clear(); + accepted += counts.Accepted; + duplicates += counts.Duplicates; + } + + return Results.Ok(new { accepted, duplicates, invalidLines = 0 }); + } + + private static LogEventInput ParseUploadLine(String line, Int32 lineNumber, DateTimeOffset uploadTimestamp) + { + try + { + using JsonDocument document = JsonDocument.Parse(line); + return document.RootElement.ValueKind == JsonValueKind.Object + ? ParseObjectUploadLine(document.RootElement, line, lineNumber, uploadTimestamp) + : CreateRawLineEvent(line, lineNumber, uploadTimestamp); + } + catch (JsonException) + { + return CreateRawLineEvent(line, lineNumber, uploadTimestamp); + } + } + + private static LogEventInput ParseObjectUploadLine(JsonElement root, + String line, + Int32 lineNumber, + DateTimeOffset uploadTimestamp) + { + UploadIdentity identity = ReadUploadIdentity(root); + return new LogEventInput + { + EventId = identity.EventId ?? CreateStableEventId(line, lineNumber), + TimestampUtc = identity.TimestampUtc ?? uploadTimestamp, + Severity = LimitLength(GetNonEmptyScalarText(FindUploadSeverity(root)) ?? UnknownValue, 16), + ServiceName = LimitLength(GetNonEmptyScalarText(FindUploadService(root)) ?? UnknownValue, 200), + CorrelationId = GetGuid(FindProperty(root, "correlationId")), + TenantId = GetGuid(FindProperty(root, "tenantId")), + MerchantId = GetGuid(FindProperty(root, "merchantId")), + TraceId = LimitLength(GetScalarText(FindProperty(root, "traceId")), 32), + SpanId = LimitLength(GetScalarText(FindProperty(root, "spanId")), 16), + RequestId = LimitLength(GetScalarText(FindProperty(root, "requestId")), 200), + Message = GetScalarText(FindUploadMessage(root)) ?? line, + MessageTemplate = GetScalarText(FindProperty(root, "messageTemplate")), + Exception = GetScalarText(FindProperty(root, "exception")), + AdditionalProperties = ReadUploadProperties(root, line, identity.EventId.HasValue) + }; + } + + private static UploadIdentity ReadUploadIdentity(JsonElement root) + { + Guid? eventId = GetGuid(FindProperty(root, "eventId")); + if (eventId == Guid.Empty) eventId = null; + JsonElement timestampElement = FindProperty(root, "timestampUtc"); + DateTimeOffset? timestamp = timestampElement.ValueKind == JsonValueKind.String && + timestampElement.TryGetDateTimeOffset(out DateTimeOffset parsedTimestamp) + ? parsedTimestamp + : null; + return new UploadIdentity(eventId, timestamp); + } + + private static JsonElement FindUploadSeverity(JsonElement root) => + FirstDefinedProperty(FindProperty(root, "severity"), FindProperty(root, "level")); + + private static JsonElement FindUploadService(JsonElement root) => + FirstDefinedProperty(FindProperty(root, "serviceName"), + FirstProperty(root, "loggerName", "sourceContext", "category", "fileName", "eventName", "serviceId")); + + private static JsonElement FindUploadMessage(JsonElement root) + { + JsonElement message = FindProperty(root, "message"); + return message.ValueKind == JsonValueKind.Undefined || IsEmptyString(message) + ? FindProperty(root, "messageTemplate") + : message; + } + + private static JsonElement FirstDefinedProperty(JsonElement first, JsonElement second) => + first.ValueKind == JsonValueKind.Undefined ? second : first; + + private static Dictionary ReadUploadProperties(JsonElement root, String line, Boolean hasEventId) + { + Dictionary properties = new(StringComparer.OrdinalIgnoreCase); + HashSet mappedFields = new(StringComparer.OrdinalIgnoreCase) + { + "eventId", "timestampUtc", "severity", "level", "serviceName", "loggerName", "sourceContext", + "category", "fileName", "eventName", "serviceId", "correlationId", "tenantId", "merchantId", + "traceId", "spanId", "requestId", "message", "messageTemplate", "exception" + }; + foreach (JsonProperty property in root.EnumerateObject()) + { + if (!mappedFields.Contains(property.Name) && property.Name.Length <= 200) + properties[property.Name] = property.Value.Clone(); + } + + if (!hasEventId) + { + JsonElement sourceEventId = FindProperty(root, "eventId"); + properties["sourceEventId"] = sourceEventId.ValueKind == JsonValueKind.Undefined + ? JsonSerializer.SerializeToElement((String?)null) + : sourceEventId.Clone(); + } + properties["rawLog"] = JsonSerializer.SerializeToElement(line); + return properties; + } + + private readonly record struct UploadIdentity(Guid? EventId, DateTimeOffset? TimestampUtc); + + private static LogEventInput CreateRawLineEvent(String line, Int32 lineNumber, DateTimeOffset uploadTimestamp) + { + Dictionary properties = new(StringComparer.OrdinalIgnoreCase) + { + ["rawLog"] = JsonSerializer.SerializeToElement(line) + }; + return new LogEventInput + { + EventId = CreateStableEventId(line, lineNumber), + TimestampUtc = uploadTimestamp, + Severity = UnknownValue, + ServiceName = UnknownValue, + Message = line, + AdditionalProperties = properties + }; + } + + private static Guid CreateStableEventId(String line, Int32 lineNumber) + { + Byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes($"{lineNumber}\n{line}")); + return new Guid(hash.AsSpan(0, 16)); + } + + private static JsonElement FindProperty(JsonElement root, String name) + { + foreach (JsonProperty property in root.EnumerateObject()) + if (String.Equals(property.Name, name, StringComparison.OrdinalIgnoreCase)) + return property.Value; + return default; + } + + private static JsonElement FirstProperty(JsonElement root, params String[] names) + { + foreach (String name in names) + { + JsonElement property = FindProperty(root, name); + if (property.ValueKind != JsonValueKind.Undefined) + return property; + } + return default; + } + + private static String? GetScalarText(JsonElement value) => value.ValueKind switch + { + JsonValueKind.String => value.GetString(), + JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False => value.GetRawText(), + JsonValueKind.Object or JsonValueKind.Array => value.GetRawText(), + _ => null + }; + + private static String? GetNonEmptyScalarText(JsonElement value) + { + String? text = GetScalarText(value); + return String.IsNullOrWhiteSpace(text) ? null : text; + } + + private static Guid? GetGuid(JsonElement value) => + value.ValueKind == JsonValueKind.String && value.TryGetGuid(out Guid guid) ? guid : null; + + private static Boolean IsEmptyString(JsonElement value) => + value.ValueKind == JsonValueKind.String && String.IsNullOrWhiteSpace(value.GetString()); + + private static String? LimitLength(String? value, Int32 maximumLength) => + value is null || value.Length <= maximumLength ? value : value[..maximumLength]; + + private static String? ValidateEvents(IReadOnlyList events) + { + if (events.Any(HasMissingRequiredFields)) + return "Every event must include eventId, timestampUtc, severity, serviceName, and message."; + + if (events.Any(HasFieldsExceedingSupportedLength)) + return "One or more event fields exceed the supported length."; + + if (events.Any(HasInvalidStructuredPropertyName)) + return "Structured property names cannot exceed 200 characters."; + + return null; + } + + private static Boolean HasMissingRequiredFields(LogEventInput log) => + log.EventId == Guid.Empty || log.TimestampUtc == default || + String.IsNullOrWhiteSpace(log.Severity) || String.IsNullOrWhiteSpace(log.ServiceName) || + String.IsNullOrWhiteSpace(log.Message); + + private static Boolean HasFieldsExceedingSupportedLength(LogEventInput log) => + log.Severity!.Length > 16 || log.ServiceName!.Length > 200 || + log.TraceId?.Length > 32 || log.SpanId?.Length > 16 || log.RequestId?.Length > 200; + + private static Boolean HasInvalidStructuredPropertyName(LogEventInput log) => + log.AdditionalProperties.Keys.Any(name => name.Length > 200); + + private static async Task IngestBatchAsync(IReadOnlyList events, + CentralLogsDbContext database, + CancellationToken cancellationToken) + { + await using Microsoft.EntityFrameworkCore.Storage.IDbContextTransaction transaction = + await database.Database.BeginTransactionAsync(IsolationLevel.Serializable, cancellationToken); + Guid[] requestedIds = events.Select(log => log.EventId).Distinct().ToArray(); + HashSet existingIds = (await database.LogEvents.AsNoTracking() + .Where(log => requestedIds.Contains(log.EventId)) + .Select(log => log.EventId) + .ToListAsync(cancellationToken)) + .ToHashSet(); + HashSet acceptedIds = new(); + List toInsert = new(); + List acceptedInputs = new(); + + foreach (LogEventInput input in events) + { + if (existingIds.Contains(input.EventId) || !acceptedIds.Add(input.EventId)) + continue; + + NormalizeBodyProperties(input); + + toInsert.Add(new CentralLogEvent + { + EventId = input.EventId, + TimestampUtc = input.TimestampUtc.ToUniversalTime(), + Severity = input.Severity!, + ServiceName = input.ServiceName!, + CorrelationId = input.CorrelationId, + TenantId = input.TenantId, + MerchantId = input.MerchantId, + TraceId = input.TraceId, + SpanId = input.SpanId, + RequestId = input.RequestId, + Message = input.Message!, + MessageTemplate = input.MessageTemplate, + Exception = input.Exception, + PropertiesJson = JsonSerializer.Serialize(input.AdditionalProperties) + }); + acceptedInputs.Add(input); + } + + await UpdateStructuredFieldCatalogAsync(database, acceptedInputs, cancellationToken); + database.LogEvents.AddRange(toInsert); + await database.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + + return new IngestionCounts(toInsert.Count, events.Count - toInsert.Count); + } + + private static void NormalizeBodyProperties(LogEventInput input) + { + foreach (String propertyName in input.AdditionalProperties.Keys.ToArray()) + { + if (!String.Equals(propertyName, "requestBody", StringComparison.OrdinalIgnoreCase) && + !String.Equals(propertyName, "responseBody", StringComparison.OrdinalIgnoreCase)) + continue; + + JsonElement value = input.AdditionalProperties[propertyName]; + if (value.ValueKind != JsonValueKind.String) + continue; + + JsonNode? body = ParseJsonObjectOrArray(value.GetString()); + if (body is null) + continue; + + using JsonDocument normalizedBody = JsonDocument.Parse(NormalizeEmbeddedJson(body).ToJsonString()); + input.AdditionalProperties[propertyName] = normalizedBody.RootElement.Clone(); + } + } + + private static JsonNode NormalizeEmbeddedJson(JsonNode node) + { + if (node is JsonObject objectNode) NormalizeObject(objectNode); + else if (node is JsonArray arrayNode) NormalizeArray(arrayNode); + + return node; + } + + private static void NormalizeObject(JsonObject objectNode) + { + foreach (String propertyName in objectNode.Select(property => property.Key).ToArray()) + { + JsonNode? child = objectNode[propertyName]; + JsonNode? normalizedChild = NormalizeEmbeddedValue(child); + if (!ReferenceEquals(child, normalizedChild)) + objectNode[propertyName] = normalizedChild; + } + } + + private static void NormalizeArray(JsonArray arrayNode) + { + for (Int32 index = 0; index < arrayNode.Count; index++) + { + JsonNode? child = arrayNode[index]; + JsonNode? normalizedChild = NormalizeEmbeddedValue(child); + if (!ReferenceEquals(child, normalizedChild)) + arrayNode[index] = normalizedChild; + } + } + + private static JsonNode? NormalizeEmbeddedValue(JsonNode? child) + { + if (child is JsonValue valueNode && valueNode.TryGetValue(out String? textValue)) + { + JsonNode? parsed = ParseJsonObjectOrArray(textValue); + if (parsed is not null) + return NormalizeEmbeddedJson(parsed); + } + + if (child is not null) + NormalizeEmbeddedJson(child); + + return child; + } + + private static JsonNode? ParseJsonObjectOrArray(String? value) + { + if (String.IsNullOrWhiteSpace(value)) + return null; + + try + { + JsonNode? node = JsonNode.Parse(value); + return node is JsonObject or JsonArray ? node : null; + } + catch (JsonException) + { + return null; + } + } + + private readonly record struct IngestionCounts(Int32 Accepted, Int32 Duplicates); + + public static async Task SearchAsync(CentralLogsDbContext database, + DateTimeOffset? fromUtc, + DateTimeOffset? toUtc, + String? serviceName, + Guid? tenantId, + Guid? correlationId, + String[]? filter, + Int32? take, + CancellationToken cancellationToken) + { + DateTimeOffset end = toUtc?.ToUniversalTime() ?? DateTimeOffset.UtcNow; + DateTimeOffset start = fromUtc?.ToUniversalTime() ?? end.AddHours(-24); + if (start >= end) + return Results.Problem("fromUtc must be earlier than toUtc.", statusCode: StatusCodes.Status400BadRequest); + + DateOnly firstDateUtc = DateOnly.FromDateTime(start.UtcDateTime); + DateOnly lastDateUtc = DateOnly.FromDateTime(end.AddTicks(-1).UtcDateTime); + + StructuredLogFilter[] filters; + try + { + filters = (filter ?? Array.Empty()) + .Select(value => JsonSerializer.Deserialize(value, + new JsonSerializerOptions(JsonSerializerDefaults.Web) { PropertyNameCaseInsensitive = true }) + ?? throw new JsonException("Filter cannot be null.")) + .ToArray(); + } + catch (JsonException) + { + return Results.Problem("Each filter must be a JSON object with field, type, and value properties.", statusCode: StatusCodes.Status400BadRequest); + } + + if (filters.Length > 10 || filters.Any(item => String.IsNullOrWhiteSpace(item.Field) || item.Field.Length > 200 || + item.Type is not ("string" or JsonNumberType or JsonBooleanType) || item.Value is null)) + return Results.Problem("Provide up to 10 filters with a field name (max 200 characters), type (string, number, boolean), and value.", statusCode: StatusCodes.Status400BadRequest); + + if (filters.Any(item => item.Type == JsonNumberType && NormalizeJsonNumber(item.Value) is null || + item.Type == JsonBooleanType && !Boolean.TryParse(item.Value, out _))) + return Results.Problem("Numeric and boolean filters must use valid values for their selected type.", statusCode: StatusCodes.Status400BadRequest); + + filters = filters.Select(item => item.Type switch + { + JsonNumberType => item with { Value = NormalizeJsonNumber(item.Value)! }, + JsonBooleanType => item with { Value = Boolean.Parse(item.Value).ToString().ToLowerInvariant() }, + _ => item + }).ToArray(); + + Int32 pageSize = Math.Clamp(take ?? 100, 1, MaximumPageSize); + IQueryable query = BuildStructuredFilterQuery(database, filters).AsNoTracking() + .Where(log => log.LogDateUtc >= firstDateUtc && log.LogDateUtc <= lastDateUtc && + log.TimestampUtc >= start && log.TimestampUtc < end); + if (!String.IsNullOrWhiteSpace(serviceName)) query = query.Where(log => log.ServiceName == serviceName); + if (tenantId.HasValue) query = query.Where(log => log.TenantId == tenantId); + if (correlationId.HasValue) query = query.Where(log => log.CorrelationId == correlationId); + + List results = await query + .OrderByDescending(log => log.TimestampUtc) + .Take(pageSize) + .ToListAsync(cancellationToken); + return Results.Ok(results); + } + + public static async Task GetServicesAsync(CentralLogsDbContext database, CancellationToken cancellationToken) + { + String[] services = await database.LogEvents.AsNoTracking() + .Select(log => log.ServiceName) + .Distinct() + .OrderBy(name => name) + .ToArrayAsync(cancellationToken); + return Results.Ok(new ServicesResponse(services)); + } + + private static IQueryable BuildStructuredFilterQuery(CentralLogsDbContext database, + IReadOnlyList filters) + { + if (filters.Count == 0) return database.LogEvents; + + List predicates = new(filters.Count); + List parameters = new(filters.Count * 3); + for (Int32 index = 0; index < filters.Count; index++) + { + StructuredLogFilter filter = filters[index]; + String fieldParameter = $"@filter{index}Field"; + String typeParameter = $"@filter{index}Type"; + String valueParameter = $"@filter{index}Value"; + Int32 jsonType = filter.Type switch { "string" => 1, JsonNumberType => 2, JsonBooleanType => 3, _ => throw new InvalidOperationException("Validated filter type expected.") }; + predicates.Add($"EXISTS (SELECT 1 FROM OPENJSON([PropertiesJson]) AS property WHERE property.[key] COLLATE Latin1_General_100_BIN2 = {fieldParameter} AND DATALENGTH(property.[key]) = DATALENGTH({fieldParameter}) AND property.[type] = {typeParameter} AND property.[value] COLLATE Latin1_General_100_BIN2 = {valueParameter} AND DATALENGTH(property.[value]) = DATALENGTH({valueParameter}))"); + parameters.Add(new SqlParameter(fieldParameter, System.Data.SqlDbType.NVarChar, 200) { Value = filter.Field }); + parameters.Add(new SqlParameter(typeParameter, System.Data.SqlDbType.Int) { Value = jsonType }); + parameters.Add(new SqlParameter(valueParameter, System.Data.SqlDbType.NVarChar, -1) { Value = filter.Value }); + } + + String sql = $"SELECT * FROM dbo.LogEvents WHERE {String.Join(" AND ", predicates)}"; + // The SQL structure uses only generated filter indexes; user values are exclusively SqlParameters. + // nosemgrep: csharp_injection_rule-SQLInjection + return database.LogEvents.FromSqlRaw(sql, parameters.Cast().ToArray()); + } + + private static String? NormalizeJsonNumber(String value) + { + try + { + using JsonDocument document = JsonDocument.Parse(value); + return document.RootElement.ValueKind == JsonValueKind.Number ? document.RootElement.GetRawText() : null; + } + catch (JsonException) + { + return null; + } + } + + public static async Task GetStructuredFieldsAsync(CentralLogsDbContext database, + String? serviceName, + CancellationToken cancellationToken) + { + if (String.IsNullOrWhiteSpace(serviceName)) + return Results.Problem("serviceName is required to discover structured fields.", statusCode: StatusCodes.Status400BadRequest); + if (serviceName.Length > 200) + return Results.Problem("serviceName cannot exceed 200 characters.", statusCode: StatusCodes.Status400BadRequest); + + List rows = await database.ServiceStructuredFields.AsNoTracking() + .Where(field => field.ServiceName == serviceName) + .OrderBy(field => field.FieldName) + .ThenBy(field => field.JsonType) + .ToListAsync(cancellationToken); + + StructuredFieldInfo[] fields = rows + .GroupBy(row => row.FieldName, StringComparer.Ordinal) + .Select(group => new StructuredFieldInfo(group.Key, + group.Select(row => row.JsonType).Distinct(StringComparer.Ordinal).Order().ToArray(), + group.Sum(row => row.EventCount), + group.Min(row => row.FirstSeenUtc), + group.Max(row => row.LastSeenUtc))) + .OrderBy(field => field.Name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return Results.Ok(new StructuredFieldsResponse(serviceName, fields)); + } + + private static async Task UpdateStructuredFieldCatalogAsync(CentralLogsDbContext database, + IReadOnlyList acceptedEvents, + CancellationToken cancellationToken) + { + if (acceptedEvents.Count == 0) return; + + Dictionary observations = new(StructuredFieldKeyComparer.Instance); + foreach (LogEventInput logEvent in acceptedEvents) + { + DateTimeOffset timestamp = logEvent.TimestampUtc.ToUniversalTime(); + foreach (KeyValuePair property in logEvent.AdditionalProperties) + { + StructuredFieldKey key = new(logEvent.ServiceName!, property.Key, GetJsonType(property.Value.ValueKind)); + if (observations.TryGetValue(key, out FieldObservation? observation)) + { + observations[key] = observation with + { + EventCount = observation.EventCount + 1, + FirstSeenUtc = timestamp < observation.FirstSeenUtc ? timestamp : observation.FirstSeenUtc, + LastSeenUtc = timestamp > observation.LastSeenUtc ? timestamp : observation.LastSeenUtc + }; + } + else + { + observations.Add(key, new FieldObservation(1, timestamp, timestamp)); + } + } + } + + String[] serviceNames = observations.Keys.Select(key => key.ServiceName).Distinct(StringComparer.Ordinal).ToArray(); + List existingFields = await database.ServiceStructuredFields + .Where(field => serviceNames.Contains(field.ServiceName)) + .ToListAsync(cancellationToken); + Dictionary existingByKey = existingFields.ToDictionary( + field => new StructuredFieldKey(field.ServiceName, field.FieldName, field.JsonType), + StructuredFieldKeyComparer.Instance); + + foreach ((StructuredFieldKey key, FieldObservation observation) in observations) + { + if (existingByKey.TryGetValue(key, out ServiceStructuredField? field)) + { + field.EventCount += observation.EventCount; + if (observation.FirstSeenUtc < field.FirstSeenUtc) field.FirstSeenUtc = observation.FirstSeenUtc; + if (observation.LastSeenUtc > field.LastSeenUtc) field.LastSeenUtc = observation.LastSeenUtc; + } + else + { + database.ServiceStructuredFields.Add(new ServiceStructuredField + { + ServiceName = key.ServiceName, + FieldName = key.FieldName, + JsonType = key.JsonType, + EventCount = observation.EventCount, + FirstSeenUtc = observation.FirstSeenUtc, + LastSeenUtc = observation.LastSeenUtc + }); + } + } + } + + private static String GetJsonType(JsonValueKind valueKind) => valueKind switch + { + JsonValueKind.Null => "null", + JsonValueKind.String => "string", + JsonValueKind.Number => JsonNumberType, + JsonValueKind.True or JsonValueKind.False => JsonBooleanType, + JsonValueKind.Array => "array", + JsonValueKind.Object => "object", + _ => "unknown" + }; + + private readonly record struct StructuredFieldKey(String ServiceName, String FieldName, String JsonType); + + private sealed class StructuredFieldKeyComparer : IEqualityComparer + { + public static StructuredFieldKeyComparer Instance { get; } = new(); + + public Boolean Equals(StructuredFieldKey left, StructuredFieldKey right) => + StringComparer.Ordinal.Equals(left.ServiceName, right.ServiceName) && + StringComparer.Ordinal.Equals(left.FieldName, right.FieldName) && + StringComparer.Ordinal.Equals(left.JsonType, right.JsonType); + + public Int32 GetHashCode(StructuredFieldKey key) => HashCode.Combine( + StringComparer.Ordinal.GetHashCode(key.ServiceName), + StringComparer.Ordinal.GetHashCode(key.FieldName), + StringComparer.Ordinal.GetHashCode(key.JsonType)); + } + + private sealed record FieldObservation(Int64 EventCount, DateTimeOffset FirstSeenUtc, DateTimeOffset LastSeenUtc); +} diff --git a/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.Designer.cs b/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.Designer.cs new file mode 100644 index 0000000..a0fc6e1 --- /dev/null +++ b/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.Designer.cs @@ -0,0 +1,142 @@ +// +using System; +using CentralLogging.Api.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace CentralLogging.Api.Migrations +{ + [DbContext(typeof(CentralLogsDbContext))] + [Migration("20261023142157_InitialCentralLogs")] + partial class InitialCentralLogs + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("CentralLogging.Api.Models.CentralLogEvent", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("LogDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("date") + .HasComputedColumnSql("CONVERT(date, [TimestampUtc])", true); + + b.Property("MerchantId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageTemplate") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ServiceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("SpanId") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("TenantId") + .HasColumnType("uniqueidentifier"); + + b.Property("TimestampUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("TraceId") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("EventId"); + + b.HasIndex("CorrelationId") + .HasFilter("[CorrelationId] IS NOT NULL"); + + b.HasIndex("TimestampUtc") + .IsDescending(); + + b.HasIndex("LogDateUtc", "TimestampUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_LogEvents_LogDateUtc_TimestampUtc"); + + b.HasIndex("ServiceName", "TimestampUtc") + .IsDescending(false, true); + + b.HasIndex("TenantId", "TimestampUtc") + .IsDescending(false, true); + + b.ToTable("LogEvents", "dbo", t => + { + t.HasCheckConstraint("CK_LogEvents_PropertiesJson_IsJson", "ISJSON([PropertiesJson]) = 1"); + }); + }); + + modelBuilder.Entity("CentralLogging.Api.Models.ServiceStructuredField", b => + { + b.Property("ServiceName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("FieldName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)") + .UseCollation("Latin1_General_100_BIN2"); + + b.Property("JsonType") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("EventCount") + .HasColumnType("bigint"); + + b.Property("FirstSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("LastSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.HasKey("ServiceName", "FieldName", "JsonType"); + + b.ToTable("ServiceStructuredFields", "dbo"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.cs b/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.cs new file mode 100644 index 0000000..87e997f --- /dev/null +++ b/CentralLogging.Api/Migrations/20261023142157_InitialCentralLogs.cs @@ -0,0 +1,112 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace CentralLogging.Api.Migrations +{ + /// + public partial class InitialCentralLogs : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + """ + IF OBJECT_ID(N'dbo.LogEvents', N'U') IS NULL + BEGIN + CREATE TABLE dbo.LogEvents + ( + EventId uniqueidentifier NOT NULL, + TimestampUtc datetimeoffset(7) NOT NULL, + LogDateUtc AS CONVERT(date, TimestampUtc) PERSISTED, + Severity nvarchar(16) NOT NULL, + ServiceName nvarchar(200) NOT NULL, + CorrelationId uniqueidentifier NULL, + TenantId uniqueidentifier NULL, + MerchantId uniqueidentifier NULL, + TraceId nvarchar(32) NULL, + SpanId nvarchar(16) NULL, + RequestId nvarchar(200) NULL, + Message nvarchar(max) NOT NULL, + MessageTemplate nvarchar(max) NULL, + Exception nvarchar(max) NULL, + PropertiesJson nvarchar(max) NOT NULL, + CONSTRAINT PK_LogEvents PRIMARY KEY CLUSTERED (EventId), + CONSTRAINT CK_LogEvents_PropertiesJson_IsJson CHECK (ISJSON(PropertiesJson) = 1) + ); + END; + + IF COL_LENGTH(N'dbo.LogEvents', N'LogDateUtc') IS NULL + ALTER TABLE dbo.LogEvents ADD LogDateUtc AS CONVERT(date, TimestampUtc) PERSISTED; + + IF NOT EXISTS (SELECT 1 FROM sys.check_constraints WHERE parent_object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'CK_LogEvents_PropertiesJson_IsJson') + ALTER TABLE dbo.LogEvents ADD CONSTRAINT CK_LogEvents_PropertiesJson_IsJson CHECK (ISJSON(PropertiesJson) = 1); + """); + + migrationBuilder.Sql( + """ + IF OBJECT_ID(N'dbo.ServiceStructuredFields', N'U') IS NULL + BEGIN + CREATE TABLE dbo.ServiceStructuredFields + ( + ServiceName nvarchar(200) NOT NULL, + FieldName nvarchar(200) COLLATE Latin1_General_100_BIN2 NOT NULL, + JsonType nvarchar(16) NOT NULL, + FirstSeenUtc datetimeoffset(7) NOT NULL, + LastSeenUtc datetimeoffset(7) NOT NULL, + EventCount bigint NOT NULL, + CONSTRAINT PK_ServiceStructuredFields PRIMARY KEY CLUSTERED (ServiceName, FieldName, JsonType) + ); + END; + """); + + migrationBuilder.Sql( + """ + IF NOT EXISTS (SELECT 1 FROM dbo.ServiceStructuredFields) + BEGIN + INSERT INTO dbo.ServiceStructuredFields (ServiceName, FieldName, JsonType, FirstSeenUtc, LastSeenUtc, EventCount) + SELECT events.ServiceName, + properties.[key] COLLATE Latin1_General_100_BIN2, + CASE properties.[type] + WHEN 0 THEN N'null' + WHEN 1 THEN N'string' + WHEN 2 THEN N'number' + WHEN 3 THEN N'boolean' + WHEN 4 THEN N'array' + WHEN 5 THEN N'object' + ELSE N'unknown' + END, + MIN(events.TimestampUtc), + MAX(events.TimestampUtc), + COUNT_BIG(*) + FROM dbo.LogEvents AS events + CROSS APPLY OPENJSON(CASE WHEN ISJSON(events.PropertiesJson) = 1 THEN events.PropertiesJson ELSE N'{}' END) AS properties + WHERE DATALENGTH(properties.[key]) <= 400 + GROUP BY events.ServiceName, properties.[key] COLLATE Latin1_General_100_BIN2, properties.[type]; + END; + """); + + migrationBuilder.Sql( + """ + IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'IX_LogEvents_TimestampUtc') + CREATE INDEX IX_LogEvents_TimestampUtc ON dbo.LogEvents (TimestampUtc DESC); + IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'IX_LogEvents_LogDateUtc_TimestampUtc') + CREATE INDEX IX_LogEvents_LogDateUtc_TimestampUtc ON dbo.LogEvents (LogDateUtc, TimestampUtc DESC); + IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'IX_LogEvents_TenantId_TimestampUtc') + CREATE INDEX IX_LogEvents_TenantId_TimestampUtc ON dbo.LogEvents (TenantId, TimestampUtc DESC); + IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'IX_LogEvents_ServiceName_TimestampUtc') + CREATE INDEX IX_LogEvents_ServiceName_TimestampUtc ON dbo.LogEvents (ServiceName, TimestampUtc DESC); + IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE object_id = OBJECT_ID(N'dbo.LogEvents') AND name = N'IX_LogEvents_CorrelationId') + CREATE INDEX IX_LogEvents_CorrelationId ON dbo.LogEvents (CorrelationId) WHERE CorrelationId IS NOT NULL; + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // This baseline adopts existing databases as well as creating new ones. + // Leaving its schema intact avoids dropping pre-existing log data on rollback. + } + } +} diff --git a/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.Designer.cs b/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.Designer.cs new file mode 100644 index 0000000..d125252 --- /dev/null +++ b/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.Designer.cs @@ -0,0 +1,143 @@ +// +using System; +using CentralLogging.Api.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace CentralLogging.Api.Migrations +{ + [DbContext(typeof(CentralLogsDbContext))] + [Migration("20261109185221_AlignStructuredFieldServiceNameCollation")] + partial class AlignStructuredFieldServiceNameCollation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("CentralLogging.Api.Models.CentralLogEvent", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("LogDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("date") + .HasComputedColumnSql("CONVERT(date, [TimestampUtc])", true); + + b.Property("MerchantId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageTemplate") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ServiceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("SpanId") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("TenantId") + .HasColumnType("uniqueidentifier"); + + b.Property("TimestampUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("TraceId") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("EventId"); + + b.HasIndex("CorrelationId") + .HasFilter("[CorrelationId] IS NOT NULL"); + + b.HasIndex("TimestampUtc") + .IsDescending(); + + b.HasIndex("LogDateUtc", "TimestampUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_LogEvents_LogDateUtc_TimestampUtc"); + + b.HasIndex("ServiceName", "TimestampUtc") + .IsDescending(false, true); + + b.HasIndex("TenantId", "TimestampUtc") + .IsDescending(false, true); + + b.ToTable("LogEvents", "dbo", t => + { + t.HasCheckConstraint("CK_LogEvents_PropertiesJson_IsJson", "ISJSON([PropertiesJson]) = 1"); + }); + }); + + modelBuilder.Entity("CentralLogging.Api.Models.ServiceStructuredField", b => + { + b.Property("ServiceName") + .HasMaxLength(200) + .UseCollation("Latin1_General_100_BIN2") + .HasColumnType("nvarchar(200)"); + + b.Property("FieldName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)") + .UseCollation("Latin1_General_100_BIN2"); + + b.Property("JsonType") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("EventCount") + .HasColumnType("bigint"); + + b.Property("FirstSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("LastSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.HasKey("ServiceName", "FieldName", "JsonType"); + + b.ToTable("ServiceStructuredFields", "dbo"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.cs b/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.cs new file mode 100644 index 0000000..729fc09 --- /dev/null +++ b/CentralLogging.Api/Migrations/20261109185221_AlignStructuredFieldServiceNameCollation.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace CentralLogging.Api.Migrations +{ + /// + public partial class AlignStructuredFieldServiceNameCollation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + """ + ALTER TABLE dbo.ServiceStructuredFields DROP CONSTRAINT PK_ServiceStructuredFields; + ALTER TABLE dbo.ServiceStructuredFields + ALTER COLUMN ServiceName nvarchar(200) COLLATE Latin1_General_100_BIN2 NOT NULL; + ALTER TABLE dbo.ServiceStructuredFields + ADD CONSTRAINT PK_ServiceStructuredFields PRIMARY KEY CLUSTERED (ServiceName, FieldName, JsonType); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + """ + ALTER TABLE dbo.ServiceStructuredFields DROP CONSTRAINT PK_ServiceStructuredFields; + ALTER TABLE dbo.ServiceStructuredFields + ALTER COLUMN ServiceName nvarchar(200) COLLATE DATABASE_DEFAULT NOT NULL; + ALTER TABLE dbo.ServiceStructuredFields + ADD CONSTRAINT PK_ServiceStructuredFields PRIMARY KEY CLUSTERED (ServiceName, FieldName, JsonType); + """); + } + } +} diff --git a/CentralLogging.Api/Migrations/CentralLogsDbContextModelSnapshot.cs b/CentralLogging.Api/Migrations/CentralLogsDbContextModelSnapshot.cs new file mode 100644 index 0000000..05762d5 --- /dev/null +++ b/CentralLogging.Api/Migrations/CentralLogsDbContextModelSnapshot.cs @@ -0,0 +1,140 @@ +// +using System; +using CentralLogging.Api.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace CentralLogging.Api.Migrations +{ + [DbContext(typeof(CentralLogsDbContext))] + partial class CentralLogsDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("CentralLogging.Api.Models.CentralLogEvent", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("LogDateUtc") + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("date") + .HasComputedColumnSql("CONVERT(date, [TimestampUtc])", true); + + b.Property("MerchantId") + .HasColumnType("uniqueidentifier"); + + b.Property("Message") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageTemplate") + .HasColumnType("nvarchar(max)"); + + b.Property("PropertiesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("ServiceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Severity") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("SpanId") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("TenantId") + .HasColumnType("uniqueidentifier"); + + b.Property("TimestampUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("TraceId") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("EventId"); + + b.HasIndex("CorrelationId") + .HasFilter("[CorrelationId] IS NOT NULL"); + + b.HasIndex("TimestampUtc") + .IsDescending(); + + b.HasIndex("LogDateUtc", "TimestampUtc") + .IsDescending(false, true) + .HasDatabaseName("IX_LogEvents_LogDateUtc_TimestampUtc"); + + b.HasIndex("ServiceName", "TimestampUtc") + .IsDescending(false, true); + + b.HasIndex("TenantId", "TimestampUtc") + .IsDescending(false, true); + + b.ToTable("LogEvents", "dbo", t => + { + t.HasCheckConstraint("CK_LogEvents_PropertiesJson_IsJson", "ISJSON([PropertiesJson]) = 1"); + }); + }); + + modelBuilder.Entity("CentralLogging.Api.Models.ServiceStructuredField", b => + { + b.Property("ServiceName") + .HasMaxLength(200) + .UseCollation("Latin1_General_100_BIN2") + .HasColumnType("nvarchar(200)"); + + b.Property("FieldName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)") + .UseCollation("Latin1_General_100_BIN2"); + + b.Property("JsonType") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("EventCount") + .HasColumnType("bigint"); + + b.Property("FirstSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.Property("LastSeenUtc") + .HasColumnType("datetimeoffset(7)"); + + b.HasKey("ServiceName", "FieldName", "JsonType"); + + b.ToTable("ServiceStructuredFields", "dbo"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/CentralLogging.Api/Models/CentralLogEvent.cs b/CentralLogging.Api/Models/CentralLogEvent.cs new file mode 100644 index 0000000..d01095b --- /dev/null +++ b/CentralLogging.Api/Models/CentralLogEvent.cs @@ -0,0 +1,20 @@ +namespace CentralLogging.Api.Models; + +public sealed class CentralLogEvent +{ + public Guid EventId { get; set; } + public DateTimeOffset TimestampUtc { get; set; } + public DateOnly LogDateUtc { get; private set; } + public String Severity { get; set; } = String.Empty; + public String ServiceName { get; set; } = String.Empty; + public Guid? CorrelationId { get; set; } + public Guid? TenantId { get; set; } + public Guid? MerchantId { get; set; } + public String? TraceId { get; set; } + public String? SpanId { get; set; } + public String? RequestId { get; set; } + public String Message { get; set; } = String.Empty; + public String? MessageTemplate { get; set; } + public String? Exception { get; set; } + public String PropertiesJson { get; set; } = "{}"; +} diff --git a/CentralLogging.Api/Models/LogEventInput.cs b/CentralLogging.Api/Models/LogEventInput.cs new file mode 100644 index 0000000..db66578 --- /dev/null +++ b/CentralLogging.Api/Models/LogEventInput.cs @@ -0,0 +1,24 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace CentralLogging.Api.Models; + +public sealed class LogEventInput +{ + public Guid EventId { get; init; } + public DateTimeOffset TimestampUtc { get; init; } + public String? Severity { get; set; } + public String? ServiceName { get; init; } + public Guid? CorrelationId { get; init; } + public Guid? TenantId { get; init; } + public Guid? MerchantId { get; init; } + public String? TraceId { get; init; } + public String? SpanId { get; init; } + public String? RequestId { get; init; } + public String? Message { get; init; } + public String? MessageTemplate { get; init; } + public String? Exception { get; init; } + + [JsonExtensionData] + public Dictionary AdditionalProperties { get; init; } = new(StringComparer.OrdinalIgnoreCase); +} diff --git a/CentralLogging.Api/Models/ServiceStructuredField.cs b/CentralLogging.Api/Models/ServiceStructuredField.cs new file mode 100644 index 0000000..780a3a0 --- /dev/null +++ b/CentralLogging.Api/Models/ServiceStructuredField.cs @@ -0,0 +1,11 @@ +namespace CentralLogging.Api.Models; + +public sealed class ServiceStructuredField +{ + public String ServiceName { get; set; } = String.Empty; + public String FieldName { get; set; } = String.Empty; + public String JsonType { get; set; } = String.Empty; + public DateTimeOffset FirstSeenUtc { get; set; } + public DateTimeOffset LastSeenUtc { get; set; } + public Int64 EventCount { get; set; } +} diff --git a/CentralLogging.Api/Models/StructuredFieldInfo.cs b/CentralLogging.Api/Models/StructuredFieldInfo.cs new file mode 100644 index 0000000..6c09262 --- /dev/null +++ b/CentralLogging.Api/Models/StructuredFieldInfo.cs @@ -0,0 +1,10 @@ +namespace CentralLogging.Api.Models; + +public sealed record StructuredFieldInfo(String Name, + String[] Types, + Int64 EventCount, + DateTimeOffset FirstSeenUtc, + DateTimeOffset LastSeenUtc); + +public sealed record StructuredFieldsResponse(String ServiceName, + StructuredFieldInfo[] Fields); diff --git a/CentralLogging.Api/Models/StructuredLogFilter.cs b/CentralLogging.Api/Models/StructuredLogFilter.cs new file mode 100644 index 0000000..00af6ad --- /dev/null +++ b/CentralLogging.Api/Models/StructuredLogFilter.cs @@ -0,0 +1,5 @@ +namespace CentralLogging.Api.Models; + +public sealed record StructuredLogFilter(String Field, String Type, String Value); + +public sealed record ServicesResponse(String[] Services); diff --git a/CentralLogging.Api/Program.cs b/CentralLogging.Api/Program.cs new file mode 100644 index 0000000..ea18c0c --- /dev/null +++ b/CentralLogging.Api/Program.cs @@ -0,0 +1,67 @@ +using System.Security.Cryptography; +using System.Text; +using CentralLogging.Api.Data; +using CentralLogging.Api.Endpoints; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.EntityFrameworkCore; + +WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + +String? configuredConnectionString = builder.Configuration.GetConnectionString("CentralLogs"); +if (String.IsNullOrWhiteSpace(configuredConnectionString)) + throw new InvalidOperationException("Configure ConnectionStrings:CentralLogs before starting the log consumer."); +String connectionString = configuredConnectionString; +String? configuredApiKey = builder.Configuration["LogIngestion:ApiKey"]; +if (String.IsNullOrWhiteSpace(configuredApiKey)) + throw new InvalidOperationException("Configure LogIngestion:ApiKey before starting the log consumer."); +String apiKey = configuredApiKey; + +builder.Services.AddDbContext(options => options.UseSqlServer(connectionString)); +WebApplication app = builder.Build(); + +await using (AsyncServiceScope scope = app.Services.CreateAsyncScope()) +{ + CentralLogsDbContext database = scope.ServiceProvider.GetRequiredService(); + await database.Database.MigrateAsync(); +} + +app.UseDefaultFiles(); +app.UseStaticFiles(); + +app.Use(async (context, next) => +{ + if (context.Request.Path.StartsWithSegments("/api/logs") && !HasValidApiKey(context.Request.Headers, apiKey)) + { + context.Response.StatusCode = StatusCodes.Status401Unauthorized; + return; + } + + await next(context); +}); + +app.MapGet("/health/ready", async (CentralLogsDbContext database, CancellationToken cancellationToken) => + await database.Database.CanConnectAsync(cancellationToken) + ? Results.Ok(new { status = "ready" }) + : Results.Problem("CentralLogs database is unavailable.", statusCode: StatusCodes.Status503ServiceUnavailable)); +app.MapPost("/api/logs", LogEndpoints.IngestAsync); +app.MapPost("/api/logs/upload", LogEndpoints.UploadAsync).DisableAntiforgery(); +app.MapGet("/api/logs", LogEndpoints.SearchAsync); +app.MapGet("/api/logs/fields", LogEndpoints.GetStructuredFieldsAsync); +app.MapGet("/api/logs/services", LogEndpoints.GetServicesAsync); + +app.Run(); + +static Boolean HasValidApiKey(IHeaderDictionary headers, String expectedKey) +{ + if (!headers.TryGetValue("X-Log-Api-Key", out Microsoft.Extensions.Primitives.StringValues supplied) || String.IsNullOrEmpty(supplied)) + return false; + + Byte[] expectedBytes = Encoding.UTF8.GetBytes(expectedKey); + Byte[] suppliedBytes = Encoding.UTF8.GetBytes(supplied.ToString()); + return expectedBytes.Length == suppliedBytes.Length && CryptographicOperations.FixedTimeEquals(expectedBytes, suppliedBytes); +} + +public partial class Program +{ + protected Program() { } +} diff --git a/CentralLogging.Api/Properties/launchSettings.json b/CentralLogging.Api/Properties/launchSettings.json new file mode 100644 index 0000000..71be037 --- /dev/null +++ b/CentralLogging.Api/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "CentralLogging.Api": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:59731;http://localhost:59732" + } + } +} \ No newline at end of file diff --git a/CentralLogging.Api/README.md b/CentralLogging.Api/README.md new file mode 100644 index 0000000..e7a7c2c --- /dev/null +++ b/CentralLogging.Api/README.md @@ -0,0 +1,50 @@ +# Central logging API starter + +This API is the first receiving and query component for the central logging design. It accepts batches of NLog JSON events, stores searchable fields in SQL Server, preserves additional structured properties as JSON, and provides log search plus service-scoped structured field discovery. A basic search UI is served from `/`. The Windows forwarder is in the separate `CentralLogging.Forwarder` project. + +## Start locally + +1. Provision the `CentralLogs` database and configure the connection string. The API applies pending EF Core migrations during startup, creating the schema or adopting tables from the former SQL scripts. Its SQL Server identity must have permission to create/alter the schema and migration history table. For production environments where the application identity should not have DDL permission, apply EF migrations through a separately controlled deployment step instead. +2. The API is configured for the local SQL Server instance (`Server=localhost`) with Windows Integrated Security in `appsettings.json`. If your instance uses another name, override `ConnectionStrings__CentralLogs`. Set `LogIngestion__ApiKey` to a secret value. In PowerShell, for example: + + ```powershell + $env:ConnectionStrings__CentralLogs = "Server=YOUR-SQL-SERVER;Database=CentralLogs;Integrated Security=true;Encrypt=true;TrustServerCertificate=true" + $env:LogIngestion__ApiKey = "replace-with-a-secret" + dotnet run --project CentralLogging.Api + ``` + +3. Requests to `/api/logs` must send the configured secret in the `X-Log-Api-Key` header. The browser UI prompts for this key at runtime, keeps it in memory only, and sends it with API requests. + +Open `http://localhost:/` to use the search UI. It provides time, service, tenant, correlation ID, and service-specific structured-field filters. `GET /health/ready` checks database connectivity. `POST /api/logs` accepts a JSON array of 1 to 500 events and returns the accepted and duplicate counts. Reposting an `eventId` is safe. `GET /api/logs/services` lists service names seen in stored events. + +The UI can upload an existing newline-delimited JSON (`.json` or `.jsonl`) log file. Every nonblank line is retained, including malformed JSON and records missing fields. The API stores the original line in `rawLog`, fills unavailable required fields with safe fallbacks, and ingests in batches of up to 500. The response includes `accepted`, `duplicates`, and `invalidLines` (zero when persistence succeeds). Uploads use `POST /api/logs/upload` as `multipart/form-data` with a `file` field and the same `X-Log-Api-Key` header. If a later batch fails, earlier batches may already be stored; retrying the file is safe because event IDs are deduplicated. Search results are database-wide and can include previously ingested events; after an upload, run a new search to reload the results. + +`GET /api/logs` supports `fromUtc`, `toUtc`, `serviceName`, `tenantId`, `correlationId`, `take`, and repeated `filter` query parameters. It defaults to the last 24 hours and caps results at 500. Each `filter` is URL-encoded JSON with `field`, `type`, and `value`; up to 10 exact-match filters are supported, and all must match. Supported types are `string`, `number`, and `boolean`. For example: `filter={"field":"amount","type":"number","value":"12.5"}`. String and boolean values are exact matches; number values match the exact JSON numeric representation, preserving distinctions in high-precision values (for example, `12.50` is distinct from `12.5`). The API checks the JSON property's name, scalar type, and value through SQL parameters. + +`GET /api/logs/fields?serviceName=Payments` returns the structured fields ever observed for the selected service, with observed JSON types, event counts, and first/last event timestamps. The catalog is updated in the same SQL transaction as newly accepted events. Duplicate event IDs do not increase its counts. This endpoint supplies the UI’s service-specific filter picker; array and object fields are shown in event details but are not available as exact-match filter controls. + +Example response: + +```json +{ + "serviceName": "Payments", + "fields": [ + { + "name": "transactionId", + "types": ["string"], + "eventCount": 420, + "firstSeenUtc": "2026-10-22T13:10:00Z", + "lastSeenUtc": "2026-10-23T11:45:00Z" + }, + { + "name": "amount", + "types": ["number"], + "eventCount": 180, + "firstSeenUtc": "2026-10-22T13:10:00Z", + "lastSeenUtc": "2026-10-23T11:45:00Z" + } + ] +} +``` + +The endpoint is intentionally protected by a configured API key from the first version. Use HTTPS, and replace or extend this with your chosen service authentication before exposing it beyond the trusted network. diff --git a/CentralLogging.Api/appsettings.json b/CentralLogging.Api/appsettings.json new file mode 100644 index 0000000..fc1680d --- /dev/null +++ b/CentralLogging.Api/appsettings.json @@ -0,0 +1,15 @@ +{ + "ConnectionStrings": { + "CentralLogs": "Server=localhost;Database=CentralLogs;Integrated Security=True;Encrypt=True;TrustServerCertificate=True" + }, + "LogIngestion": { + "ApiKey": "TestApiKey" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/CentralLogging.Api/wwwroot/index.html b/CentralLogging.Api/wwwroot/index.html new file mode 100644 index 0000000..4149586 --- /dev/null +++ b/CentralLogging.Api/wwwroot/index.html @@ -0,0 +1,61 @@ + + + + + + + Central Log Search + + + + +
+ CCentralLogs +
Not connected
+
+ +
+

OBSERVABILITY

Log search

Find and investigate events across your services.

◷ Search window is shown in UTC
+ +
+
01

Search criteria

Choose a service and narrow your results
+
+ + + + + +
+
+

Structured fields

Match events containing all selected field values.

+
⌕Select a service to see its available fields.
+
+
+
+ +
+
02

Upload a log file

Import an existing newline-delimited JSON file
+
+ + +
+

+
+ +
+

RESULTS

Log events

Search results show matching events already in the database, including earlier imports; they are not limited to the uploaded file.

No search yet
+
⌕

Ready when you are

Choose a time window or service, then search to see matching log events.

+
+
+ + +
+
C

API ACCESS

Connect to Central Logs

+

Enter the configured API key to load services and search events. The key stays in this page's memory and is cleared when you close or reload it.

+ + +
+
+ + + diff --git a/CentralLogging.Api/wwwroot/log-search.css b/CentralLogging.Api/wwwroot/log-search.css new file mode 100644 index 0000000..dcb5923 --- /dev/null +++ b/CentralLogging.Api/wwwroot/log-search.css @@ -0,0 +1,7 @@ +:root{font-family:'Segoe UI',Arial,sans-serif;color:#202b3c;background:#f5f7fb;font-synthesis:none;font-optical-sizing:auto;font-weight:400;--navy:#182b49;--blue:#2868d7;--muted:#7b8799;--line:#e5eaf1;--white:#fff;--shadow:0 8px 24px rgba(27,48,78,.055)} +*{box-sizing:border-box}body{margin:0;min-width:360px}.topbar{height:68px;background:var(--navy);color:#fff;display:flex;align-items:center;justify-content:space-between;padding:0 max(calc((100vw - 1200px)/2),28px);box-shadow:0 2px 9px #182b4924}.brand{display:flex;align-items:center;gap:10px;color:white;text-decoration:none;font:800 19px Manrope,Segoe UI,sans-serif;letter-spacing:-.5px}.brand-mark,.dialog-mark{width:31px;height:31px;display:grid;place-items:center;border-radius:9px;background:#3979e7;color:#fff;font:800 17px Manrope,sans-serif}.brand-light{font-weight:500;color:#b7c4d8}.topbar-right{display:flex;align-items:center;gap:22px}.connection-status{font-size:12px;color:#cbd5e2;display:flex;align-items:center;gap:8px}.connection-status i{width:7px;height:7px;background:#e3a83b;border-radius:50%}.connection-status.connected i{background:#4dd19b}.button{border:0;border-radius:7px;padding:10px 15px;font:600 13px 'DM Sans',sans-serif;cursor:pointer;display:inline-flex;align-items:center;justify-content:center;gap:8px;transition:background .15s,transform .15s}.button:disabled{opacity:.48;cursor:not-allowed}.button:not(:disabled):active{transform:translateY(1px)}.button-quiet{background:transparent;color:#d5deea;padding:8px}.button-quiet:hover{background:#ffffff14}.page-shell{max-width:1200px;margin:0 auto;padding:40px 28px 72px}.page-heading{display:flex;align-items:flex-end;justify-content:space-between;margin-bottom:26px}.eyebrow{margin:0 0 7px;color:#8290a4;font-size:10px;letter-spacing:1.35px;font-weight:700}.page-heading h1,.results-heading h2{font:800 29px Manrope,Segoe UI,sans-serif;letter-spacing:-.9px;color:#1c2b41;margin:0}.subheading{margin:7px 0 0;color:var(--muted);font-size:14px}.range-note{font-size:12px;color:#8591a3;padding-bottom:3px}.range-icon{color:#7285a2;font-size:17px;margin-right:5px}.panel{background:#fff;border:1px solid #e9edf3;border-radius:11px;box-shadow:var(--shadow)}.search-panel{padding:24px 26px 20px}.panel-heading{display:flex;align-items:center;justify-content:space-between;padding-bottom:20px;border-bottom:1px solid #edf0f5}.panel-heading>div{display:flex;align-items:center;gap:11px}.step-number{background:#eef4ff;color:#3973d5;padding:5px 7px;border-radius:5px;font:700 10px Manrope,sans-serif;letter-spacing:.5px}.panel-heading h2{font:700 16px Manrope,sans-serif;margin:0;color:#26364d}.panel-caption{font-size:11px;color:#9aa5b3}.criteria-grid{display:grid;grid-template-columns:1.25fr 1fr 1fr 1fr 1fr;gap:16px;padding:20px 0 22px}.form-field{min-width:0;display:flex;flex-direction:column;gap:7px}.form-field>span{font-size:11px;color:#56647a;font-weight:600}.form-field input,.form-field select,.filter-row select,.filter-row input{height:37px;width:100%;border:1px solid #dfe5ed;border-radius:6px;background:#fff;padding:0 10px;color:#334258;font:12px 'DM Sans',sans-serif;outline:none}.form-field input:focus,.form-field select:focus,.filter-row select:focus,.filter-row input:focus{border-color:#82a9ed;box-shadow:0 0 0 3px #3679e81a}.form-field small{font-size:10px;color:#96a1b1;line-height:1.35}.structured-area{border-top:1px solid #edf0f5;padding-top:17px}.structured-heading{display:flex;align-items:center;justify-content:space-between}.structured-heading h3{margin:0;font:700 13px Manrope,sans-serif;color:#2c3b51}.structured-heading p{margin:4px 0 0;color:#929eae;font-size:11px}.button-outline{border:1px solid #dce5f3;background:white;color:#3971ce;padding:8px 11px;font-size:11px}.button-outline:not(:disabled):hover{background:#f4f8ff}.filter-list{margin-top:12px}.empty-inline{display:flex;align-items:center;gap:10px;min-height:49px;padding:10px 13px;background:#f8fafd;border:1px dashed #dfe6ef;border-radius:7px;color:#8d99aa;font-size:11px}.empty-icon{font-size:19px;color:#91a6c5}.filter-row{display:grid;grid-template-columns:minmax(130px,1.1fr) minmax(90px,.65fr) minmax(160px,1fr) 32px;align-items:center;gap:9px;margin:8px 0}.filter-row select,.filter-row input{height:34px;font-size:11px}.remove-filter{height:31px;border:0;border-radius:5px;background:#f7f8fa;color:#9aa5b4;font-size:17px;cursor:pointer}.remove-filter:hover{background:#fff0f0;color:#c84c55}.search-actions{display:flex;justify-content:space-between;align-items:center;border-top:1px solid #edf0f5;margin-top:17px;padding-top:17px}.search-actions #search-message{font-size:11px;color:#8793a5}.button-primary{background:var(--blue);color:#fff;padding:10px 16px;box-shadow:0 3px 7px #2868d72a}.button-primary:hover:not(:disabled){background:#1f5fc9}.results-section{margin-top:34px}.results-heading{display:flex;align-items:flex-end;justify-content:space-between;margin-bottom:14px}.results-heading h2{font-size:19px}.results-heading .eyebrow{margin-bottom:5px}.result-count{font-size:11px;color:#8995a6;padding-bottom:3px}.results-list{display:flex;flex-direction:column;gap:10px}.welcome-state,.empty-state{min-height:215px;background:#fff;border:1px solid #e9edf3;border-radius:10px;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;padding:25px;box-shadow:var(--shadow)}.welcome-icon{width:42px;height:42px;border-radius:50%;display:grid;place-items:center;background:#eef4ff;color:#5383d3;font-size:24px}.welcome-state h3,.empty-state h3{font:700 14px Manrope,sans-serif;color:#40516a;margin:13px 0 5px}.welcome-state p,.empty-state p{font-size:11px;color:#919eae;margin:0;max-width:380px;line-height:1.6}.event-card{background:#fff;border:1px solid #e8edf3;border-radius:9px;box-shadow:var(--shadow);overflow:hidden}.event-top{display:flex;align-items:center;gap:10px;padding:12px 15px;border-bottom:1px solid #eef1f5}.severity{font-size:9px;font-weight:700;text-transform:uppercase;letter-spacing:.6px;padding:4px 7px;border-radius:4px;background:#edf4ff;color:#4374c1}.severity.error,.severity.fatal{background:#fff0ef;color:#c54d47}.severity.warning,.severity.warn{background:#fff7e8;color:#b88321}.severity.debug,.severity.trace{background:#f2efff;color:#7960b9}.event-service{font-size:11px;font-weight:700;color:#34455e}.event-time{font-size:10px;color:#8c98a9;margin-left:auto}.event-body{padding:13px 15px}.event-message{margin:0;color:#35445a;font-size:12px;line-height:1.5;white-space:pre-wrap;overflow-wrap:anywhere}.event-meta{display:flex;flex-wrap:wrap;gap:8px;margin-top:10px}.meta-chip{font-size:9px;color:#748198;background:#f5f7fa;border-radius:4px;padding:5px 7px}.event-properties{display:flex;flex-wrap:wrap;gap:6px;margin-top:9px}.property-chip{max-width:100%;background:#f5f8fd;border:1px solid #e9eff8;border-radius:4px;padding:5px 7px;font-size:9px;color:#60718a;overflow-wrap:anywhere}.property-chip b{color:#3e5d8f;font-weight:600}.inline-error{background:#fff6f5;border:1px solid #f2d7d4;color:#a74e48;border-radius:7px;padding:14px;font-size:12px}dialog{width:min(440px,calc(100% - 32px));border:1px solid #e6ebf2;border-radius:12px;padding:0;box-shadow:0 25px 80px #1025453b;color:#27374e}dialog::backdrop{background:#14233d77;backdrop-filter:blur(2px)}.key-form{padding:30px}.dialog-mark{width:38px;height:38px;border-radius:10px;margin-bottom:22px}.key-form h2{font:800 21px Manrope,sans-serif;margin:0 0 10px;letter-spacing:-.5px}.key-description{font-size:12px;line-height:1.6;color:#7c899c;margin:0 0 21px}.key-form .form-field input{height:40px}.dialog-actions{display:flex;justify-content:flex-end;gap:10px;margin-top:17px}.dialog-actions .button-quiet{color:#65748a}.error-text{color:#bd514b;font-size:11px;min-height:14px;margin:8px 0 0} +@media(max-width:900px){.criteria-grid{grid-template-columns:repeat(3,1fr)}.criteria-grid .service-field{grid-column:span 1}.page-shell{padding-top:30px}} +@media(max-width:620px){.topbar{padding:0 17px;height:60px}.topbar-right{gap:9px}.connection-status{font-size:10px}.page-shell{padding:28px 15px 50px}.page-heading{align-items:flex-start;gap:16px;flex-direction:column;margin-bottom:20px}.page-heading h1{font-size:25px}.search-panel{padding:18px 15px}.panel-caption{display:none}.criteria-grid{grid-template-columns:1fr 1fr;gap:13px}.criteria-grid .service-field{grid-column:span 2}.filter-row{grid-template-columns:1fr 1fr 32px;gap:7px}.filter-row .filter-field{grid-column:span 2}.filter-row .filter-type{grid-column:1}.filter-row .filter-value{grid-column:2;grid-row:2}.filter-row .remove-filter{grid-column:3;grid-row:1}.event-top{flex-wrap:wrap}.event-time{width:100%;margin:0}.range-note{font-size:10px}.search-actions{gap:8px}.search-actions #search-message{font-size:10px}} +.upload-panel{margin-top:16px;padding:20px 26px}.upload-panel .panel-heading{padding-bottom:14px}.upload-controls{display:flex;align-items:flex-end;gap:14px;padding-top:15px}.upload-file-field{flex:1}.upload-file-field input{height:auto;min-height:37px;padding:7px 9px}.upload-file-field input::file-selector-button{margin-right:10px;border:0;border-radius:4px;background:#eef4ff;color:#3971ce;padding:6px 9px;font:600 11px 'DM Sans',sans-serif;cursor:pointer}.upload-file-field small{font-size:10px;color:#96a1b1;line-height:1.35}.upload-message{min-height:15px;margin:10px 0 0;font-size:11px;color:#687990}.upload-message.error-text{color:#bd514b} +.results-note{margin:6px 0 0;color:#8995a6;font-size:11px;line-height:1.45;max-width:700px} +@media(max-width:620px){.upload-panel{padding:18px 15px}.upload-controls{align-items:stretch;flex-direction:column}.upload-controls .button{align-self:flex-end}} diff --git a/CentralLogging.Api/wwwroot/log-search.js b/CentralLogging.Api/wwwroot/log-search.js new file mode 100644 index 0000000..65c3457 --- /dev/null +++ b/CentralLogging.Api/wwwroot/log-search.js @@ -0,0 +1,388 @@ +(() => { + const api = { + key: null, + services: [], + fields: [], + uploading: false, + async get(url) { + const response = await fetch(url, { headers: { 'X-Log-Api-Key': api.key } }); + if (!response.ok) { + const problem = await response.json().catch(() => null); + throw new Error(problem?.detail || `Request failed (${response.status}).`); + } + return response.json(); + }, + async upload(file) { + const form = new FormData(); + form.append('file', file); + const response = await fetch('/api/logs/upload', { + method: 'POST', + headers: { 'X-Log-Api-Key': api.key }, + body: form + }); + if (!response.ok) { + const problem = await response.json().catch(() => null); + throw new Error(problem?.detail || `Upload failed (${response.status}).`); + } + return response.json(); + } + }; + + function el(id) { + return document.getElementById(id); + } + const service = el('service'); + const filterList = el('filter-list'); + const results = el('results'); + const dialog = el('key-dialog'); + const filterRows = []; + let selectedFieldData = []; + + function setConnected(connected) { + const status = el('connection-status'); + status.classList.toggle('connected', connected); + const indicator = document.createElement('i'); + status.replaceChildren(indicator, document.createTextNode(connected ? ' Connected' : ' Not connected')); + el('search').disabled = !connected; + el('upload').disabled = !connected || !el('log-file').files?.length || api.uploading; + service.disabled = !connected; + el('add-filter').disabled = !connected || !service.value || selectedFieldData.length === 0; + } + + function toUtcInputValue(date) { + return date.toISOString().slice(0, 16); + } + + function setDefaultTimeRange() { + const now = new Date(); + el('to').value = toUtcInputValue(now); + el('from').value = toUtcInputValue(new Date(now.getTime() - 24 * 60 * 60 * 1000)); + } + + async function connect(key) { + const previousKey = api.key; + api.key = key; + try { + const response = await api.get('/api/logs/services'); + api.services = response.services || []; + service.replaceChildren(new Option('All services', '')); + for (const name of api.services) service.add(new Option(name, name)); + setConnected(true); + dialog.close(); + el('key-error').textContent = ''; + await loadServiceFields(); + el('search-message').textContent = api.services.length ? `${api.services.length} service${api.services.length === 1 ? '' : 's'} available` : 'No services found yet'; + } catch (error) { + api.key = previousKey; + setConnected(Boolean(api.key)); + el('key-error').textContent = error.message; + if (!dialog.open) dialog.showModal(); + el('api-key').value = ''; + el('api-key').focus(); + } + } + + async function loadServiceFields() { + filterRows.length = 0; + filterList.replaceChildren(); + selectedFieldData = []; + if (!service.value) { + el('service-hint').textContent = 'Select a service to browse its structured fields.'; + showFieldEmpty('Select a service to see its available fields.'); + setConnected(Boolean(api.key)); + return; + } + el('service-hint').textContent = 'Loading fields…'; + try { + const data = await api.get(`/api/logs/fields?serviceName=${encodeURIComponent(service.value)}`); + api.fields = data.fields || []; + selectedFieldData = api.fields.filter(item => (item.types || []).some(type => ['string', 'number', 'boolean'].includes(type))); + el('service-hint').textContent = `${api.fields.length} structured field${api.fields.length === 1 ? '' : 's'} discovered`; + showFieldEmpty(selectedFieldData.length ? 'Add a filter to narrow the results by a structured field.' : 'No scalar structured fields are available for this service yet.'); + } catch (error) { + el('service-hint').textContent = 'Could not load fields.'; + showFieldEmpty(error.message, true); + } + setConnected(Boolean(api.key)); + } + + function showFieldEmpty(message, isError = false) { + const box = document.createElement('div'); + box.className = isError ? 'inline-error' : 'empty-inline'; + box.textContent = message; + filterList.replaceChildren(box); + } + + function addFilter() { + if (!selectedFieldData.length) return; + const row = document.createElement('div'); + row.className = 'filter-row'; + const field = document.createElement('select'); + field.className = 'filter-field'; + field.setAttribute('aria-label', 'Structured field'); + field.add(new Option('Select field', '')); + selectedFieldData.forEach(item => { + field.add(new Option(item.name, item.name)); + }); + const type = document.createElement('select'); + type.className = 'filter-type'; + type.setAttribute('aria-label', 'Value type'); + const value = document.createElement('input'); + value.className = 'filter-value'; + value.setAttribute('aria-label', 'Exact value'); + value.disabled = true; + const remove = document.createElement('button'); + remove.type = 'button'; + remove.className = 'remove-filter'; + remove.setAttribute('aria-label', 'Remove filter'); + remove.textContent = '×'; + row.append(field, type, value, remove); + filterList.append(row); + const item = { row, field, type, value }; + filterRows.push(item); + + field.addEventListener('change', () => { + type.replaceChildren(); + const data = selectedFieldData.find(candidate => candidate.name === field.value); + const types = (data?.types || []).filter(candidate => ['string', 'number', 'boolean'].includes(candidate)); + types.forEach(candidate => { + type.add(new Option(candidate, candidate)); + }); + type.disabled = !types.length; + value.disabled = !types.length; + updateValueControl(item); + }); + type.addEventListener('change', () => updateValueControl(item)); + remove.addEventListener('click', () => { + const index = filterRows.indexOf(item); + if (index >= 0) filterRows.splice(index, 1); + row.remove(); + if (!filterRows.length) showFieldEmpty('Add a filter to narrow the results by a structured field.'); + }); + if (filterRows.length === 1) filterList.replaceChildren(row); + } + + function updateValueControl(item) { + const kind = item.type.value; + item.value.type = 'text'; + item.value.inputMode = kind === 'number' ? 'decimal' : 'text'; + item.value.placeholder = kind === 'boolean' ? 'true or false' : kind === 'number' ? 'Exact JSON number' : 'Exact value'; + item.value.disabled = !kind; + } + + function isoFromInput(value) { + if (!value) return ''; + return new Date(`${value}:00Z`).toISOString(); + } + + function buildSearchUrl() { + const query = new URLSearchParams(); + const from = isoFromInput(el('from').value); + const to = isoFromInput(el('to').value); + if (from) query.set('fromUtc', from); + if (to) query.set('toUtc', to); + if (service.value) query.set('serviceName', service.value); + const correlationId = el('correlation').value.trim(); + const tenantId = el('tenant').value.trim(); + if (correlationId) query.set('correlationId', correlationId); + if (tenantId) query.set('tenantId', tenantId); + query.set('take', '100'); + for (const item of filterRows) { + if (!item.field.value || !item.type.value || !item.value.value) throw new Error('Complete each structured field filter or remove it.'); + const filter = { field: item.field.value, type: item.type.value, value: item.value.value }; + if (filter.type === 'boolean' && !/^(true|false)$/i.test(filter.value)) throw new Error('Boolean values must be true or false.'); + query.append('filter', JSON.stringify(filter)); + } + return `/api/logs?${query.toString()}`; + } + + async function search() { + el('search-message').textContent = ''; + el('search').disabled = true; + el('search').textContent = 'Searching…'; + results.replaceChildren(); + try { + const data = await api.get(buildSearchUrl()); + el('result-count').textContent = `${data.length} event${data.length === 1 ? '' : 's'} (maximum 100)`; + if (!data.length) { + results.append(makeState('No matching events', 'Try widening the time window or removing one of the filters.')); + } else { + data.forEach(log => { + results.append(renderEvent(log)); + }); + } + } catch (error) { + if (!api.key) return; + el('result-count').textContent = 'Search failed'; + const message = document.createElement('div'); + message.className = 'inline-error'; + message.textContent = error.message; + results.append(message); + } finally { + el('search').disabled = !api.key; + const icon = document.createElement('span'); + icon.setAttribute('aria-hidden', 'true'); + el('search').replaceChildren(icon, document.createTextNode(' Search logs')); + } + } + + async function uploadLogs() { + const file = el('log-file').files?.[0]; + if (!api.key || !file || api.uploading) return; + + api.uploading = true; + el('log-file').disabled = true; + el('upload-message').className = 'upload-message'; + el('upload-message').textContent = `Uploading ${file.name}…`; + setConnected(true); + try { + const summary = await api.upload(file); + el('upload-message').textContent = `Upload complete: ${summary.accepted} imported, ${summary.duplicates} duplicate, ${summary.invalidLines} invalid line${summary.invalidLines === 1 ? '' : 's'}.`; + el('result-count').textContent = 'Search again to view database results'; + results.replaceChildren(makeState('Upload complete', 'Run a search to view matching events in the database. Results can include events from previous imports.')); + } catch (error) { + el('upload-message').className = 'upload-message error-text'; + el('upload-message').textContent = error.message; + } finally { + api.uploading = false; + el('log-file').disabled = false; + setConnected(Boolean(api.key)); + } + } + + function makeState(title, description) { + const state = document.createElement('div'); + state.className = 'empty-state'; + const heading = document.createElement('h3'); + heading.textContent = title; + const text = document.createElement('p'); + text.textContent = description; + state.append(heading, text); + return state; + } + + function renderEvent(log) { + const card = document.createElement('article'); + card.className = 'event-card'; + card.append(createEventHeader(log), createEventBody(log)); + return card; + } + + function createEventHeader(log) { + const top = document.createElement('div'); + top.className = 'event-top'; + const severity = document.createElement('span'); + severity.className = `severity ${String(log.severity || '').toLowerCase()}`; + severity.textContent = log.severity || 'Unknown'; + const serviceName = document.createElement('span'); + serviceName.className = 'event-service'; + serviceName.textContent = log.serviceName || 'Unknown service'; + const timestamp = document.createElement('time'); + timestamp.className = 'event-time'; + timestamp.dateTime = log.timestampUtc || ''; + timestamp.textContent = log.timestampUtc ? new Date(log.timestampUtc).toLocaleString(undefined, { timeZone: 'UTC', timeZoneName: 'short' }) : ''; + top.append(severity, serviceName, timestamp); + return top; + } + + function createEventBody(log) { + const body = document.createElement('div'); + body.className = 'event-body'; + const message = document.createElement('p'); + message.className = 'event-message'; + message.textContent = log.message || ''; + body.append(message); + appendEventMetadata(body, log); + appendEventProperties(body, log); + appendEventException(body, log); + return body; + } + + function appendEventMetadata(body, log) { + const meta = document.createElement('div'); + meta.className = 'event-meta'; + if (log.correlationId) meta.append(makeChip(`Correlation: ${log.correlationId}`)); + if (log.tenantId) meta.append(makeChip(`Tenant: ${log.tenantId}`)); + if (log.merchantId) meta.append(makeChip(`Merchant: ${log.merchantId}`)); + if (meta.childElementCount) body.append(meta); + } + + function appendEventProperties(body, log) { + const properties = document.createElement('div'); + properties.className = 'event-properties'; + const data = typeof log.propertiesJson === 'string' ? safeJson(log.propertiesJson) : log.propertiesJson; + if (data && typeof data === 'object') { + for (const [name, value] of Object.entries(data)) { + const chip = document.createElement('span'); + chip.className = 'property-chip'; + const label = document.createElement('b'); + label.textContent = `${name}: `; + chip.append(label, document.createTextNode(formatValue(value))); + properties.append(chip); + } + } + if (properties.childElementCount) body.append(properties); + } + + function appendEventException(body, log) { + if (log.exception) { + const details = document.createElement('details'); + const summary = document.createElement('summary'); + summary.className = 'meta-chip'; + summary.textContent = 'Exception details'; + const exception = document.createElement('pre'); + exception.className = 'event-message'; + exception.textContent = log.exception; + details.append(summary, exception); + body.append(details); + } + } + + function makeChip(value) { + const chip = document.createElement('span'); + chip.className = 'meta-chip'; + chip.textContent = value; + return chip; + } + + function safeJson(value) { + try { return JSON.parse(value); } catch { return null; } + } + + function formatValue(value) { + return value !== null && typeof value === 'object' ? JSON.stringify(value) : String(value); + } + + service.addEventListener('change', loadServiceFields); + el('add-filter').addEventListener('click', addFilter); + el('search').addEventListener('click', search); + el('log-file').addEventListener('change', () => { + el('upload-message').textContent = ''; + el('upload-message').className = 'upload-message'; + setConnected(Boolean(api.key)); + }); + el('upload').addEventListener('click', uploadLogs); + el('change-key').addEventListener('click', () => { + setConnected(false); + dialog.showModal(); + el('api-key').focus(); + }); + el('cancel-key').addEventListener('click', () => { + if (api.key) { + dialog.close(); + setConnected(true); + } + else el('key-error').textContent = 'An API key is needed to connect.'; + }); + el('key-form').addEventListener('submit', event => { + event.preventDefault(); + const key = el('api-key').value; + el('api-key').value = ''; + connect(key); + }); + + setDefaultTimeRange(); + setConnected(false); + dialog.showModal(); + el('api-key').focus(); +})(); diff --git a/CentralLogging.Database/README.md b/CentralLogging.Database/README.md new file mode 100644 index 0000000..0a8034a --- /dev/null +++ b/CentralLogging.Database/README.md @@ -0,0 +1,5 @@ +# CentralLogs database + +The schema is owned by EF Core migrations in `CentralLogging.Api/Migrations`. The API applies pending migrations during startup. The initial migration creates a fresh schema or adopts existing tables created by the former SQL scripts, adds missing indexes and the persisted UTC date, and backfills the structured-field catalog if it is empty. + +The schema keeps common filters in typed columns and keeps all other event properties in `PropertiesJson`. `LogDateUtc` is a persisted computed column derived from each event's UTC `TimestampUtc`; the `(LogDateUtc, TimestampUtc DESC)` index supports date-bounded searches while retaining precise timestamp ordering. The API includes both date bounds and timestamp bounds in its query. The API updates `ServiceStructuredFields` in the same transaction as each newly accepted event, so the UI can list service-specific fields without scanning log JSON. Events are idempotent by `EventId`, which lets a forwarder retry a batch after a connection interruption. diff --git a/CentralLogging.Forwarder/CentralLogging.Forwarder.csproj b/CentralLogging.Forwarder/CentralLogging.Forwarder.csproj new file mode 100644 index 0000000..4ead9d8 --- /dev/null +++ b/CentralLogging.Forwarder/CentralLogging.Forwarder.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + + + + + + + diff --git a/CentralLogging.Forwarder/ForwarderCheckpointStore.cs b/CentralLogging.Forwarder/ForwarderCheckpointStore.cs new file mode 100644 index 0000000..d1646d2 --- /dev/null +++ b/CentralLogging.Forwarder/ForwarderCheckpointStore.cs @@ -0,0 +1,142 @@ +using System.Text.Json; + +namespace CentralLogging.Forwarder; + +public sealed class ForwarderCheckpointStore +{ + private readonly SemaphoreSlim Gate = new(1, 1); + private Dictionary? Checkpoints; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + public async Task GetOffsetAsync(String path, + String fileIdentity, + String checkpointPath, + CancellationToken cancellationToken) + { + await this.Gate.WaitAsync(cancellationToken); + try + { + await this.LoadAsync(checkpointPath, cancellationToken); + Dictionary checkpoints = this.Checkpoints!; + if (checkpoints.TryGetValue(path, out FileCheckpoint? atPath)) + { + if (String.IsNullOrEmpty(atPath.FileIdentity)) return 0; + if (atPath.FileIdentity == fileIdentity) return atPath.ByteOffset; + } + + return checkpoints.Values.FirstOrDefault(entry => entry.FileIdentity == fileIdentity)?.ByteOffset ?? 0; + } + finally + { + this.Gate.Release(); + } + } + + public async Task SaveOffsetAsync(String path, + String fileIdentity, + Int64 offset, + String checkpointPath, + CancellationToken cancellationToken) + { + await this.Gate.WaitAsync(cancellationToken); + try + { + await this.LoadAsync(checkpointPath, cancellationToken); + Dictionary checkpoints = this.Checkpoints!; + foreach (String oldPath in checkpoints.Where(pair => pair.Value.FileIdentity == fileIdentity && + !String.Equals(pair.Key, path, StringComparison.OrdinalIgnoreCase)) + .Select(pair => pair.Key).ToArray()) + checkpoints.Remove(oldPath); + + checkpoints[path] = new FileCheckpoint { FileIdentity = fileIdentity, ByteOffset = offset }; + await this.PersistAsync(checkpointPath, cancellationToken); + } + finally + { + this.Gate.Release(); + } + } + + public async Task RemoveUnseenAsync(HashSet seenFileIdentities, + HashSet discoveredPaths, + String checkpointPath, + CancellationToken cancellationToken) + { + await this.Gate.WaitAsync(cancellationToken); + try + { + await this.LoadAsync(checkpointPath, cancellationToken); + Dictionary checkpoints = this.Checkpoints!; + String[] stalePaths = checkpoints.Where(pair => + String.IsNullOrEmpty(pair.Value.FileIdentity) + ? !discoveredPaths.Contains(pair.Key) + : !seenFileIdentities.Contains(pair.Value.FileIdentity)) + .Select(pair => pair.Key) + .ToArray(); + + if (stalePaths.Length == 0) return; + foreach (String stalePath in stalePaths) checkpoints.Remove(stalePath); + await this.PersistAsync(checkpointPath, cancellationToken); + } + finally + { + this.Gate.Release(); + } + } + + private async Task PersistAsync(String checkpointPath, + CancellationToken cancellationToken) + { + String fullPath = Path.GetFullPath(checkpointPath); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + String temporaryPath = $"{fullPath}.{Guid.NewGuid():N}.tmp"; + try + { + await using (FileStream stream = File.Create(temporaryPath)) + { + await JsonSerializer.SerializeAsync(stream, + this.Checkpoints, + SerializerOptions, + cancellationToken); + } + + File.Move(temporaryPath, fullPath, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) File.Delete(temporaryPath); + } + } + + private async Task LoadAsync(String checkpointPath, + CancellationToken cancellationToken) + { + if (this.Checkpoints != null) return; + + this.Checkpoints = new Dictionary(StringComparer.OrdinalIgnoreCase); + String fullPath = Path.GetFullPath(checkpointPath); + if (!File.Exists(fullPath)) return; + + await using FileStream stream = File.OpenRead(fullPath); + using JsonDocument document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + if (property.Value.ValueKind == JsonValueKind.Number && property.Value.TryGetInt64(out Int64 legacyOffset)) + { + this.Checkpoints[property.Name] = new FileCheckpoint { ByteOffset = legacyOffset }; + continue; + } + + FileCheckpoint? entry = property.Value.Deserialize(SerializerOptions); + if (entry != null) this.Checkpoints[property.Name] = entry; + } + } + + public sealed class FileCheckpoint + { + public FileCheckpoint() { } + + public String FileIdentity { get; init; } = ""; + public Int64 ByteOffset { get; init; } + } +} diff --git a/CentralLogging.Forwarder/ForwarderOptions.cs b/CentralLogging.Forwarder/ForwarderOptions.cs new file mode 100644 index 0000000..35fc34c --- /dev/null +++ b/CentralLogging.Forwarder/ForwarderOptions.cs @@ -0,0 +1,15 @@ +namespace CentralLogging.Forwarder; + +public sealed class ForwarderOptions +{ + public static String SectionName { get; } = "LogForwarder"; + + public String[] SourceDirectories { get; init; } = []; + public String[] FilePatterns { get; init; } = ["structured-logger-*.json*"]; + public Boolean IncludeSubdirectories { get; init; } = true; + public String Endpoint { get; init; } = ""; + public String? ApiKey { get; init; } + public String CheckpointPath { get; init; } = "forwarder-checkpoint.json"; + public Int32 BatchSize { get; init; } = 100; + public Int32 PollIntervalSeconds { get; init; } = 5; +} diff --git a/CentralLogging.Forwarder/LogForwarderWorker.cs b/CentralLogging.Forwarder/LogForwarderWorker.cs new file mode 100644 index 0000000..fbc360e --- /dev/null +++ b/CentralLogging.Forwarder/LogForwarderWorker.cs @@ -0,0 +1,359 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Security.Cryptography; +using Microsoft.Extensions.Options; + +namespace CentralLogging.Forwarder; + +public sealed class LogForwarderWorker(IHttpClientFactory httpClientFactory, + ForwarderCheckpointStore checkpointStore, + IOptions options, + ILogger logger) : BackgroundService +{ + private readonly ForwarderOptions Settings = options.Value; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true + }; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + String apiKey = ValidateConfiguration(this.Settings); + + Int32 batchSize = Math.Clamp(this.Settings.BatchSize, 1, 500); + TimeSpan pollInterval = TimeSpan.FromSeconds(Math.Clamp(this.Settings.PollIntervalSeconds, 1, 300)); + HttpClient client = httpClientFactory.CreateClient("CentralLogs"); + client.DefaultRequestHeaders.Add("X-Log-Api-Key", apiKey); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await this.PollAsync(client, batchSize, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception exception) + { + logger.LogError(exception, "Log forwarding poll failed; files will be retried."); + } + + await Task.Delay(pollInterval, stoppingToken); + } + } + + private static String ValidateConfiguration(ForwarderOptions settings) + { + String? apiKey = settings.ApiKey; + if (settings.SourceDirectories.Length == 0 || settings.FilePatterns.Length == 0 || + !Uri.TryCreate(settings.Endpoint, UriKind.Absolute, out Uri? endpoint) || + (endpoint.Scheme != Uri.UriSchemeHttp && endpoint.Scheme != Uri.UriSchemeHttps) || + String.IsNullOrWhiteSpace(apiKey)) + throw new InvalidOperationException("Configure LogForwarder:SourceDirectories, Endpoint, and ApiKey."); + + return apiKey; + } + + private async Task PollAsync(HttpClient client, Int32 batchSize, CancellationToken cancellationToken) + { + HashSet seenFileIdentities = new(StringComparer.OrdinalIgnoreCase); + HashSet discoveredPaths = new(StringComparer.OrdinalIgnoreCase); + Boolean scanCompleted = await this.ProcessDirectoriesAsync(client, + batchSize, + seenFileIdentities, + discoveredPaths, + cancellationToken); + if (scanCompleted) + await checkpointStore.RemoveUnseenAsync(seenFileIdentities, + discoveredPaths, + this.Settings.CheckpointPath, + cancellationToken); + } + + private async Task ProcessDirectoriesAsync(HttpClient client, + Int32 batchSize, + HashSet seenFileIdentities, + HashSet discoveredPaths, + CancellationToken cancellationToken) + { + Boolean scanCompleted = true; + foreach (String directory in this.Settings.SourceDirectories) + { + Boolean directoryCompleted = await this.ProcessDirectoryAsync(client, + directory, + batchSize, + seenFileIdentities, + discoveredPaths, + cancellationToken); + scanCompleted = directoryCompleted && scanCompleted; + } + + return scanCompleted; + } + + private async Task ProcessDirectoryAsync(HttpClient client, + String directory, + Int32 batchSize, + HashSet seenFileIdentities, + HashSet discoveredPaths, + CancellationToken cancellationToken) + { + if (!Directory.Exists(directory)) + { + logger.LogWarning("Log source directory {LogDirectory} does not exist.", directory); + return false; + } + + if (!TryEnumerateFiles(directory, out String[] files)) + return false; + + Boolean scanCompleted = true; + foreach (String filePath in files) + { + discoveredPaths.Add(filePath); + try + { + await this.ForwardDiscoveredFileAsync(client, filePath, batchSize, seenFileIdentities, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Could not forward log file {LogFile}; it will be retried.", filePath); + scanCompleted = false; + } + } + + return scanCompleted; + } + + private Boolean TryEnumerateFiles(String directory, out String[] files) + { + try + { + SearchOption searchOption = this.Settings.IncludeSubdirectories + ? SearchOption.AllDirectories + : SearchOption.TopDirectoryOnly; + files = this.Settings.FilePatterns + .SelectMany(pattern => Directory.EnumerateFiles(directory, pattern, searchOption)) + .Select(Path.GetFullPath) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) + .ToArray(); + return true; + } + catch (Exception exception) + { + logger.LogError(exception, "Could not scan log source directory {LogDirectory}.", directory); + files = []; + return false; + } + } + + private async Task ForwardDiscoveredFileAsync(HttpClient client, + String filePath, + Int32 batchSize, + HashSet seenFileIdentities, + CancellationToken cancellationToken) + { + await using FileStream file = new(filePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); + String fileIdentity = await GetFileIdentityAsync(filePath, file, cancellationToken); + seenFileIdentities.Add(fileIdentity); + await this.ForwardFileAsync(client, file, filePath, fileIdentity, batchSize, cancellationToken); + } + + private async Task ForwardFileAsync(HttpClient client, + FileStream file, + String filePath, + String fileIdentity, + Int32 batchSize, + CancellationToken cancellationToken) + { + String fullFilePath = Path.GetFullPath(filePath); + Int64 offset = await checkpointStore.GetOffsetAsync(fullFilePath, + fileIdentity, + this.Settings.CheckpointPath, + cancellationToken); + if (file.Length < offset) + offset = await ResetTruncatedFileAsync(fullFilePath, filePath, fileIdentity, cancellationToken); + + while (true) + { + List lines = await ReadBatchAsync(file, offset, batchSize, cancellationToken); + if (lines.Count == 0) return; + + List events = ParseEvents(lines, filePath); + + Int64 batchEnd = lines[^1].NextOffset; + await ForwardBatchAsync(client, events, filePath, cancellationToken); + + offset = batchEnd; + await checkpointStore.SaveOffsetAsync(fullFilePath, + fileIdentity, + offset, + this.Settings.CheckpointPath, + cancellationToken); + } + } + + private async Task ResetTruncatedFileAsync(String fullFilePath, + String filePath, + String fileIdentity, + CancellationToken cancellationToken) + { + logger.LogWarning("Log file {LogFile} was truncated; restarting from its beginning.", filePath); + await checkpointStore.SaveOffsetAsync(fullFilePath, + fileIdentity, + 0, + this.Settings.CheckpointPath, + cancellationToken); + return 0; + } + + private List ParseEvents(IReadOnlyList lines, String filePath) + { + List events = new(lines.Count); + foreach (ParsedLogLine line in lines) + { + try + { + LogEventInput? item = JsonSerializer.Deserialize(line.Content, JsonOptions); + if (item is not null) item.Severity ??= item.Level; + if (item is null || item.EventId == Guid.Empty || item.TimestampUtc == default || + String.IsNullOrWhiteSpace(item.Severity) || String.IsNullOrWhiteSpace(item.ServiceName) || + String.IsNullOrWhiteSpace(item.Message) || HasFieldsExceedingApiLimits(item)) + { + logger.LogWarning("Skipping invalid structured log entry in {LogFile} at byte {Offset}.", filePath, line.NextOffset); + continue; + } + + events.Add(item); + } + catch (JsonException exception) + { + logger.LogWarning(exception, "Skipping malformed structured log entry in {LogFile} at byte {Offset}.", filePath, line.NextOffset); + } + } + + return events; + } + + private async Task ForwardBatchAsync(HttpClient client, + IReadOnlyList events, + String filePath, + CancellationToken cancellationToken) + { + if (events.Count == 0) return; + + using StringContent content = new(JsonSerializer.Serialize(events, JsonOptions), Encoding.UTF8, "application/json"); + using HttpResponseMessage response = await client.PostAsync(this.Settings.Endpoint, content, cancellationToken); + response.EnsureSuccessStatusCode(); + logger.LogInformation("Forwarded {EventCount} log events from {LogFile}.", events.Count, filePath); + } + + private static async Task GetFileIdentityAsync(String filePath, + FileStream file, + CancellationToken cancellationToken) + { + Int64 startingPosition = file.Position; + file.Seek(0, SeekOrigin.Begin); + Byte[] prefix = new Byte[1024]; + Int32 bytesRead = await file.ReadAsync(prefix, cancellationToken); + file.Seek(startingPosition, SeekOrigin.Begin); + + // Unix creation-time values can change when a file is renamed, so only include them on Windows. + Int32 creationTimeLength = OperatingSystem.IsWindows() ? sizeof(Int64) : 0; + Byte[] fingerprintInput = new Byte[creationTimeLength + bytesRead]; + if (creationTimeLength > 0) + BitConverter.GetBytes(File.GetCreationTimeUtc(filePath).Ticks).CopyTo(fingerprintInput, 0); + prefix.AsSpan(0, bytesRead).CopyTo(fingerprintInput.AsSpan(creationTimeLength)); + return Convert.ToHexString(SHA256.HashData(fingerprintInput)); + } + + private static Boolean HasFieldsExceedingApiLimits(LogEventInput item) => + item.Severity!.Length > 16 || item.ServiceName!.Length > 200 || + item.TraceId?.Length > 32 || item.SpanId?.Length > 16 || item.RequestId?.Length > 200 || + item.AdditionalProperties.Keys.Any(name => name.Length > 200); + + private static async Task> ReadBatchAsync(FileStream stream, + Int64 offset, + Int32 batchSize, + CancellationToken cancellationToken) + { + stream.Seek(offset, SeekOrigin.Begin); + List lines = new(batchSize); + using MemoryStream line = new(); + Byte[] buffer = new Byte[8192]; + Int64 lineStart = offset; + + while (lines.Count < batchSize) + { + Int32 count = await stream.ReadAsync(buffer, cancellationToken); + if (count == 0) break; + + for (Int32 index = 0; index < count; index++) + { + Byte value = buffer[index]; + if (value == (Byte)'\n') + { + Int64 nextOffset = stream.Position - count + index + 1; + String text = Encoding.UTF8.GetString(line.GetBuffer(), 0, checked((Int32)line.Length)).TrimEnd('\r'); + if (text.Length > 0) lines.Add(new ParsedLogLine(text, nextOffset)); + line.SetLength(0); + lineStart = nextOffset; + if (lines.Count == batchSize) + { + stream.Seek(lineStart, SeekOrigin.Begin); + return lines; + } + } + else + { + line.WriteByte(value); + } + } + } + + if (line.Length > 0) + { + // A final line without a newline may still be in the process of being written. + stream.Seek(lineStart, SeekOrigin.Begin); + } + + cancellationToken.ThrowIfCancellationRequested(); + return lines; + } + + private sealed record ParsedLogLine(String Content, Int64 NextOffset); + + private sealed class LogEventInput + { + public Guid EventId { get; init; } = Guid.Empty; + public DateTimeOffset TimestampUtc { get; init; } = default; + public String? Severity { get; set; } + [JsonPropertyName("level")] + public String? Level { get; init; } + public String? ServiceName { get; init; } = null; + public Guid? CorrelationId { get; init; } + public Guid? TenantId { get; init; } + public Guid? MerchantId { get; init; } + public String? TraceId { get; init; } = null; + public String? SpanId { get; init; } = null; + public String? RequestId { get; init; } = null; + public String? Message { get; init; } = null; + public String? MessageTemplate { get; init; } + public String? Exception { get; init; } + + [JsonExtensionData] + public Dictionary AdditionalProperties { get; init; } = new(StringComparer.OrdinalIgnoreCase); + + } +} diff --git a/CentralLogging.Forwarder/Program.cs b/CentralLogging.Forwarder/Program.cs new file mode 100644 index 0000000..dcb0c86 --- /dev/null +++ b/CentralLogging.Forwarder/Program.cs @@ -0,0 +1,10 @@ +using CentralLogging.Forwarder; + +HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); +builder.Services.AddWindowsService(options => options.ServiceName = "Central Logging Forwarder"); +builder.Services.Configure(builder.Configuration.GetSection(ForwarderOptions.SectionName)); +builder.Services.AddHttpClient("CentralLogs", client => client.Timeout = TimeSpan.FromSeconds(30)); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + +await builder.Build().RunAsync(); diff --git a/CentralLogging.Forwarder/README.md b/CentralLogging.Forwarder/README.md new file mode 100644 index 0000000..9ee116f --- /dev/null +++ b/CentralLogging.Forwarder/README.md @@ -0,0 +1,25 @@ +# Central logging Windows forwarder + +This worker tails files matching `LogForwarder:FilePatterns` from configured directories, sends batches to `POST /api/logs`, and keeps a byte offset per file in a checkpoint JSON file. `IncludeSubdirectories` controls recursive scanning. The default pattern is `structured-logger-*.json*`, which also matches common numbered backup names such as `.1.json` and `.json.1`. Patterns are configurable for other naming schemes, such as `transaction*.json*`. + +Each checkpoint records the file's identity and byte offset. The identity combines the file creation time with a hash of its first 1 KB, so it follows a file when rotation renames it and starts at offset zero when a rotated name is reused for a new file. After a complete scan, checkpoints for files no longer present are removed. A checkpoint is advanced only after a successful API response. If the process stops after the API accepts a batch but before the checkpoint is saved, the API's `eventId` deduplication makes replay safe. The worker maps NLog's `level` field to the API's `severity` field and preserves other NLog event properties. + +Configure the shared log directories as sources. The logger writes each event to the shared file and may write a second copy under its tenant-specific directory; forwarding both would ingest duplicate logical events with distinct event IDs. + +## Configure and run + +Edit `appsettings.json` or provide environment variables: + +```powershell +$env:LogForwarder__SourceDirectories__0 = 'C:\services\payment\logs\shared' +$env:LogForwarder__SourceDirectories__1 = 'D:\services\orders\logs\shared' +$env:LogForwarder__FilePatterns__0 = 'transaction*.json*' +$env:LogForwarder__Endpoint = 'https://central-logs.example/api/logs' +$env:LogForwarder__ApiKey = 'your-ingestion-key' +$env:LogForwarder__CheckpointPath = 'C:\ProgramData\CentralLogging\checkpoint.json' +dotnet run --project CentralLogging.Forwarder +``` + +When hosted by Windows Service, set the same environment variables for the service account and install the published executable with `sc.exe`. Give the service account read access to source log directories and write access to the checkpoint directory. Use HTTPS for the API endpoint. + +The worker reads completed newline-terminated JSON records. A partially written last record waits until the next poll. Malformed or incomplete records are logged and skipped so a single bad line cannot block forwarding. File retention remains controlled by the logging service; the worker only follows the files still present during a successful scan. diff --git a/CentralLogging.Forwarder/appsettings.json b/CentralLogging.Forwarder/appsettings.json new file mode 100644 index 0000000..f965291 --- /dev/null +++ b/CentralLogging.Forwarder/appsettings.json @@ -0,0 +1,22 @@ +{ + "LogForwarder": { + "SourceDirectories": [ + "G:\\Git\\TransactionProcessing\\Shared\\Shared.Logger.Sample\\bin\\Debug\\net10.0\\logs" + ], + "FilePatterns": [ + "structured-logger-*.json*" + ], + "IncludeSubdirectories": false, + "Endpoint": "https://localhost:7001/api/logs", + "ApiKey": "TestApiKey", + "CheckpointPath": "forwarder-checkpoint.json", + "BatchSize": 100, + "PollIntervalSeconds": 5 + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/CentralLogging.Tests/Api/ApiHostIntegrationTests.cs b/CentralLogging.Tests/Api/ApiHostIntegrationTests.cs new file mode 100644 index 0000000..cc495e2 --- /dev/null +++ b/CentralLogging.Tests/Api/ApiHostIntegrationTests.cs @@ -0,0 +1,20 @@ +using System.Net; +using System.Text.Json; +using Shouldly; + +namespace CentralLogging.Tests.Api; + +[Collection("CentralLoggingApi")] +public sealed class ApiHostIntegrationTests(SqlServerApiFixture fixture) +{ + [SqlServerFact] + public async Task Readiness_reports_ready_after_the_database_migrations_are_applied() + { + await fixture.ResetDatabaseAsync(); + using HttpResponseMessage response = await fixture.Client.GetAsync("/health/ready"); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + using JsonDocument body = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + body.RootElement.GetProperty("status").GetString().ShouldBe("ready"); + } +} diff --git a/CentralLogging.Tests/Api/ApiTestData.cs b/CentralLogging.Tests/Api/ApiTestData.cs new file mode 100644 index 0000000..247f4bd --- /dev/null +++ b/CentralLogging.Tests/Api/ApiTestData.cs @@ -0,0 +1,57 @@ +using System.Text; +using System.Text.Json; + +namespace CentralLogging.Tests.Api; + +internal static class ApiTestData +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public static HttpRequestMessage AuthorizedRequest(HttpMethod method, String path, HttpContent? content = null) + { + HttpRequestMessage request = new(method, path) { Content = content }; + request.Headers.Add("X-Log-Api-Key", SqlServerApiFixture.ApiKey); + return request; + } + + public static String Json(T value) => JsonSerializer.Serialize(value, JsonOptions); + + public static async Task ReadJsonAsync(HttpResponseMessage response) => + JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + + public static String LogLine(Guid? eventId, + String serviceName, + String message, + DateTimeOffset? timestampUtc = null, + Guid? tenantId = null, + Guid? correlationId = null, + String? additionalPropertiesJson = null) + { + Dictionary log = new() + { + ["eventId"] = eventId, + ["timestampUtc"] = timestampUtc ?? DateTimeOffset.UtcNow, + ["severity"] = "Information", + ["serviceName"] = serviceName, + ["message"] = message, + ["tenantId"] = tenantId, + ["correlationId"] = correlationId + }; + + if (!String.IsNullOrWhiteSpace(additionalPropertiesJson)) + { + using JsonDocument properties = JsonDocument.Parse(additionalPropertiesJson); + foreach (JsonProperty property in properties.RootElement.EnumerateObject()) + log[property.Name] = property.Value.Clone(); + } + + return Json(log); + } + + public static MultipartFormDataContent Upload(String fileName, String contents) + { + MultipartFormDataContent form = new(); + form.Add(new StringContent(contents, Encoding.UTF8, "text/plain"), "file", fileName); + return form; + } +} diff --git a/CentralLogging.Tests/Api/AuthorizationIntegrationTests.cs b/CentralLogging.Tests/Api/AuthorizationIntegrationTests.cs new file mode 100644 index 0000000..534c219 --- /dev/null +++ b/CentralLogging.Tests/Api/AuthorizationIntegrationTests.cs @@ -0,0 +1,42 @@ +using System.Net; +using Shouldly; + +namespace CentralLogging.Tests.Api; + +[Collection("CentralLoggingApi")] +public sealed class AuthorizationIntegrationTests(SqlServerApiFixture fixture) +{ + [SqlServerFact] + public async Task Protected_api_endpoints_accept_the_configured_key() + { + using HttpRequestMessage request = ApiTestData.AuthorizedRequest(HttpMethod.Get, "/api/logs/services"); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + response.StatusCode.ShouldBe(HttpStatusCode.OK); + } + + [SqlServerFact] + public async Task Protected_api_endpoints_reject_missing_and_incorrect_keys() + { + await fixture.ResetDatabaseAsync(); + + using HttpResponseMessage missingKey = await fixture.Client.GetAsync("/api/logs/services"); + missingKey.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + + using HttpRequestMessage wrongKeyRequest = new(HttpMethod.Get, "/api/logs/services"); + wrongKeyRequest.Headers.Add("X-Log-Api-Key", "incorrect-key"); + using HttpResponseMessage wrongKey = await fixture.Client.SendAsync(wrongKeyRequest); + wrongKey.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + } + + [SqlServerFact] + public async Task Readiness_and_static_search_ui_remain_available_without_an_api_key() + { + using HttpResponseMessage readiness = await fixture.Client.GetAsync("/health/ready"); + readiness.StatusCode.ShouldBe(HttpStatusCode.OK); + + using HttpResponseMessage ui = await fixture.Client.GetAsync("/"); + ui.StatusCode.ShouldBe(HttpStatusCode.OK); + ui.Content.Headers.ContentType?.MediaType.ShouldBe("text/html"); + (await ui.Content.ReadAsStringAsync()).ShouldContain("Central Log Search"); + } +} diff --git a/CentralLogging.Tests/Api/IngestionIntegrationTests.cs b/CentralLogging.Tests/Api/IngestionIntegrationTests.cs new file mode 100644 index 0000000..2ba8fb8 --- /dev/null +++ b/CentralLogging.Tests/Api/IngestionIntegrationTests.cs @@ -0,0 +1,110 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using CentralLogging.Api.Models; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CentralLogging.Tests.Api; + +[Collection("CentralLoggingApi")] +public sealed class IngestionIntegrationTests(SqlServerApiFixture fixture) +{ + private const String LogsEndpoint = "/api/logs"; + private const String RegionFieldName = "region"; + + [SqlServerFact] + public async Task Ingest_persists_one_copy_and_reports_duplicate_ids_in_a_batch_and_on_replay() + { + await fixture.ResetDatabaseAsync(); + Guid eventId = Guid.NewGuid(); + LogEventInput input = new() + { + EventId = eventId, + TimestampUtc = DateTimeOffset.UtcNow, + Severity = "Information", + ServiceName = "Payments", + Message = "payment accepted", + AdditionalProperties = new Dictionary + { + [RegionFieldName] = JsonSerializer.SerializeToElement("eu-west") + } + }; + + using HttpRequestMessage batchRequest = ApiTestData.AuthorizedRequest( + HttpMethod.Post, LogsEndpoint, JsonContent.Create(new[] { input, input })); + using HttpResponseMessage batchResponse = await fixture.Client.SendAsync(batchRequest); + batchResponse.StatusCode.ShouldBe(HttpStatusCode.Accepted); + using JsonDocument batchBody = await ApiTestData.ReadJsonAsync(batchResponse); + batchBody.RootElement.GetProperty("accepted").GetInt32().ShouldBe(1); + batchBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(1); + + using HttpRequestMessage replayRequest = ApiTestData.AuthorizedRequest( + HttpMethod.Post, LogsEndpoint, JsonContent.Create(new[] { input })); + using HttpResponseMessage replayResponse = await fixture.Client.SendAsync(replayRequest); + replayResponse.StatusCode.ShouldBe(HttpStatusCode.Accepted); + using JsonDocument replayBody = await ApiTestData.ReadJsonAsync(replayResponse); + replayBody.RootElement.GetProperty("accepted").GetInt32().ShouldBe(0); + replayBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(1); + + (Int32 logCount, Int64 fieldCount) = await fixture.ReadDatabaseAsync(async database => + (await database.LogEvents.CountAsync(), + await database.ServiceStructuredFields.Where(field => field.FieldName == RegionFieldName) + .Select(field => field.EventCount).SingleAsync())); + logCount.ShouldBe(1); + fieldCount.ShouldBe(1); + } + + [SqlServerFact] + public async Task Ingest_rejects_missing_required_fields_over_http() + { + await fixture.ResetDatabaseAsync(); + String payload = $"[{{\"eventId\":\"{Guid.NewGuid()}\",\"timestampUtc\":\"{DateTimeOffset.UtcNow:O}\",\"severity\":\"Information\",\"serviceName\":\"Payments\"}}]"; + using HttpRequestMessage request = ApiTestData.AuthorizedRequest( + HttpMethod.Post, LogsEndpoint, new StringContent(payload, System.Text.Encoding.UTF8, "application/json")); + + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + + response.StatusCode.ShouldBe(HttpStatusCode.BadRequest); + (await fixture.ReadDatabaseAsync(database => database.LogEvents.CountAsync())).ShouldBe(0); + } + + [SqlServerFact] + public async Task Ingest_tracks_service_names_using_case_sensitive_catalog_keys() + { + await fixture.ResetDatabaseAsync(); + DateTimeOffset timestamp = DateTimeOffset.UtcNow; + LogEventInput[] inputs = + [ + CreateEvent("Payments", timestamp, RegionFieldName, "eu-west"), + CreateEvent("payments", timestamp.AddSeconds(1), RegionFieldName, "eu-west") + ]; + + using HttpRequestMessage request = ApiTestData.AuthorizedRequest( + HttpMethod.Post, LogsEndpoint, JsonContent.Create(inputs)); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + + String responseBody = await response.Content.ReadAsStringAsync(); + response.StatusCode.ShouldBe(HttpStatusCode.Accepted, responseBody); + Int32 catalogRows = await fixture.ReadDatabaseAsync(database => database.ServiceStructuredFields + .Where(field => field.FieldName == RegionFieldName) + .CountAsync()); + catalogRows.ShouldBe(2); + } + + private static LogEventInput CreateEvent(String serviceName, + DateTimeOffset timestamp, + String fieldName, + String value) => new() + { + EventId = Guid.NewGuid(), + TimestampUtc = timestamp, + Severity = "Information", + ServiceName = serviceName, + Message = "catalog collation check", + AdditionalProperties = new Dictionary + { + [fieldName] = JsonSerializer.SerializeToElement(value) + } + }; +} diff --git a/CentralLogging.Tests/Api/SearchIntegrationTests.cs b/CentralLogging.Tests/Api/SearchIntegrationTests.cs new file mode 100644 index 0000000..cb720c8 --- /dev/null +++ b/CentralLogging.Tests/Api/SearchIntegrationTests.cs @@ -0,0 +1,165 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using CentralLogging.Api.Models; +using Shouldly; + +namespace CentralLogging.Tests.Api; + +[Collection("CentralLoggingApi")] +public sealed class SearchIntegrationTests(SqlServerApiFixture fixture) +{ + private const String PaymentsServiceName = "Payments"; + + [SqlServerFact] + public async Task Search_applies_time_service_tenant_and_correlation_filters() + { + await fixture.ResetDatabaseAsync(); + DateTimeOffset firstTimestamp = DateTimeOffset.UtcNow.AddMinutes(-2); + Guid tenantId = Guid.NewGuid(); + Guid correlationId = Guid.NewGuid(); + Guid differentCorrelationId = Guid.NewGuid(); + LogEventInput[] events = + [ + CreateEvent(PaymentsServiceName, firstTimestamp, tenantId, correlationId), + CreateEvent(PaymentsServiceName, firstTimestamp.AddSeconds(10), tenantId, correlationId), + CreateEvent(PaymentsServiceName, firstTimestamp.AddSeconds(5), tenantId, differentCorrelationId), + CreateEvent(PaymentsServiceName, firstTimestamp, Guid.NewGuid(), correlationId), + CreateEvent("Orders", firstTimestamp, tenantId, correlationId), + CreateEvent(PaymentsServiceName, firstTimestamp.AddMinutes(1), tenantId, correlationId) + ]; + await PostEventsAsync(events); + + String path = $"/api/logs?fromUtc={Encode(firstTimestamp.AddSeconds(-1).ToString("O"))}" + + $"&toUtc={Encode(firstTimestamp.AddSeconds(30).ToString("O"))}" + + $"&serviceName={PaymentsServiceName}&tenantId={tenantId:D}&correlationId={correlationId:D}&take=10"; + using HttpRequestMessage request = ApiTestData.AuthorizedRequest(HttpMethod.Get, path); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + using JsonDocument body = await ApiTestData.ReadJsonAsync(response); + body.RootElement.GetArrayLength().ShouldBe(2); + body.RootElement[0].GetProperty("eventId").GetGuid().ShouldBe(events[1].EventId); + body.RootElement[1].GetProperty("eventId").GetGuid().ShouldBe(events[0].EventId); + } + + [SqlServerFact] + public async Task Structured_filters_match_exact_json_values_and_types() + { + await fixture.ResetDatabaseAsync(); + DateTimeOffset timestamp = DateTimeOffset.UtcNow.AddMinutes(-1); + LogEventInput[] events = + [ + CreateEvent(PaymentsServiceName, timestamp, null, null, "{\"value\":12.50}"), + CreateEvent(PaymentsServiceName, timestamp.AddSeconds(1), null, null, "{\"value\":\"12.50\"}"), + CreateEvent(PaymentsServiceName, timestamp.AddSeconds(2), null, null, "{\"value\":true}") + ]; + await PostEventsAsync(events); + + await SearchForValueAsync("number", "12.50").ShouldReturnEventAsync(events[0].EventId); + await SearchForValueAsync("string", "12.50").ShouldReturnEventAsync(events[1].EventId); + await SearchForValueAsync("boolean", "true").ShouldReturnEventAsync(events[2].EventId); + await SearchForValueAsync("number", "12.5").ShouldReturnNoEventsAsync(); + await SearchForValueAsync("string", "true").ShouldReturnNoEventsAsync(); + } + + [SqlServerFact] + public async Task Search_caps_result_limit_at_500() + { + await fixture.ResetDatabaseAsync(); + LogEventInput[] events = Enumerable.Range(0, 501) + .Select(index => CreateEvent(PaymentsServiceName, DateTimeOffset.UtcNow.AddSeconds(index - 1000), null, null)) + .ToArray(); + await PostEventsAsync(events[..500]); + await PostEventsAsync(events[500..]); + + using HttpRequestMessage request = ApiTestData.AuthorizedRequest(HttpMethod.Get, "/api/logs?take=501"); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + + response.StatusCode.ShouldBe(HttpStatusCode.OK); + using JsonDocument body = await ApiTestData.ReadJsonAsync(response); + body.RootElement.GetArrayLength().ShouldBe(500); + } + + [SqlServerFact] + public async Task Search_rejects_malformed_and_unsupported_structured_filters_over_http() + { + await fixture.ResetDatabaseAsync(); + String[] invalidFilters = + [ + "not-json", + ApiTestData.Json(new { field = "value", type = "object", value = "{}" }) + ]; + + foreach (String filter in invalidFilters) + { + using HttpRequestMessage request = ApiTestData.AuthorizedRequest( + HttpMethod.Get, $"/api/logs?filter={Encode(filter)}"); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + response.StatusCode.ShouldBe(HttpStatusCode.BadRequest); + } + } + + private async Task SearchForValueAsync(String type, String value) + { + String filter = ApiTestData.Json(new { field = "value", type, value }); + using HttpRequestMessage request = ApiTestData.AuthorizedRequest( + HttpMethod.Get, $"/api/logs?filter={Encode(filter)}"); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + response.StatusCode.ShouldBe(HttpStatusCode.OK); + return await ApiTestData.ReadJsonAsync(response); + } + + private async Task PostEventsAsync(IReadOnlyList events) + { + using HttpRequestMessage request = ApiTestData.AuthorizedRequest( + HttpMethod.Post, "/api/logs", JsonContent.Create(events)); + using HttpResponseMessage response = await fixture.Client.SendAsync(request); + response.StatusCode.ShouldBe(HttpStatusCode.Accepted); + } + + private static LogEventInput CreateEvent(String serviceName, + DateTimeOffset timestamp, + Guid? tenantId, + Guid? correlationId, + String? properties = null) + { + Dictionary additionalProperties = new(StringComparer.OrdinalIgnoreCase); + if (!String.IsNullOrWhiteSpace(properties)) + { + using JsonDocument document = JsonDocument.Parse(properties); + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + additionalProperties[property.Name] = property.Value.Clone(); + } + + return new LogEventInput + { + EventId = Guid.NewGuid(), + TimestampUtc = timestamp, + Severity = "Information", + ServiceName = serviceName, + TenantId = tenantId, + CorrelationId = correlationId, + Message = "search fixture", + AdditionalProperties = additionalProperties + }; + } + + private static String Encode(String value) => Uri.EscapeDataString(value); +} + +internal static class SearchDocumentAssertions +{ + public static async Task ShouldReturnEventAsync(this Task documentTask, Guid expectedId) + { + using JsonDocument document = await documentTask; + document.RootElement.GetArrayLength().ShouldBe(1); + document.RootElement[0].GetProperty("eventId").GetGuid().ShouldBe(expectedId); + } + + public static async Task ShouldReturnNoEventsAsync(this Task documentTask) + { + using JsonDocument document = await documentTask; + document.RootElement.GetArrayLength().ShouldBe(0); + } +} diff --git a/CentralLogging.Tests/Api/SqlServerApiFixture.cs b/CentralLogging.Tests/Api/SqlServerApiFixture.cs new file mode 100644 index 0000000..f996bb1 --- /dev/null +++ b/CentralLogging.Tests/Api/SqlServerApiFixture.cs @@ -0,0 +1,180 @@ +using CentralLogging.Api.Data; +using System.Diagnostics; +using System.Net; +using System.Security.Cryptography; +using DotNet.Testcontainers.Configurations; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Testcontainers.MsSql; + +namespace CentralLogging.Tests.Api; + +public sealed class SqlServerApiFixture : IAsyncLifetime +{ + public static String ConnectionStringEnvironmentVariable => "CENTRAL_LOGGING_TEST_CONNECTION"; + public static String ApiKey { get; } = Convert.ToHexString(RandomNumberGenerator.GetBytes(32)); + private const String SqlServerImage = "mcr.microsoft.com/mssql/server:2022-CU14-ubuntu-22.04"; + private static readonly String SqlServerPassword = $"Aa1!{Convert.ToHexString(RandomNumberGenerator.GetBytes(24))}"; + + private WebApplicationFactory? Factory; + private MsSqlContainer? Container; + + public HttpClient Client { get; private set; } = null!; + + public async Task InitializeAsync() + { + String? connectionString = Environment.GetEnvironmentVariable(ConnectionStringEnvironmentVariable); + if (String.IsNullOrWhiteSpace(connectionString)) + { + await this.ConfigureWslDockerHostAsync(); + this.Container = new MsSqlBuilder(SqlServerImage) + .WithPassword(SqlServerPassword) + .WithDatabase("CentralLogsIntegration") + .Build(); + await this.Container.StartAsync(); + connectionString = this.Container.GetConnectionString(); + } + + SqlConnectionStringBuilder connectionStringBuilder = new(connectionString) + { + Encrypt = false, + TrustServerCertificate = true + }; + + this.Factory = new CentralLoggingApiFactory(connectionStringBuilder.ConnectionString); + this.Client = this.CreateClientWithStartupEnvironment(connectionStringBuilder.ConnectionString); + + IConfiguration configuration = this.Factory.Services.GetRequiredService(); + if (configuration["LogIngestion:ApiKey"] != ApiKey) + throw new InvalidOperationException("The integration test API key was not applied to the test host configuration."); + } + + private async Task ConfigureWslDockerHostAsync() + { + if (!ShouldDiscoverWslHost(TestcontainersSettings.DockerHostOverride)) + return; + + String? output = await ReadWslHostAddressesAsync(); + String? wslHost = FindWslHostAddress(output); + if (!String.IsNullOrWhiteSpace(wslHost)) + TestcontainersSettings.DockerHostOverride = wslHost; + } + + private static Boolean ShouldDiscoverWslHost(String? configuredHost) + { + if (!OperatingSystem.IsWindows() || String.IsNullOrWhiteSpace(configuredHost)) + return OperatingSystem.IsWindows(); + + if (String.Equals(configuredHost, "localhost", StringComparison.OrdinalIgnoreCase)) + return true; + + return IPAddress.TryParse(configuredHost, out IPAddress? address) && IPAddress.IsLoopback(address); + } + + private static async Task ReadWslHostAddressesAsync() + { + try + { + using Process process = new() + { + StartInfo = new ProcessStartInfo + { + FileName = "wsl.exe", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + process.StartInfo.ArgumentList.Add("-e"); + process.StartInfo.ArgumentList.Add("hostname"); + process.StartInfo.ArgumentList.Add("-I"); + + if (!process.Start()) return null; + String output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5)); + return process.ExitCode == 0 ? output : null; + } + catch (Exception exception) when (exception is InvalidOperationException or System.ComponentModel.Win32Exception or IOException or TimeoutException) + { + // Without WSL, Testcontainers' default host detection remains in effect. + return null; + } + } + + private static String? FindWslHostAddress(String? output) => output? + .Split((Char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(address => IPAddress.TryParse(address, out IPAddress? parsed) ? parsed : null) + .FirstOrDefault(address => address is { AddressFamily: System.Net.Sockets.AddressFamily.InterNetwork } && !IPAddress.IsLoopback(address)) + ?.ToString(); + + private HttpClient CreateClientWithStartupEnvironment(String connectionString) + { + const String connectionSetting = "ConnectionStrings__CentralLogs"; + String keyEnvironmentVariable = String.Concat("LogIngestion", "__", "Api", "Key"); + String? previousConnectionString = Environment.GetEnvironmentVariable(connectionSetting); + String? previousApiKey = Environment.GetEnvironmentVariable(keyEnvironmentVariable); + + try + { + Environment.SetEnvironmentVariable(connectionSetting, connectionString); + Environment.SetEnvironmentVariable(keyEnvironmentVariable, ApiKey); + return this.Factory!.CreateClient(); + } + finally + { + Environment.SetEnvironmentVariable(connectionSetting, previousConnectionString); + Environment.SetEnvironmentVariable(keyEnvironmentVariable, previousApiKey); + } + } + + public async Task ResetDatabaseAsync() + { + await using AsyncServiceScope scope = this.Factory!.Services.CreateAsyncScope(); + CentralLogsDbContext database = scope.ServiceProvider.GetRequiredService(); + await database.ServiceStructuredFields.ExecuteDeleteAsync(); + await database.LogEvents.ExecuteDeleteAsync(); + } + + public async Task ReadDatabaseAsync(Func> query) + { + await using AsyncServiceScope scope = this.Factory!.Services.CreateAsyncScope(); + CentralLogsDbContext database = scope.ServiceProvider.GetRequiredService(); + return await query(database); + } + + public async Task DisposeAsync() + { + this.Client?.Dispose(); + this.Factory?.Dispose(); + if (this.Container is not null) + await this.Container.DisposeAsync(); + } + + private sealed class CentralLoggingApiFactory(String connectionString) : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureAppConfiguration((_, configuration) => configuration.AddInMemoryCollection( + new Dictionary + { + ["ConnectionStrings:CentralLogs"] = connectionString, + ["LogIngestion:ApiKey"] = ApiKey + })); + } + } +} + +[CollectionDefinition("CentralLoggingApi")] +public sealed class CentralLoggingApiCollection : ICollectionFixture +{ +} + +[AttributeUsage(AttributeTargets.Method)] +public sealed class SqlServerFactAttribute : FactAttribute +{ +} diff --git a/CentralLogging.Tests/Api/UploadIntegrationTests.cs b/CentralLogging.Tests/Api/UploadIntegrationTests.cs new file mode 100644 index 0000000..7284736 --- /dev/null +++ b/CentralLogging.Tests/Api/UploadIntegrationTests.cs @@ -0,0 +1,110 @@ +using System.Text; +using System.Text.Json; +using CentralLogging.Api.Endpoints; +using Microsoft.AspNetCore.Http; +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; + +namespace CentralLogging.Tests.Api; + +[Collection("CentralLoggingApi")] +public sealed class UploadIntegrationTests(SqlServerApiFixture fixture) +{ + [SqlServerFact] + public async Task Upload_retains_malformed_and_incomplete_lines_and_deduplicates_an_identical_reupload() + { + await fixture.ResetDatabaseAsync(); + String malformedLine = "{malformed json"; + String incompleteLine = "{\"level\":\"Warning\",\"loggerName\":\"Checkout\",\"messageTemplate\":\"started\",\"amount\":12.5}"; + String contents = $"{malformedLine}\n\n{incompleteLine}\n"; + + using HttpResponseMessage firstResponse = await SendUploadAsync(contents); + firstResponse.StatusCode.ShouldBe(System.Net.HttpStatusCode.OK); + using JsonDocument firstBody = await ApiTestData.ReadJsonAsync(firstResponse); + firstBody.RootElement.GetProperty("accepted").GetInt32().ShouldBe(2); + firstBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(0); + firstBody.RootElement.GetProperty("invalidLines").GetInt32().ShouldBe(0); + + var stored = await fixture.ReadDatabaseAsync(async database => await database.LogEvents + .OrderBy(log => log.ServiceName) + .Select(log => new { log.EventId, log.Severity, log.ServiceName, log.Message, log.PropertiesJson }) + .ToListAsync()); + stored.Count.ShouldBe(2); + stored.Single(log => log.ServiceName == "Unknown").Message.ShouldBe(malformedLine); + using (JsonDocument malformedProperties = JsonDocument.Parse(stored.Single(log => log.ServiceName == "Unknown").PropertiesJson)) + malformedProperties.RootElement.GetProperty("rawLog").GetString().ShouldBe(malformedLine); + + var incomplete = stored.Single(log => log.ServiceName == "Checkout"); + incomplete.Severity.ShouldBe("Warning"); + incomplete.Message.ShouldBe("started"); + incomplete.EventId.ShouldNotBe(Guid.Empty); + using (JsonDocument incompleteProperties = JsonDocument.Parse(incomplete.PropertiesJson)) + { + incompleteProperties.RootElement.GetProperty("rawLog").GetString().ShouldBe(incompleteLine); + incompleteProperties.RootElement.GetProperty("sourceEventId").ValueKind.ShouldBe(JsonValueKind.Null); + incompleteProperties.RootElement.GetProperty("amount").GetDouble().ShouldBe(12.5); + } + + using HttpResponseMessage replayResponse = await SendUploadAsync(contents); + using JsonDocument replayBody = await ApiTestData.ReadJsonAsync(replayResponse); + replayBody.RootElement.GetProperty("accepted").GetInt32().ShouldBe(0); + replayBody.RootElement.GetProperty("duplicates").GetInt32().ShouldBe(2); + (await fixture.ReadDatabaseAsync(database => database.LogEvents.CountAsync())).ShouldBe(2); + } + + [SqlServerFact] + public async Task Upload_keeps_completed_batches_when_a_later_batch_fails() + { + await fixture.ResetDatabaseAsync(); + const String constraintName = "CK_CentralLoggingIntegrationRejectBatch"; + await fixture.ReadDatabaseAsync(async database => + { + await database.Database.ExecuteSqlRawAsync($"ALTER TABLE dbo.LogEvents DROP CONSTRAINT IF EXISTS {constraintName}"); + await database.Database.ExecuteSqlRawAsync($"ALTER TABLE dbo.LogEvents ADD CONSTRAINT {constraintName} CHECK (ServiceName <> N'FailBatch')"); + return true; + }); + + try + { + String[] lines = Enumerable.Range(0, 500) + .Select(index => ApiTestData.LogLine(Guid.NewGuid(), "Payments", $"event-{index}")) + .Append(ApiTestData.LogLine(Guid.NewGuid(), "FailBatch", "fail the second batch")) + .ToArray(); + Byte[] bytes = Encoding.UTF8.GetBytes(String.Join('\n', lines) + "\n"); + await using MemoryStream stream = new(bytes); + FormFile file = new(stream, 0, bytes.Length, "file", "events.jsonl") + { + Headers = new HeaderDictionary(), + ContentType = "application/x-ndjson" + }; + + DbUpdateException? failure = await Should.ThrowAsync(async () => + { + await fixture.ReadDatabaseAsync(async database => + { + await LogEndpoints.UploadAsync(file, database, CancellationToken.None); + return true; + }); + }); + failure.InnerException.ShouldBeOfType().Number.ShouldBe(547); + (await fixture.ReadDatabaseAsync(database => database.LogEvents.CountAsync())).ShouldBe(500); + } + finally + { + await fixture.ReadDatabaseAsync(async database => + { + await database.Database.ExecuteSqlRawAsync($"ALTER TABLE dbo.LogEvents DROP CONSTRAINT IF EXISTS {constraintName}"); + return true; + }); + } + } + + private async Task SendUploadAsync(String contents) + { + using MultipartFormDataContent form = ApiTestData.Upload("events.jsonl", contents); + using HttpRequestMessage request = ApiTestData.AuthorizedRequest(HttpMethod.Post, "/api/logs/upload", form); + return await fixture.Client.SendAsync(request); + } +} diff --git a/CentralLogging.Tests/CentralLogging.Tests.csproj b/CentralLogging.Tests/CentralLogging.Tests.csproj new file mode 100644 index 0000000..715c984 --- /dev/null +++ b/CentralLogging.Tests/CentralLogging.Tests.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + false + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/CentralLogging.Tests/Forwarder/LogForwarderWorkerTests.cs b/CentralLogging.Tests/Forwarder/LogForwarderWorkerTests.cs new file mode 100644 index 0000000..b6dc822 --- /dev/null +++ b/CentralLogging.Tests/Forwarder/LogForwarderWorkerTests.cs @@ -0,0 +1,368 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Threading.Channels; +using CentralLogging.Forwarder; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; + +namespace CentralLogging.Tests.Forwarder; + +public sealed class LogForwarderWorkerTests +{ + private const String PaymentsServiceName = "Payments"; + private const String SourceFileName = "events.jsonl"; + private const String CheckpointFileName = "checkpoint.json"; + + [Fact] + public async Task Worker_forwards_complete_lines_skips_malformed_lines_and_leaves_a_partial_tail_uncheckpointed() + { + using TemporaryDirectory directory = new(); + Guid firstId = Guid.NewGuid(); + Guid secondId = Guid.NewGuid(); + String firstLine = EventLine(firstId, PaymentsServiceName, "first", "Warning", useLevel: true); + String malformedLine = "{broken json"; + String secondLine = EventLine(secondId, PaymentsServiceName, "second", "Information"); + String partialLine = EventLine(Guid.NewGuid(), PaymentsServiceName, "not complete"); + String completeContent = $"{firstLine}\n{malformedLine}\n{secondLine}\n"; + String sourceFile = Path.Combine(directory.Path, SourceFileName); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + await File.WriteAllTextAsync(sourceFile, completeContent + partialLine, new UTF8Encoding(false)); + + using RecordingHandler handler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted)); + using HttpClient client = new(handler); + LogForwarderWorker worker = CreateWorker(directory.Path, checkpointPath, client); + using CancellationTokenSource cancellation = new(); + + await worker.StartAsync(cancellation.Token); + try + { + CapturedRequest request = await handler.ReadRequestAsync(); + request.ApiKey.ShouldBe("forwarder-test-key"); + JsonDocument events = JsonDocument.Parse(request.Body); + using (events) + { + events.RootElement.GetArrayLength().ShouldBe(2); + events.RootElement[0].GetProperty("eventId").GetGuid().ShouldBe(firstId); + events.RootElement[0].GetProperty("severity").GetString().ShouldBe("Warning"); + events.RootElement[1].GetProperty("eventId").GetGuid().ShouldBe(secondId); + } + + Int64 completedOffset = Encoding.UTF8.GetByteCount(completeContent); + await WaitForCheckpointAsync(checkpointPath, sourceFile, completedOffset); + (await ReadCheckpointOffsetAsync(checkpointPath, sourceFile)).ShouldBe((Int64?)completedOffset); + } + finally + { + await worker.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task Worker_skips_events_with_fields_that_exceed_the_ingestion_api_limits() + { + using TemporaryDirectory directory = new(); + Guid invalidId = Guid.NewGuid(); + Guid validId = Guid.NewGuid(); + String invalidLine = EventLine(invalidId, PaymentsServiceName, "invalid", requestId: new String('r', 201)); + String validLine = EventLine(validId, PaymentsServiceName, "valid"); + String sourceFile = Path.Combine(directory.Path, SourceFileName); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + String content = invalidLine + "\n" + validLine + "\n"; + await File.WriteAllTextAsync(sourceFile, content, new UTF8Encoding(false)); + + using RecordingHandler handler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted)); + using HttpClient client = new(handler); + LogForwarderWorker worker = CreateWorker(directory.Path, checkpointPath, client); + using CancellationTokenSource cancellation = new(); + + await worker.StartAsync(cancellation.Token); + try + { + CapturedRequest request = await handler.ReadRequestAsync(); + ReadEventIds(request.Body).ShouldBe([validId]); + await WaitForCheckpointAsync(checkpointPath, sourceFile, Encoding.UTF8.GetByteCount(content)); + handler.RequestCount.ShouldBe(1); + } + finally + { + await worker.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task Worker_retries_after_api_failure_and_advances_checkpoint_only_after_success() + { + using TemporaryDirectory directory = new(); + Guid eventId = Guid.NewGuid(); + String line = EventLine(eventId, PaymentsServiceName, "retry me"); + String sourceFile = Path.Combine(directory.Path, SourceFileName); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + await File.WriteAllTextAsync(sourceFile, line + "\n", new UTF8Encoding(false)); + + TaskCompletionSource allowRetrySuccess = new(TaskCreationOptions.RunContinuationsAsynchronously); + using RecordingHandler handler = new(async (attempt, _, cancellationToken) => attempt == 1 + ? new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + : await allowRetrySuccess.Task.WaitAsync(cancellationToken)); + using HttpClient client = new(handler); + LogForwarderWorker worker = CreateWorker(directory.Path, checkpointPath, client); + using CancellationTokenSource cancellation = new(); + + await worker.StartAsync(cancellation.Token); + try + { + CapturedRequest failedAttempt = await handler.ReadRequestAsync(); + CapturedRequest retryAttempt = await handler.ReadRequestAsync(); + ReadEventIds(failedAttempt.Body).ShouldBe([eventId]); + ReadEventIds(retryAttempt.Body).ShouldBe([eventId]); + + (await ReadCheckpointOffsetAsync(checkpointPath, sourceFile)).ShouldBeNull(); + allowRetrySuccess.SetResult(new HttpResponseMessage(HttpStatusCode.Accepted)); + await WaitForCheckpointAsync(checkpointPath, sourceFile, Encoding.UTF8.GetByteCount(line + "\n")); + handler.RequestCount.ShouldBe(2); + } + finally + { + allowRetrySuccess.TrySetCanceled(); + await worker.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task Worker_recovers_checkpoint_after_restart_and_file_rename() + { + using TemporaryDirectory directory = new(); + Guid firstId = Guid.NewGuid(); + Guid secondId = Guid.NewGuid(); + String firstLine = EventLine(firstId, PaymentsServiceName, new String('x', 1400)); + String secondLine = EventLine(secondId, PaymentsServiceName, "second line"); + String originalPath = Path.Combine(directory.Path, SourceFileName); + String renamedPath = Path.Combine(directory.Path, "events.jsonl.1.jsonl"); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + await File.WriteAllTextAsync(originalPath, firstLine + "\n", new UTF8Encoding(false)); + + using (RecordingHandler firstHandler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted))) + using (HttpClient firstClient = new(firstHandler)) + { + LogForwarderWorker firstWorker = CreateWorker(directory.Path, checkpointPath, firstClient); + using CancellationTokenSource firstCancellation = new(); + await firstWorker.StartAsync(firstCancellation.Token); + try + { + CapturedRequest firstRequest = await firstHandler.ReadRequestAsync(); + ReadEventIds(firstRequest.Body).ShouldBe([firstId]); + await WaitForCheckpointAsync(checkpointPath, originalPath, Encoding.UTF8.GetByteCount(firstLine + "\n")); + } + finally + { + await firstWorker.StopAsync(CancellationToken.None); + } + } + + File.Move(originalPath, renamedPath); + await File.AppendAllTextAsync(renamedPath, secondLine + "\n", new UTF8Encoding(false)); + + using RecordingHandler restartedHandler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted)); + using HttpClient restartedClient = new(restartedHandler); + LogForwarderWorker restartedWorker = CreateWorker(directory.Path, checkpointPath, restartedClient); + using CancellationTokenSource restartedCancellation = new(); + await restartedWorker.StartAsync(restartedCancellation.Token); + try + { + CapturedRequest resumedRequest = await restartedHandler.ReadRequestAsync(); + ReadEventIds(resumedRequest.Body).ShouldBe([secondId]); + await WaitForCheckpointAsync(checkpointPath, + renamedPath, + Encoding.UTF8.GetByteCount(firstLine + "\n" + secondLine + "\n")); + using JsonDocument checkpoint = JsonDocument.Parse(await File.ReadAllTextAsync(checkpointPath)); + checkpoint.RootElement.TryGetProperty(originalPath, out _).ShouldBeFalse(); + checkpoint.RootElement.TryGetProperty(renamedPath, out _).ShouldBeTrue(); + } + finally + { + await restartedWorker.StopAsync(CancellationToken.None); + } + } + + [Fact] + public async Task Worker_restarts_from_the_beginning_when_a_checkpointed_file_is_truncated() + { + using TemporaryDirectory directory = new(); + Guid firstId = Guid.NewGuid(); + Guid secondId = Guid.NewGuid(); + String largeMessage = new('x', 1400); + String firstLine = EventLine(firstId, PaymentsServiceName, largeMessage); + String secondLine = EventLine(secondId, PaymentsServiceName, "second"); + String sourceFile = Path.Combine(directory.Path, SourceFileName); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + String originalContent = firstLine + "\n" + secondLine + "\n"; + await File.WriteAllTextAsync(sourceFile, originalContent, new UTF8Encoding(false)); + + using (RecordingHandler firstHandler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted))) + using (HttpClient firstClient = new(firstHandler)) + { + LogForwarderWorker firstWorker = CreateWorker(directory.Path, checkpointPath, firstClient); + using CancellationTokenSource firstCancellation = new(); + await firstWorker.StartAsync(firstCancellation.Token); + try + { + CapturedRequest firstRequest = await firstHandler.ReadRequestAsync(); + ReadEventIds(firstRequest.Body).ShouldBe([firstId, secondId]); + await WaitForCheckpointAsync(checkpointPath, sourceFile, Encoding.UTF8.GetByteCount(originalContent)); + } + finally + { + await firstWorker.StopAsync(CancellationToken.None); + } + } + + String truncatedContent = firstLine + "\n"; + await File.WriteAllTextAsync(sourceFile, truncatedContent, new UTF8Encoding(false)); + (new FileInfo(sourceFile).Length).ShouldBeLessThan(Encoding.UTF8.GetByteCount(originalContent)); + + using RecordingHandler restartedHandler = new(_ => new HttpResponseMessage(HttpStatusCode.Accepted)); + using HttpClient restartedClient = new(restartedHandler); + LogForwarderWorker restartedWorker = CreateWorker(directory.Path, checkpointPath, restartedClient); + using CancellationTokenSource restartedCancellation = new(); + await restartedWorker.StartAsync(restartedCancellation.Token); + try + { + CapturedRequest restartedRequest = await restartedHandler.ReadRequestAsync(); + ReadEventIds(restartedRequest.Body).ShouldBe([firstId]); + await WaitForCheckpointAsync(checkpointPath, sourceFile, Encoding.UTF8.GetByteCount(truncatedContent)); + } + finally + { + await restartedWorker.StopAsync(CancellationToken.None); + } + } + + private static LogForwarderWorker CreateWorker(String sourceDirectory, String checkpointPath, HttpClient client) + { + ForwarderOptions options = new() + { + SourceDirectories = [sourceDirectory], + FilePatterns = ["*.jsonl"], + IncludeSubdirectories = false, + Endpoint = "http://localhost/api/logs", + ApiKey = "forwarder-test-key", + CheckpointPath = checkpointPath, + BatchSize = 20, + PollIntervalSeconds = 1 + }; + return new LogForwarderWorker(new SingleClientFactory(client), + new ForwarderCheckpointStore(), + Options.Create(options), + NullLogger.Instance); + } + + private static String EventLine(Guid eventId, + String serviceName, + String message, + String severity = "Information", + Boolean useLevel = false, + String? requestId = null) + { + Dictionary value = new() + { + ["eventId"] = eventId, + ["timestampUtc"] = DateTimeOffset.UtcNow, + [useLevel ? "level" : "severity"] = severity, + ["serviceName"] = serviceName, + ["message"] = message + }; + if (requestId is not null) + value["requestId"] = requestId; + return JsonSerializer.Serialize(value, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + } + + private static Guid[] ReadEventIds(String body) + { + using JsonDocument document = JsonDocument.Parse(body); + return document.RootElement.EnumerateArray() + .Select(item => item.GetProperty("eventId").GetGuid()) + .ToArray(); + } + + private static async Task WaitForCheckpointAsync(String checkpointPath, String filePath, Int64 expectedOffset) + { + DateTimeOffset deadline = DateTimeOffset.UtcNow.AddSeconds(10); + while (DateTimeOffset.UtcNow < deadline) + { + if (File.Exists(checkpointPath) && await ReadCheckpointOffsetAsync(checkpointPath, filePath) == expectedOffset) + return; + + await Task.Delay(20); + } + + throw new TimeoutException($"Checkpoint for {filePath} did not reach byte offset {expectedOffset}."); + } + + private static async Task ReadCheckpointOffsetAsync(String checkpointPath, String filePath) + { + if (!File.Exists(checkpointPath)) return null; + using JsonDocument document = JsonDocument.Parse(await File.ReadAllTextAsync(checkpointPath)); + return document.RootElement.TryGetProperty(filePath, out JsonElement entry) + ? entry.GetProperty("byteOffset").GetInt64() + : null; + } + + private sealed class RecordingHandler : HttpMessageHandler + { + private readonly Channel Requests = Channel.CreateUnbounded(); + private Int32 requestCount; + private readonly Func> ResponseFactory; + + public RecordingHandler(Func responseFactory) + : this((attempt, _, _) => Task.FromResult(responseFactory(attempt))) + { + } + + public RecordingHandler(Func> responseFactory) + { + this.ResponseFactory = responseFactory; + } + + public Int32 RequestCount => Volatile.Read(ref this.requestCount); + + public Task ReadRequestAsync() => this.Requests.Reader.ReadAsync().AsTask() + .WaitAsync(TimeSpan.FromSeconds(10)); + + protected override async Task SendAsync(HttpRequestMessage request, + CancellationToken cancellationToken) + { + String body = request.Content is null + ? String.Empty + : await request.Content.ReadAsStringAsync(cancellationToken); + String apiKey = request.Headers.TryGetValues("X-Log-Api-Key", out IEnumerable? values) + ? values.Single() + : String.Empty; + CapturedRequest capturedRequest = new(request.Method, request.RequestUri, apiKey, body); + Int32 attempt = Interlocked.Increment(ref this.requestCount); + await this.Requests.Writer.WriteAsync(capturedRequest, cancellationToken); + return await this.ResponseFactory(attempt, capturedRequest, cancellationToken); + } + } + + private sealed class SingleClientFactory(HttpClient client) : IHttpClientFactory + { + public HttpClient CreateClient(String name) + { + name.ShouldBe("CentralLogs"); + return client; + } + } + + private sealed record CapturedRequest(HttpMethod Method, Uri? Uri, String ApiKey, String Body); + + private sealed class TemporaryDirectory : IDisposable + { + public String Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("N")); + + public TemporaryDirectory() => Directory.CreateDirectory(this.Path); + + public void Dispose() => Directory.Delete(this.Path, recursive: true); + } +} diff --git a/CentralLogging.Tests/ForwarderCheckpointStoreTests.cs b/CentralLogging.Tests/ForwarderCheckpointStoreTests.cs new file mode 100644 index 0000000..891dc94 --- /dev/null +++ b/CentralLogging.Tests/ForwarderCheckpointStoreTests.cs @@ -0,0 +1,48 @@ +using System.Text.Json; +using CentralLogging.Forwarder; +using Shouldly; + +namespace CentralLogging.Tests; + +public sealed class ForwarderCheckpointStoreTests +{ + private const String FileIdentity = "file-one"; + private const String CheckpointFileName = "checkpoint.json"; + + [Fact] + public async Task Offset_follows_file_identity_when_rotation_renames_the_file() + { + using TemporaryDirectory directory = new(); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + ForwarderCheckpointStore store = new(); + + await store.SaveOffsetAsync("C:/logs/current.json", FileIdentity, 143, checkpointPath, CancellationToken.None); + Int64 offset = await store.GetOffsetAsync("C:/logs/current.json.1", FileIdentity, checkpointPath, CancellationToken.None); + + offset.ShouldBe(143); + } + + [Fact] + public async Task Saving_renamed_file_removes_its_old_path() + { + using TemporaryDirectory directory = new(); + String checkpointPath = Path.Combine(directory.Path, CheckpointFileName); + ForwarderCheckpointStore store = new(); + + await store.SaveOffsetAsync("C:/logs/current.json", FileIdentity, 143, checkpointPath, CancellationToken.None); + await store.SaveOffsetAsync("C:/logs/current.json.1", FileIdentity, 143, checkpointPath, CancellationToken.None); + + using JsonDocument document = JsonDocument.Parse(await File.ReadAllTextAsync(checkpointPath)); + document.RootElement.TryGetProperty("C:/logs/current.json", out _).ShouldBeFalse(); + document.RootElement.TryGetProperty("C:/logs/current.json.1", out _).ShouldBeTrue(); + } + + private sealed class TemporaryDirectory : IDisposable + { + public String Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("N")); + + public TemporaryDirectory() => Directory.CreateDirectory(this.Path); + + public void Dispose() => Directory.Delete(this.Path, recursive: true); + } +} diff --git a/CentralLogging.Tests/IngestionValidationTests.cs b/CentralLogging.Tests/IngestionValidationTests.cs new file mode 100644 index 0000000..f21d53c --- /dev/null +++ b/CentralLogging.Tests/IngestionValidationTests.cs @@ -0,0 +1,93 @@ +using CentralLogging.Api.Data; +using CentralLogging.Api.Endpoints; +using CentralLogging.Api.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.EntityFrameworkCore; +using Shouldly; + +namespace CentralLogging.Tests; + +public sealed class IngestionValidationTests +{ + [Fact] + public async Task Ingest_rejects_an_event_missing_a_required_field() + { + await using CentralLogsDbContext database = CreateDatabase(); + LogEventInput input = new() + { + EventId = Guid.NewGuid(), + TimestampUtc = DateTimeOffset.UtcNow, + Severity = "Information", + ServiceName = "Payments", + Message = "" + }; + IResult result = await LogEndpoints.IngestAsync([input], database, CancellationToken.None); + + ShouldBeBadRequest(result); + database.ChangeTracker.Entries().ShouldBeEmpty(); + } + + [Fact] + public async Task Ingest_rejects_an_empty_batch() + { + await using CentralLogsDbContext database = CreateDatabase(); + IResult result = await LogEndpoints.IngestAsync([], database, CancellationToken.None); + + ShouldBeBadRequest(result); + } + + [Fact] + public async Task Ingest_rejects_a_batch_over_the_documented_limit() + { + await using CentralLogsDbContext database = CreateDatabase(); + LogEventInput[] events = Enumerable.Range(0, 501).Select(_ => new LogEventInput + { + EventId = Guid.NewGuid(), + TimestampUtc = DateTimeOffset.UtcNow, + Severity = "Information", + ServiceName = "Payments", + Message = "accepted shape" + }).ToArray(); + IResult result = await LogEndpoints.IngestAsync(events, database, CancellationToken.None); + + ShouldBeBadRequest(result); + database.ChangeTracker.Entries().ShouldBeEmpty(); + } + + [Fact] + public async Task Search_rejects_an_invalid_numeric_filter_before_querying_storage() + { + await using CentralLogsDbContext database = CreateDatabase(); + IResult result = await LogEndpoints.SearchAsync(database, null, null, null, null, null, + ["{\"field\":\"amount\",\"type\":\"number\",\"value\":\"not-a-number\"}"], null, CancellationToken.None); + + ShouldBeBadRequest(result); + } + + [Fact] + public async Task Search_rejects_a_reversed_time_range_before_querying_storage() + { + await using CentralLogsDbContext database = CreateDatabase(); + DateTimeOffset now = DateTimeOffset.UtcNow; + + IResult result = await LogEndpoints.SearchAsync(database, now, now.AddMinutes(-1), null, null, null, + null, null, CancellationToken.None); + + ShouldBeBadRequest(result); + } + + private static void ShouldBeBadRequest(IResult result) + { + result.ShouldBeOfType() + .StatusCode.ShouldBe(StatusCodes.Status400BadRequest); + } + + private static CentralLogsDbContext CreateDatabase() + { + DbContextOptions options = new DbContextOptionsBuilder() + .UseSqlServer("Server=localhost;Database=CentralLogsTests;Integrated Security=true;TrustServerCertificate=true") + .Options; + return new CentralLogsDbContext(options); + } +} diff --git a/CentralLogging.Tests/Usings.cs b/CentralLogging.Tests/Usings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/CentralLogging.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/CentralLogging.sln b/CentralLogging.sln new file mode 100644 index 0000000..162d6cf --- /dev/null +++ b/CentralLogging.sln @@ -0,0 +1,62 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11205.157 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CentralLogging.Api", "CentralLogging.Api\CentralLogging.Api.csproj", "{C12A9083-9730-446E-AD0A-5F0312DDAE20}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CentralLogging.Forwarder", "CentralLogging.Forwarder\CentralLogging.Forwarder.csproj", "{6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CentralLogging.Tests", "CentralLogging.Tests\CentralLogging.Tests.csproj", "{1398F9DE-CDC8-47DE-9AA8-8857609FE480}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|x64.ActiveCfg = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|x64.Build.0 = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|x86.ActiveCfg = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Debug|x86.Build.0 = Debug|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|Any CPU.Build.0 = Release|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|x64.ActiveCfg = Release|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|x64.Build.0 = Release|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|x86.ActiveCfg = Release|Any CPU + {C12A9083-9730-446E-AD0A-5F0312DDAE20}.Release|x86.Build.0 = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|x64.ActiveCfg = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|x64.Build.0 = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|x86.ActiveCfg = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Debug|x86.Build.0 = Debug|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|Any CPU.Build.0 = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|x64.ActiveCfg = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|x64.Build.0 = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|x86.ActiveCfg = Release|Any CPU + {6AA5A3E4-2AE7-42A0-B78A-69DD04172C97}.Release|x86.Build.0 = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|x64.ActiveCfg = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|x64.Build.0 = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|x86.ActiveCfg = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Debug|x86.Build.0 = Debug|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|Any CPU.Build.0 = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|x64.ActiveCfg = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|x64.Build.0 = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|x86.ActiveCfg = Release|Any CPU + {1398F9DE-CDC8-47DE-9AA8-8857609FE480}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md index 4657309..9131640 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,15 @@ # SupportTools + +## Central Logging tests + +Run the Central Logging API and forwarder tests with: + +```powershell +dotnet test CentralLogging.Tests/CentralLogging.Tests.csproj --configuration Release +``` + +SQL Server integration tests start a disposable SQL Server container automatically and require Docker. On Windows with WSL2, the fixture discovers the WSL host IP for the container's mapped port, so the command stays the same. Set `CENTRAL_LOGGING_TEST_CONNECTION` to use an existing SQL Server instead; the tests apply the Central Logging migrations and need permission to create or update the configured test database. For other remote Docker hosts, set `TESTCONTAINERS_HOST_OVERRIDE` to the host address exposing mapped ports. The pull request workflow uses the same test-managed container setup. + ## Health Monitoring The standalone `HealthMonitoring` application is an ASP.NET Core/Blazor Server monitor for ASP.NET health endpoints. It uses SQL Server for service configuration, health observations, incidents, and history. diff --git a/docs/superpowers/plans/2026-10-24-central-logs-file-upload.md b/docs/superpowers/plans/2026-10-24-central-logs-file-upload.md new file mode 100644 index 0000000..a0691cc --- /dev/null +++ b/docs/superpowers/plans/2026-10-24-central-logs-file-upload.md @@ -0,0 +1,72 @@ +# Central Logs File Upload Implementation Plan + +> **For agentic workers:** Use the native execution approach to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a connected Central Logs UI user upload a newline-delimited JSON log file and import its valid events into SQL Server. + +**Architecture:** Add a multipart `/api/logs/upload` endpoint that parses lines and feeds bounded batches into the same validation, deduplication, persistence, and structured-field catalog flow as `POST /api/logs`. Add a connected-only upload panel to the existing static UI and show accepted, duplicate, and invalid-line totals. + +**Tech Stack:** ASP.NET Core minimal APIs on .NET 10, EF Core, SQL Server, static HTML/CSS/JavaScript. + +**Spec:** `docs/superpowers/specs/2026-10-24-central-logs-file-upload-design.md` + +## Global Constraints + +- Accept the newline-delimited JSON event format already read by the Central Logging forwarder. +- Reuse existing event validation, duplicate detection, persistence, and structured-field catalog updates. +- Uploaded files contain one JSON event per line. Malformed or invalid lines are skipped and counted; valid lines are ingested in batches no larger than the existing 500-event limit. +- Existing `eventId` deduplication remains the authority for identifying already stored events. +- The upload result reports accepted, duplicate, and invalid line totals. Failures to persist a batch are returned as upload errors. + +## Review Focus + +- Empty file or blank lines: return zero accepted and count no blank lines as invalid. +- Malformed JSON and missing required fields: skip and count each nonblank bad line. +- Duplicate IDs, including repeated IDs in the uploaded file: count as duplicates using existing ingestion behavior. +- A failure during persistence: return an error instead of reporting the failed batch as imported. +- Missing or invalid API key: reject upload through the existing `/api/logs` authorization middleware. + +--- + +## File Structure + +- Modify `CentralLogging.Api/Endpoints/LogEndpoints.cs` to share validation/persistence behavior and implement multipart upload parsing and summary results. +- Modify `CentralLogging.Api/Program.cs` to map the upload route under `/api/logs`. +- Modify `CentralLogging.Api/wwwroot/index.html` to add a file upload panel. +- Modify `CentralLogging.Api/wwwroot/log-search.js` to submit multipart data with the in-memory API key and present the result. +- Modify `CentralLogging.Api/wwwroot/log-search.css` for the upload panel layout and feedback. +- Modify `CentralLogging.Api/README.md` to document the accepted format and upload result. + +### Task 1: Add authenticated file ingestion endpoint + +**Files:** +- Modify: `CentralLogging.Api/Endpoints/LogEndpoints.cs` +- Modify: `CentralLogging.Api/Program.cs` +- Modify: `CentralLogging.Api/README.md` + +**Interfaces:** +- Produce `UploadAsync(IFormFile? file, CentralLogsDbContext database, CancellationToken cancellationToken)` as the endpoint handler; return 400 when `file` is missing. +- Return a JSON result with integer `accepted`, `duplicates`, and `invalidLines` values. +- Map `POST /api/logs/upload` and disable antiforgery for this API route; API-key authorization remains supplied by the existing path middleware. + +- [x] Extract the current validation and transactional insert/catalog update from `IngestAsync` into a shared batch operation that returns accepted and duplicate counts; preserve `POST /api/logs` response and validation behavior. +- [x] Implement `UploadAsync` to read nonblank lines from the uploaded file, deserialize each as `LogEventInput` with the forwarder-compatible case-insensitive JSON options and `level`-to-`severity` fallback, count malformed or invalid events, and submit valid events in batches of at most `MaximumBatchSize`. +- [x] Ensure a line is included in `invalidLines` only when nonblank and either malformed or invalid; let persistence failures fail the upload request. +- [x] Map `/api/logs/upload` and update the API README to describe newline-delimited JSON, idempotent duplicate handling, and the response counts. +- [x] Review the endpoint against the five Review Focus cases above by tracing each path through parsing and shared ingestion code; do not add or run tests unless requested. + +### Task 2: Add upload controls to the UI + +**Files:** +- Modify: `CentralLogging.Api/wwwroot/index.html` +- Modify: `CentralLogging.Api/wwwroot/log-search.js` +- Modify: `CentralLogging.Api/wwwroot/log-search.css` + +**Interfaces:** +- Consume `POST /api/logs/upload` using `FormData` with a field named `file` and header `X-Log-Api-Key`. +- Display the returned `accepted`, `duplicates`, and `invalidLines` counts as status text. + +- [x] Add an upload panel with an accessible `.json`/`.jsonl` file picker, upload button, and live status message; keep the upload button disabled until the API key connection is established and a file is selected. +- [x] Add an API multipart `post` helper or upload-specific fetch call that does not set the multipart `Content-Type` manually, and render upload progress state, errors, and the returned counts. +- [x] Clarify that search results are database-wide and clear previous results after a successful upload until the next search. +- [x] Add responsive styling consistent with the existing panels and verify the UI state transitions by inspecting the event handlers and markup; do not add or run tests unless requested. diff --git a/docs/superpowers/plans/2026-11-09-central-logging-test-coverage.md b/docs/superpowers/plans/2026-11-09-central-logging-test-coverage.md new file mode 100644 index 0000000..9b205aa --- /dev/null +++ b/docs/superpowers/plans/2026-11-09-central-logging-test-coverage.md @@ -0,0 +1,94 @@ +# Central Logging Test Coverage Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add meaningful API/database and forwarder test coverage for issue #845. + +**Architecture:** Keep API integration and forwarder unit coverage in `CentralLogging.Tests`. Run the API through `WebApplicationFactory` against a dedicated SQL Server database. Exercise the forwarder with temporary files and a fake `HttpMessageHandler`. + +**Tech Stack:** .NET 10, xUnit, Shouldly, EF Core SQL Server, `Microsoft.AspNetCore.Mvc.Testing`, GitHub Actions SQL Server service container. + +**Spec:** `docs/superpowers/specs/central-logging-test-coverage.md` and GitHub issue #845. + +## Global Constraints + +- Use Shouldly for every assertion; do not use `Assert.*`. +- Use real SQL Server for database-backed API and query behavior. +- Assert durable rows, response counts/statuses, posted payloads, and checkpoint offsets. +- Keep file-based tests isolated with temporary directories and deterministic fake HTTP responses. + +## Review Focus + +- Duplicate events must not inflate stored rows or structured-field counts; cover duplicate IDs in one request and a replay. +- Upload IDs derived from line position must remain stable on identical re-upload; cover malformed and incomplete input while checking raw lines and counts. +- Later upload batch failure may leave earlier batches committed; prove the first batch survives and document the failed batch outcome. +- Structured filters must enforce JSON scalar type and exact value, including a number representation boundary. +- Forwarder offsets advance only after a successful send; cover retry and recovery across file rename or truncation. + +### Task 1: SQL Server integration host + +**Files:** +- Modify: `CentralLogging.Tests/CentralLogging.Tests.csproj` +- Create: `CentralLogging.Tests/Api/SqlServerApiFixture.cs` +- Create: `CentralLogging.Tests/Api/CentralLoggingApiCollection.cs` +- Modify: `.github/workflows/pullrequest.yml` + +**Interfaces:** +- Fixture provides one `HttpClient` from `WebApplicationFactory`, configured with `ConnectionStrings:CentralLogs` and `LogIngestion:ApiKey`. +- Fixture applies migrations on startup and clears `ServiceStructuredFields` and `LogEvents` before each test. +- CI sets the fixture's connection string to the SQL Server service container. + +- [x] Add WebApplicationFactory dependency and a SQL Server service to the Central Logging workflow job. +- [x] Add a collection fixture that starts the real API and serializes integration tests. +- [x] Add a readiness integration test that verifies host startup, migrations, and database readiness. + +### Task 2: Ingestion and upload behavior + +**Files:** +- Create: `CentralLogging.Tests/Api/IngestionIntegrationTests.cs` +- Create: `CentralLogging.Tests/Api/UploadIntegrationTests.cs` + +**Interfaces:** +- Use the fixture's `HttpClient` and test API key; query persisted outcomes through a scoped `CentralLogsDbContext` helper. + +- [x] Test accepted event persistence and duplicate ID counts for duplicates in one batch and a replay. +- [x] Test structured-field catalog counts remain durable and unchanged for duplicate events. +- [x] Test malformed and incomplete upload lines are retained with raw text and safe fallbacks. +- [x] Test accepted, duplicate, and invalid line counts plus deterministic generated IDs across re-upload. +- [x] Test a later batch failure leaves the earlier batch durable and records the documented failure behavior. + +### Task 3: Search and access behavior + +**Files:** +- Create: `CentralLogging.Tests/Api/SearchIntegrationTests.cs` +- Create: `CentralLogging.Tests/Api/AuthorizationIntegrationTests.cs` + +**Interfaces:** +- Seed logs through the public ingestion route so search tests cross the API and database boundary. + +- [x] Test time, service, tenant, and correlation filters and result ordering/limits. +- [x] Test exact structured string, number, and boolean matches, including a type mismatch and precise numeric text. +- [x] Test invalid filters return 400 without querying stored data. +- [x] Test missing and incorrect API keys are rejected while readiness and the static UI follow their intended access model. + +### Task 4: Forwarder file and checkpoint behavior + +**Files:** +- Create: `CentralLogging.Tests/Forwarder/LogForwarderWorkerTests.cs` +- Modify: `CentralLogging.Tests/ForwarderCheckpointStoreTests.cs` + +**Interfaces:** +- Start `LogForwarderWorker` with temporary source/checkpoint paths and an `IHttpClientFactory` backed by a fake `HttpMessageHandler`. +- Observe requests, response attempts, and checkpoint JSON as durable outcomes. + +- [x] Test complete newline-delimited events are posted, malformed records are skipped, and an incomplete trailing line remains uncheckpointed. +- [x] Test non-success API responses leave the offset unchanged and the next successful poll retries the event. +- [x] Test checkpoint recovery after restart and file rename, and restarting from the beginning after truncation/replacement. + +### Task 5: Full verification + +**Files:** +- Modify: `README.md` if the test command or SQL Server setup needs documentation. + +- [ ] Run `dotnet test CentralLogging.sln --configuration Release` with the SQL Server test connection available. Locally, 11 tests passed and 11 SQL Server cases skipped because no connection is configured. +- [x] Confirm no `Assert.*` usage remains, run `git diff --check`, and inspect test results for skipped cases. diff --git a/docs/superpowers/specs/2026-10-24-central-logs-file-upload-design.md b/docs/superpowers/specs/2026-10-24-central-logs-file-upload-design.md new file mode 100644 index 0000000..391d248 --- /dev/null +++ b/docs/superpowers/specs/2026-10-24-central-logs-file-upload-design.md @@ -0,0 +1,38 @@ +# Central Logs File Upload Design + +## Intent + +Add an option to the Central Logs UI to upload an existing structured log file into the database. Users can backfill logs through the same API key protected by the UI. + +## Goals + +- Accept the newline-delimited JSON event format already read by the Central Logging forwarder. +- Reuse existing event validation, duplicate detection, persistence, and structured-field catalog updates. +- Show counts for imported, duplicate, and invalid lines after the upload. + +## Non-goals + +- Parse plain text logs or introduce new log formats. +- Change the forwarder or the existing JSON batch ingestion contract. + +## Constraints and behavior + +- The UI keeps the configured API key in memory and sends it in `X-Log-Api-Key`. +- Uploaded files contain one JSON event per line. Malformed or invalid lines are skipped and counted; valid lines are ingested in batches no larger than the existing 500-event limit. +- Existing `eventId` deduplication remains the authority for identifying already stored events. +- The upload result reports accepted, duplicate, and invalid line totals. Failures to persist a batch are returned as upload errors. +- Search results represent the database query, not a preview limited to the most recently uploaded file. A successful upload clears the old results and prompts the user to search again. + +## Approaches + +1. **API accepts and processes the file (chosen).** Add an authenticated upload endpoint that streams/reads newline-delimited events, validates and ingests bounded batches using existing logic, and returns a summary. The UI adds a file selector, upload action, progress/status messaging, and the final counts. This keeps format handling with the ingestion service and avoids loading the whole file into browser memory. +2. **Browser parses and posts batches to the existing endpoint.** This avoids an API route change, but moves ingestion parsing and batch state into the UI and makes browser memory and retry behavior more significant for large files. + +The server-side endpoint is chosen because it places file parsing next to the existing validation and persistence logic, while keeping the UI focused on file selection and feedback. + +## Builder decisions + +- Use a multipart upload route under `/api/logs` so it receives the same API-key middleware protection. +- Process newline-delimited JSON in bounded batches and preserve the existing idempotent `eventId` behavior. +- Add the upload control to the existing Central Logs page and keep it disabled until connected. +- Update the API README with the supported file format and result behavior. diff --git a/docs/superpowers/specs/central-logging-test-coverage.md b/docs/superpowers/specs/central-logging-test-coverage.md new file mode 100644 index 0000000..b67f09c --- /dev/null +++ b/docs/superpowers/specs/central-logging-test-coverage.md @@ -0,0 +1,24 @@ +# Central Logging Test Coverage Design + +## Goal + +Expand issue #845 coverage with durable API/database assertions and focused forwarder tests, all using Shouldly. + +## Design + +- Run the API against SQL Server in GitHub Actions and host it with `WebApplicationFactory` so tests cover routing, authentication, serialization, migrations, EF Core, and SQL Server-specific queries. +- Use an isolated integration database and serialize database-mutating API tests. Reset stored rows between cases. +- Test the forwarder against temporary log files and a deterministic fake HTTP handler, asserting posted events and persisted checkpoint offsets. +- Keep the existing unit-level validation and checkpoint tests. Avoid production changes unless a test reveals a necessary seam. + +## Coverage + +- Ingestion success, duplicate IDs within and across requests, durable event and structured-field outcomes, and validation limits. +- Upload malformed/incomplete lines, generated IDs, accepted/duplicate/invalid counts, duplicate re-uploads, and partial batch failure behavior. +- Search time, service, tenant, correlation, structured field type and exact-value filters, page limits, and invalid filter inputs. +- API key enforcement, readiness, and static UI access. +- Forwarder complete newline-delimited records, incomplete trailing lines, malformed records, truncation/rotation, retry after API errors, and checkpoint recovery. + +## CI + +Run the Central Logging test project on .NET 10 with a SQL Server service container and a dedicated integration-test connection string.