diff --git a/.github/workflows/createrelease.yml b/.github/workflows/createrelease.yml index 276276b..f5782a6 100644 --- a/.github/workflows/createrelease.yml +++ b/.github/workflows/createrelease.yml @@ -158,6 +158,10 @@ jobs: Where-Object { $_.FullName -notmatch '\\Tests\\' } | Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter daily-support-check.json | + Where-Object { $_.FullName -notmatch '\\Reports\\|\\Tests\\' } | + Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter *.xml | Where-Object { $_.FullName -notmatch '\\Tests\\' } | ForEach-Object { @@ -358,6 +362,10 @@ jobs: Where-Object { $_.FullName -notmatch '\\Tests\\' } | Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter daily-support-check.json | + Where-Object { $_.FullName -notmatch '\\Reports\\|\\Tests\\' } | + Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter *.xml | Where-Object { $_.FullName -notmatch '\\Tests\\' } | ForEach-Object { diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 970b746..64aba97 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -55,6 +55,7 @@ jobs: './ScheduledTasks/Tests' './ScheduledTasks/ReplayParkedQueue/Tests' './ScheduledTasks/SettlementProcessing/Tests' + './ScheduledTasks/DailySupportCheck/Tests' ) -CI health-monitoring-tests: diff --git a/.github/workflows/pushtomain.yml b/.github/workflows/pushtomain.yml index 82c2403..a777749 100644 --- a/.github/workflows/pushtomain.yml +++ b/.github/workflows/pushtomain.yml @@ -172,6 +172,10 @@ jobs: Where-Object { $_.FullName -notmatch '\\Tests\\' } | Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter daily-support-check.json | + Where-Object { $_.FullName -notmatch '\\Reports\\|\\Tests\\' } | + Copy-Item -Destination $taskScriptDir -Force + Get-ChildItem -Path $scheduledTasksRoot -Recurse -Filter *.xml | Where-Object { $_.FullName -notmatch '\\Tests\\' } | ForEach-Object { diff --git a/.gitignore b/.gitignore index 1489dae..f629e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -360,3 +360,4 @@ MigrationBackup/ # Codex implementation worktrees .worktrees/ /MerchantPosData +/ScheduledTasks/DailySupportCheck/daily-support-check.local.json diff --git a/ScheduledTasks/DailySupportCheck/DailySupportCheck.xml b/ScheduledTasks/DailySupportCheck/DailySupportCheck.xml new file mode 100644 index 0000000..d36de6e --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/DailySupportCheck.xml @@ -0,0 +1,44 @@ + + + + 2026-09-29T09:00:00 + Transaction Processing + \Daily Support Check + + + + 2026-09-29T09:00:00 + true + + 1 + + + + + + S-1-5-18 + ServiceAccount + HighestAvailable + + + + IgnoreNew + false + false + true + true + true + true + true + false + false + PT2H + 7 + + + + C:\Program Files\PowerShell\7\pwsh.exe + -NoProfile -NonInteractive -File "C:\txnproc\transactionprocessing\powershell\Invoke-DailySupportCheck.ps1" -OutputPath "C:\txnproc\transactionprocessing\supportreports\daily" -ConfigPath "C:\txnproc\transactionprocessing\powershell\daily-support-check.json" + + + diff --git a/ScheduledTasks/DailySupportCheck/Invoke-DailySupportCheck.ps1 b/ScheduledTasks/DailySupportCheck/Invoke-DailySupportCheck.ps1 new file mode 100644 index 0000000..260f664 --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/Invoke-DailySupportCheck.ps1 @@ -0,0 +1,1198 @@ +[CmdletBinding()] +param( + [string] $OutputPath = (Join-Path $PSScriptRoot 'Reports'), + [string] $ConfigPath = (Join-Path $PSScriptRoot 'daily-support-check.json'), + [string[]] $CheckName, + [switch] $PassThru, + [switch] $Strict +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-EnvironmentPasswordSecureString { + [OutputType([System.Security.SecureString])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [char[]] $PasswordCharacters + ) + + $securePassword = [System.Security.SecureString]::new() + foreach ($character in $PasswordCharacters) { + $securePassword.AppendChar($character) + } + $securePassword.MakeReadOnly() + return $securePassword +} + +function Get-CheckResult { + [OutputType([pscustomobject])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [ValidateSet('Passed', 'Warning', 'Failed')] [string] $Status, + [Parameter(Mandatory)] [string] $Summary, + [object] $Details, + [datetime] $StartedAt = ([datetime]::UtcNow), + [datetime] $CompletedAt = ([datetime]::UtcNow), + [double] $DurationMs = 0, + [string] $ErrorMessage + ) + + [pscustomobject]@{ + Name = $Name + Status = $Status + Summary = $Summary + Details = $Details + StartedAt = $StartedAt.ToUniversalTime().ToString('o') + CompletedAt = $CompletedAt.ToUniversalTime().ToString('o') + DurationMs = [math]::Round($DurationMs, 2) + Error = $ErrorMessage + } +} + +function Invoke-SupportCheck { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $Name, + [Parameter(Mandatory)] [scriptblock] $Action, + [object] $Context + ) + + $startedAt = [datetime]::UtcNow + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + + try { + $result = & $Action $Context + if ($null -eq $result) { + throw "Check '$Name' did not return a result." + } + + $result = @($result)[-1] + if ($result.Status -notin @('Passed', 'Warning', 'Failed')) { + throw "Check '$Name' returned an invalid status." + } + + $stopwatch.Stop() + if ($result.PSObject.Properties.Name -contains 'DurationMs') { + $result.DurationMs = [math]::Round($stopwatch.Elapsed.TotalMilliseconds, 2) + } + return $result + } + catch { + $stopwatch.Stop() + return Get-CheckResult -Name $Name -Status Failed -Summary 'The check raised an exception.' -Details $_.Exception.Message -StartedAt $startedAt -CompletedAt ([datetime]::UtcNow) -DurationMs $stopwatch.Elapsed.TotalMilliseconds -ErrorMessage $_.Exception.ToString() + } +} + +function Get-OverallStatus { + [OutputType([string])] + [CmdletBinding()] + param([Parameter(Mandatory)] [object[]] $Results) + + if (@($Results | Where-Object Status -eq 'Failed').Count -gt 0) { return 'Failed' } + if (@($Results | Where-Object Status -eq 'Warning').Count -gt 0) { return 'Warning' } + return 'Passed' +} + +function Test-PowerShellRuntime { + param([object] $Context) + + Get-CheckResult -Name 'PowerShell Runtime' -Status Passed -Summary "PowerShell $($PSVersionTable.PSVersion) is available." -Details ([pscustomobject]@{ + Edition = $PSVersionTable.PSEdition + Version = $PSVersionTable.PSVersion.ToString() + }) +} + +function Test-ReportOutputDirectory { + param([object] $Context) + + $path = [string] $Context.OutputPath + New-Item -ItemType Directory -Path $path -Force | Out-Null + $probe = Join-Path $path ('.daily-support-check-write-test-' + [guid]::NewGuid().ToString('N')) + 'write-test' | Set-Content -LiteralPath $probe -Encoding UTF8 + Remove-Item -LiteralPath $probe -Force + + Get-CheckResult -Name 'Report Output Directory' -Status Passed -Summary "Report output directory is writable: $path" -Details $path +} + +function Test-TemplateConfiguration { + param([object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'Template Configuration' -Status Failed -Summary 'Configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + Get-CheckResult -Name 'Template Configuration' -Status Passed -Summary 'Template configuration is ready for additional checks.' -Details 'Replace or extend the registered checks for environment-specific support actions.' +} + +function Merge-SupportConfiguration { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [psobject] $Base, + [Parameter(Mandatory)] [psobject] $Override + ) + + foreach ($property in $Override.PSObject.Properties) { + $baseProperty = $Base.PSObject.Properties[$property.Name] + if ($null -ne $baseProperty -and $baseProperty.Value -is [pscustomobject] -and $property.Value -is [pscustomobject]) { + Merge-SupportConfiguration -Base $baseProperty.Value -Override $property.Value | Out-Null + } + else { + $Base | Add-Member -MemberType NoteProperty -Name $property.Name -Value $property.Value -Force + } + } + + $Base +} + +function Get-SupportConfiguration { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Configuration file was not found: $Path" + } + + $configuration = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + $localPath = Join-Path (Split-Path -Parent $Path) (([IO.Path]::GetFileNameWithoutExtension($Path)) + '.local.json') + if (Test-Path -LiteralPath $localPath -PathType Leaf) { + $localConfiguration = Get-Content -LiteralPath $localPath -Raw | ConvertFrom-Json + $configuration = Merge-SupportConfiguration -Base $configuration -Override $localConfiguration + } + if (-not $configuration.PSObject.Properties['ReportRetentionDays']) { + $configuration | Add-Member -MemberType NoteProperty -Name ReportRetentionDays -Value 7 + } + if ([int] $configuration.ReportRetentionDays -lt 1) { + throw 'ReportRetentionDays must be greater than zero.' + } + if ($null -eq $configuration.DiskSpace) { + throw 'Configuration must contain a DiskSpace section.' + } + + $threshold = [double] $configuration.DiskSpace.DefaultMinimumFreePercent + if ($threshold -lt 0 -or $threshold -gt 100) { + throw 'DiskSpace.DefaultMinimumFreePercent must be between 0 and 100.' + } + + foreach ($override in @($configuration.DiskSpace.DriveOverrides.PSObject.Properties)) { + $overrideValue = [double] $override.Value + if ($overrideValue -lt 0 -or $overrideValue -gt 100) { + throw "Disk-space override for '$($override.Name)' must be between 0 and 100." + } + } + + if ($configuration.PSObject.Properties['HealthMonitoring']) { + if (-not [bool] $configuration.HealthMonitoring.Enabled) { + return $configuration + } + + $healthMonitoringUri = $null + if (-not [uri]::TryCreate([string] $configuration.HealthMonitoring.BaseUrl, [UriKind]::Absolute, [ref] $healthMonitoringUri)) { + throw 'HealthMonitoring.BaseUrl must be an absolute URI.' + } + + if ([int] $configuration.HealthMonitoring.TimeoutSeconds -le 0) { + throw 'HealthMonitoring.TimeoutSeconds must be greater than zero.' + } + } + + if ($configuration.PSObject.Properties['SubscriptionService']) { + if (-not [bool] $configuration.SubscriptionService.Enabled) { + return $configuration + } + + $subscriptionServiceUri = $null + if (-not [uri]::TryCreate([string] $configuration.SubscriptionService.BaseUrl, [UriKind]::Absolute, [ref] $subscriptionServiceUri)) { + throw 'SubscriptionService.BaseUrl must be an absolute URI.' + } + + if ([int] $configuration.SubscriptionService.TimeoutSeconds -le 0) { + throw 'SubscriptionService.TimeoutSeconds must be greater than zero.' + } + } + + if ($configuration.PSObject.Properties['KurrentDbProjections']) { + if (-not [bool] $configuration.KurrentDbProjections.Enabled) { + return $configuration + } + + $kurrentDbUri = $null + if (-not [uri]::TryCreate([string] $configuration.KurrentDbProjections.BaseUrl, [UriKind]::Absolute, [ref] $kurrentDbUri)) { + throw 'KurrentDbProjections.BaseUrl must be an absolute URI.' + } + + if ([int] $configuration.KurrentDbProjections.TimeoutSeconds -le 0) { + throw 'KurrentDbProjections.TimeoutSeconds must be greater than zero.' + } + + $hasUsername = $configuration.KurrentDbProjections.PSObject.Properties['Username'] -and -not [string]::IsNullOrWhiteSpace([string] $configuration.KurrentDbProjections.Username) + $hasPasswordEnvironmentVariable = $configuration.KurrentDbProjections.PSObject.Properties['PasswordEnvironmentVariable'] -and -not [string]::IsNullOrWhiteSpace([string] $configuration.KurrentDbProjections.PasswordEnvironmentVariable) + if ($hasUsername -xor $hasPasswordEnvironmentVariable) { + throw 'KurrentDbProjections.Username and KurrentDbProjections.PasswordEnvironmentVariable must be configured together.' + } + } + + if ($configuration.PSObject.Properties['ScheduledTasks']) { + if (-not [bool] $configuration.ScheduledTasks.Enabled) { + return $configuration + } + + if ($configuration.ScheduledTasks.PSObject.Properties['Tasks']) { + foreach ($task in @($configuration.ScheduledTasks.Tasks)) { + if ([string]::IsNullOrWhiteSpace([string] $task.Name)) { + throw 'ScheduledTasks entries must specify a Name.' + } + if ($task.PSObject.Properties['ExpectedRunIntervalMinutes'] -and [double] $task.ExpectedRunIntervalMinutes -le 0) { + throw "Scheduled task '$($task.Name)' ExpectedRunIntervalMinutes must be greater than zero." + } + if ($task.PSObject.Properties['MaxLastRunAgeHours'] -and [double] $task.MaxLastRunAgeHours -le 0) { + throw "Scheduled task '$($task.Name)' MaxLastRunAgeHours must be greater than zero." + } + } + } + } + + if ($configuration.PSObject.Properties['KurrentDbWriteActivity']) { + if (-not [bool] $configuration.KurrentDbWriteActivity.Enabled) { + return $configuration + } + + $writeActivityUri = $null + if (-not [uri]::TryCreate([string] $configuration.KurrentDbWriteActivity.BaseUrl, [UriKind]::Absolute, [ref] $writeActivityUri)) { + throw 'KurrentDbWriteActivity.BaseUrl must be an absolute URI.' + } + $streamConfigurations = if ($configuration.KurrentDbWriteActivity.PSObject.Properties['Streams']) { + @($configuration.KurrentDbWriteActivity.Streams) + } + elseif ($configuration.KurrentDbWriteActivity.PSObject.Properties['StreamName']) { + @([pscustomobject]@{ + Name = $configuration.KurrentDbWriteActivity.StreamName + EventCount = $configuration.KurrentDbWriteActivity.EventCount + MaxLatestEventAgeMinutes = $configuration.KurrentDbWriteActivity.MaxLatestEventAgeMinutes + }) + } + else { + @() + } + if ($streamConfigurations.Count -eq 0) { + throw 'KurrentDbWriteActivity.Streams must contain at least one stream.' + } + foreach ($stream in $streamConfigurations) { + if ([string]::IsNullOrWhiteSpace([string] $stream.Name)) { + throw 'KurrentDbWriteActivity stream entries must specify a Name.' + } + if ([int] $stream.EventCount -le 0) { + throw "KurrentDbWriteActivity EventCount for '$($stream.Name)' must be greater than zero." + } + if ($stream.PSObject.Properties['MaxLatestEventAgeMinutes'] -and [double] $stream.MaxLatestEventAgeMinutes -le 0) { + throw "KurrentDbWriteActivity MaxLatestEventAgeMinutes for '$($stream.Name)' must be greater than zero." + } + } + } + + return $configuration +} + +function Get-DiskSpaceSnapshot { + [OutputType([System.Array])] + [CmdletBinding()] + param() + + @(Get-PSDrive -PSProvider FileSystem | Where-Object { $null -ne $_.Free } | ForEach-Object { + $freeBytes = [int64] $_.Free + $sizeBytes = $freeBytes + [int64] $_.Used + [pscustomobject]@{ + Drive = "$($_.Name):" + SizeBytes = $sizeBytes + FreeBytes = $freeBytes + } + }) +} + +function Test-DiskSpace { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'Disk Space' -Status Failed -Summary 'Disk-space configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + $settings = $Context.Configuration.DiskSpace + if (-not [bool] $settings.Enabled) { + return Get-CheckResult -Name 'Disk Space' -Status Passed -Summary 'Disk-space check is disabled by configuration.' -Details @() + } + + try { + $provider = ${function:Get-DiskSpaceSnapshot} + if ($Context.PSObject.Properties['DiskSpaceProvider']) { + $provider = $Context.DiskSpaceProvider + } + $snapshots = @(& $provider) + if ($snapshots.Count -eq 0) { + return Get-CheckResult -Name 'Disk Space' -Status Warning -Summary 'No filesystem drives were found.' -Details @() + } + + $details = foreach ($snapshot in $snapshots) { + if ([int64] $snapshot.SizeBytes -le 0) { + throw "Drive '$($snapshot.Drive)' reported an invalid size." + } + + $drive = [string] $snapshot.Drive + $threshold = [double] $settings.DefaultMinimumFreePercent + $override = $settings.DriveOverrides.PSObject.Properties[$drive] + if ($null -ne $override) { + $threshold = [double] $override.Value + } + + $freePercent = ([double] $snapshot.FreeBytes / [double] $snapshot.SizeBytes) * 100 + [pscustomobject]@{ + Drive = $drive + SizeBytes = [int64] $snapshot.SizeBytes + FreeBytes = [int64] $snapshot.FreeBytes + FreePercent = [math]::Round($freePercent, 2) + ThresholdPercent = $threshold + Status = if ($freePercent -lt $threshold) { 'Warning' } else { 'Passed' } + } + } + + $belowThreshold = @($details | Where-Object Status -eq 'Warning') + if ($belowThreshold.Count -gt 0) { + $drives = $belowThreshold.Drive -join ', ' + return Get-CheckResult -Name 'Disk Space' -Status Warning -Summary "Drive(s) below free-space threshold: $drives" -Details @($details) + } + + Get-CheckResult -Name 'Disk Space' -Status Passed -Summary "All $($details.Count) filesystem drive(s) meet the configured free-space threshold." -Details @($details) + } + catch { + Get-CheckResult -Name 'Disk Space' -Status Failed -Summary 'Disk-space inspection failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Test-HealthMonitoring { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'HealthMonitoring' -Status Failed -Summary 'HealthMonitoring configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + if (-not $Context.Configuration.PSObject.Properties['HealthMonitoring'] -or -not [bool] $Context.Configuration.HealthMonitoring.Enabled) { + return Get-CheckResult -Name 'HealthMonitoring' -Status Passed -Summary 'HealthMonitoring check is disabled by configuration.' -Details @() + } + + try { + $settings = $Context.Configuration.HealthMonitoring + $uri = ([uri]::new("$($settings.BaseUrl.TrimEnd('/'))/$($settings.ServicesPath.TrimStart('/'))")).AbsoluteUri + $timeoutSeconds = [int] $settings.TimeoutSeconds + $provider = { + param($requestUri, $requestTimeoutSeconds) + Invoke-RestMethod -Uri $requestUri -Method Get -TimeoutSec $requestTimeoutSeconds + } + if ($Context.PSObject.Properties['HealthMonitoringProvider']) { + $provider = $Context.HealthMonitoringProvider + } + + $services = @(& $provider $uri $timeoutSeconds) + if ($services.Count -eq 0) { + return Get-CheckResult -Name 'HealthMonitoring' -Status Warning -Summary 'HealthMonitoring returned no monitored services.' -Details @() + } + + $details = foreach ($service in $services) { + $status = [string] $service.Status + if ([string]::IsNullOrWhiteSpace($status)) { + $status = 'Unknown' + } + $status = (Get-Culture).TextInfo.ToTitleCase($status.ToLowerInvariant()) + [pscustomobject]@{ + ServiceId = [string] $service.ServiceId + Name = [string] $service.Name + Status = $status + LastObservedAtUtc = if ($service.PSObject.Properties['LastObservedAtUtc']) { $service.LastObservedAtUtc } else { $null } + LastError = if ($service.PSObject.Properties['LastError']) { $service.LastError } else { $null } + } + } + + $failed = @($details | Where-Object Status -in @('Unhealthy', 'Unknown')) + $warnings = @($details | Where-Object Status -eq 'Degraded') + if ($failed.Count -gt 0) { + return Get-CheckResult -Name 'HealthMonitoring' -Status Failed -Summary "$($failed.Count) monitored service(s) are unhealthy or unknown." -Details @($details) + } + if ($warnings.Count -gt 0) { + return Get-CheckResult -Name 'HealthMonitoring' -Status Warning -Summary "$($warnings.Count) monitored service(s) are degraded." -Details @($details) + } + + Get-CheckResult -Name 'HealthMonitoring' -Status Passed -Summary "All $($details.Count) monitored service(s) are healthy." -Details @($details) + } + catch { + Get-CheckResult -Name 'HealthMonitoring' -Status Failed -Summary 'HealthMonitoring endpoint query failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Test-SubscriptionService { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'Subscription Service' -Status Failed -Summary 'Subscription service configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + if (-not $Context.Configuration.PSObject.Properties['SubscriptionService'] -or -not [bool] $Context.Configuration.SubscriptionService.Enabled) { + return Get-CheckResult -Name 'Subscription Service' -Status Passed -Summary 'Subscription service check is disabled by configuration.' -Details @() + } + + try { + $settings = $Context.Configuration.SubscriptionService + $uri = ([uri]::new("$($settings.BaseUrl.TrimEnd('/'))/$($settings.StatusPath.TrimStart('/'))")).AbsoluteUri + $timeoutSeconds = [int] $settings.TimeoutSeconds + $provider = { + param($requestUri, $requestTimeoutSeconds) + Invoke-RestMethod -Uri $requestUri -Method Get -TimeoutSec $requestTimeoutSeconds + } + if ($Context.PSObject.Properties['SubscriptionServiceProvider']) { + $provider = $Context.SubscriptionServiceProvider + } + + $subscriptions = @(& $provider $uri $timeoutSeconds) + if ($subscriptions.Count -eq 1 -and $subscriptions[0].PSObject.Properties['subscriptionId'] -and $subscriptions[0].subscriptionId -is [array]) { + $wrappedSubscriptions = $subscriptions[0] + $subscriptionCount = @($wrappedSubscriptions.subscriptionId).Count + $subscriptions = for ($index = 0; $index -lt $subscriptionCount; $index++) { + $subscription = [ordered]@{} + foreach ($property in $wrappedSubscriptions.PSObject.Properties) { + $value = $property.Value + $subscription[$property.Name] = if ($value -is [array] -and $value.Count -eq $subscriptionCount) { $value[$index] } else { $value } + } + [pscustomobject] $subscription + } + } + if ($subscriptions.Count -eq 0) { + return Get-CheckResult -Name 'Subscription Service' -Status Warning -Summary 'Subscription service returned no subscriptions.' -Details @() + } + + $details = foreach ($subscription in $subscriptions) { + $isRunning = [bool] $subscription.isRunning + $parkedEventCount = if ($subscription.PSObject.Properties['parkedEventCount'] -and $null -ne $subscription.parkedEventCount) { [int64] $subscription.parkedEventCount } else { 0 } + [pscustomobject]@{ + SubscriptionId = [string] $subscription.subscriptionId + Tag = [string] $subscription.tag + IsRunning = $isRunning + Health = [string] $subscription.health + ParkedEventCount = $parkedEventCount + OperationalReason = if ($subscription.PSObject.Properties['operationalReason']) { $subscription.operationalReason } else { $null } + RuntimeFailureReason = if ($subscription.PSObject.Properties['runtimeFailureReason']) { $subscription.runtimeFailureReason } else { $null } + Status = if (-not $isRunning) { 'Failed' } elseif ($parkedEventCount -gt 0) { 'Warning' } else { 'Passed' } + } + } + + $stopped = @($details | Where-Object Status -eq 'Failed') + $parked = @($details | Where-Object Status -eq 'Warning') + if ($stopped.Count -gt 0) { + return Get-CheckResult -Name 'Subscription Service' -Status Failed -Summary "$($stopped.Count) subscription(s) are not running." -Details @($details) + } + if ($parked.Count -gt 0) { + return Get-CheckResult -Name 'Subscription Service' -Status Warning -Summary "$($parked.Count) subscription(s) have parked messages." -Details @($details) + } + + Get-CheckResult -Name 'Subscription Service' -Status Passed -Summary "All $($details.Count) subscription(s) are running without parked messages." -Details @($details) + } + catch { + Get-CheckResult -Name 'Subscription Service' -Status Failed -Summary 'Subscription status endpoint query failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Test-KurrentDbProjections { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'KurrentDB Projections' -Status Failed -Summary 'KurrentDB projection configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + if (-not $Context.Configuration.PSObject.Properties['KurrentDbProjections'] -or -not [bool] $Context.Configuration.KurrentDbProjections.Enabled) { + return Get-CheckResult -Name 'KurrentDB Projections' -Status Passed -Summary 'KurrentDB projection check is disabled by configuration.' -Details @() + } + + try { + $settings = $Context.Configuration.KurrentDbProjections + $configuredNames = @($settings.ProjectionNames | ForEach-Object { [string] $_ } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($configuredNames.Count -eq 0) { + return Get-CheckResult -Name 'KurrentDB Projections' -Status Warning -Summary 'No KurrentDB projections are configured for checking.' -Details @() + } + + $uri = ([uri]::new("$($settings.BaseUrl.TrimEnd('/'))/$($settings.ProjectionsPath.TrimStart('/'))")).AbsoluteUri + $timeoutSeconds = [int] $settings.TimeoutSeconds + $provider = { + param($requestUri, $requestTimeoutSeconds) + $request = @{ Uri = $requestUri; Method = 'Get'; TimeoutSec = $requestTimeoutSeconds } + $hasUsername = $settings.PSObject.Properties['Username'] -and -not [string]::IsNullOrWhiteSpace([string] $settings.Username) + if ($hasUsername) { + $passwordEnvironmentVariable = [string] $settings.PasswordEnvironmentVariable + $password = [Environment]::GetEnvironmentVariable($passwordEnvironmentVariable) + if ([string]::IsNullOrWhiteSpace($password)) { + throw "KurrentDB password environment variable '$passwordEnvironmentVariable' is not set." + } + $securePassword = Get-EnvironmentPasswordSecureString -PasswordCharacters $password.ToCharArray() + $request.Authentication = 'Basic' + $request.Credential = [pscredential]::new([string] $settings.Username, $securePassword) + } + Invoke-RestMethod @request + } + if ($Context.PSObject.Properties['KurrentDbProjectionProvider']) { + $provider = $Context.KurrentDbProjectionProvider + } + + $rawResponse = @(& $provider $uri $timeoutSeconds) + $projections = if ($rawResponse.Count -eq 1 -and $rawResponse[0].PSObject.Properties['projections']) { + @($rawResponse[0].projections) + } + else { + $rawResponse + } + $projectionByName = @{} + foreach ($projection in $projections) { + $name = if ($projection.PSObject.Properties['effectiveName']) { [string] $projection.effectiveName } elseif ($projection.PSObject.Properties['name']) { [string] $projection.name } else { '' } + if (-not [string]::IsNullOrWhiteSpace($name)) { + $projectionByName[$name] = $projection + } + } + + $details = foreach ($configuredName in $configuredNames) { + $projection = $projectionByName[$configuredName] + if ($null -eq $projection) { + [pscustomobject]@{ Name = $configuredName; Status = 'Missing'; Progress = $null; StateReason = 'Projection was not returned by KurrentDB.' } + continue + } + + [pscustomobject]@{ + Name = $configuredName + Status = [string] $projection.status + Progress = if ($projection.PSObject.Properties['progress']) { $projection.progress } else { $null } + StateReason = if ($projection.PSObject.Properties['stateReason']) { $projection.stateReason } else { $null } + } + } + + $resultDetails = if ($settings.PSObject.Properties['Debug'] -and [bool] $settings.Debug) { + [pscustomobject]@{ + Projections = @($details) + RawResponse = @($rawResponse) + } + } + else { + @($details) + } + + $notRunning = @($details | Where-Object { $_.Status -ne 'Running' }) + if ($notRunning.Count -gt 0) { + $summary = ($notRunning | ForEach-Object { "$($_.Name) [$($_.Status)]" }) -join ', ' + return Get-CheckResult -Name 'KurrentDB Projections' -Status Failed -Summary "Projection(s) not running: $summary" -Details $resultDetails + } + + Get-CheckResult -Name 'KurrentDB Projections' -Status Passed -Summary "All $($details.Count) configured KurrentDB projection(s) are running." -Details $resultDetails + } + catch { + Get-CheckResult -Name 'KurrentDB Projections' -Status Failed -Summary 'KurrentDB projection status query failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Test-ScheduledTasks { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'Scheduled Tasks' -Status Failed -Summary 'Scheduled-task configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + if (-not $Context.Configuration.PSObject.Properties['ScheduledTasks'] -or -not [bool] $Context.Configuration.ScheduledTasks.Enabled) { + return Get-CheckResult -Name 'Scheduled Tasks' -Status Passed -Summary 'Scheduled-task check is disabled by configuration.' -Details @() + } + + $taskConfigurations = @($Context.Configuration.ScheduledTasks.Tasks) + if ($taskConfigurations.Count -eq 0) { + return Get-CheckResult -Name 'Scheduled Tasks' -Status Warning -Summary 'No scheduled tasks are configured for checking.' -Details @() + } + + try { + $provider = { + param($taskConfiguration) + $task = Get-ScheduledTask -TaskName $taskConfiguration.Name -TaskPath $taskConfiguration.Path -ErrorAction Stop + $info = Get-ScheduledTaskInfo -TaskName $taskConfiguration.Name -TaskPath $taskConfiguration.Path -ErrorAction Stop + [pscustomobject]@{ + Name = $task.TaskName + Path = $task.TaskPath + State = [string] $task.State + LastTaskResult = $info.LastTaskResult + LastRunTime = $info.LastRunTime + NextRunTime = $info.NextRunTime + } + } + if ($Context.PSObject.Properties['ScheduledTaskProvider']) { + $provider = $Context.ScheduledTaskProvider + } + + $now = [datetime]::Now + $details = foreach ($taskConfiguration in $taskConfigurations) { + try { + $task = & $provider $taskConfiguration + $state = [string] $task.State + $lastTaskResult = [int64] $task.LastTaskResult + $lastRunTime = if ($task.PSObject.Properties['LastRunTime']) { $task.LastRunTime } else { $null } + $nextRunTime = if ($task.PSObject.Properties['NextRunTime']) { $task.NextRunTime } else { $null } + $maxLastRunAgeMinutes = if ($taskConfiguration.PSObject.Properties['ExpectedRunIntervalMinutes']) { + [double] $taskConfiguration.ExpectedRunIntervalMinutes + } + elseif ($taskConfiguration.PSObject.Properties['MaxLastRunAgeHours']) { + [double] $taskConfiguration.MaxLastRunAgeHours * 60 + } + else { + $null + } + $failureReasons = [System.Collections.Generic.List[string]]::new() + + if ($state -eq 'Disabled') { $failureReasons.Add('Disabled') } + if ($lastTaskResult -ne 0) { $failureReasons.Add("LastTaskResult=$lastTaskResult") } + if ($null -eq $lastRunTime) { + $failureReasons.Add('No successful run recorded') + } + elseif ($null -ne $maxLastRunAgeMinutes -and $lastRunTime -lt $now.AddMinutes(-$maxLastRunAgeMinutes)) { + $failureReasons.Add('Last run is too old') + } + if ($null -ne $nextRunTime -and $nextRunTime -lt $now -and $state -ne 'Running') { + $failureReasons.Add('Next run is overdue') + } + if ($state -notin @('Ready', 'Running', 'Disabled')) { + $failureReasons.Add("Unexpected state '$state'") + } + + [pscustomobject]@{ + Name = [string] $taskConfiguration.Name + Path = [string] $taskConfiguration.Path + State = $state + LastTaskResult = $lastTaskResult + LastRunTime = $lastRunTime + NextRunTime = $nextRunTime + MaxLastRunAgeMinutes = $maxLastRunAgeMinutes + Status = if ($failureReasons.Count -gt 0) { 'Failed' } else { 'Passed' } + FailureReason = if ($failureReasons.Count -gt 0) { $failureReasons -join '; ' } else { $null } + } + } + catch { + [pscustomobject]@{ + Name = [string] $taskConfiguration.Name + Path = [string] $taskConfiguration.Path + State = 'Missing' + LastTaskResult = $null + LastRunTime = $null + NextRunTime = $null + Status = 'Failed' + FailureReason = $_.Exception.Message + } + } + } + + $failed = @($details | Where-Object Status -eq 'Failed') + if ($failed.Count -gt 0) { + $summary = ($failed | ForEach-Object { "$($_.Name) [$($_.FailureReason)]" }) -join ', ' + return Get-CheckResult -Name 'Scheduled Tasks' -Status Failed -Summary "Scheduled task(s) require attention: $summary" -Details @($details) + } + + Get-CheckResult -Name 'Scheduled Tasks' -Status Passed -Summary "All $($details.Count) configured scheduled task(s) are healthy." -Details @($details) + } + catch { + Get-CheckResult -Name 'Scheduled Tasks' -Status Failed -Summary 'Scheduled-task status query failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Import-KurrentDbClientAssembly { + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $AssemblyPath) + + if ([AppDomain]::CurrentDomain.GetAssemblies() | Where-Object { $_.GetName().Name -eq 'KurrentDB.Client' }) { + return + } + + if (-not (Test-Path -LiteralPath $AssemblyPath -PathType Leaf)) { + throw "KurrentDB client assembly was not found: $AssemblyPath" + } + + $frameworkRoot = Join-Path ([Environment]::GetFolderPath('ProgramFiles')) 'dotnet\shared\Microsoft.AspNetCore.App' + $frameworkDirectory = if (Test-Path -LiteralPath $frameworkRoot) { + @( + Get-ChildItem -LiteralPath $frameworkRoot -Directory | Where-Object Name -like '9.*' | Sort-Object Name -Descending + Get-ChildItem -LiteralPath $frameworkRoot -Directory | Sort-Object Name -Descending + ) | Select-Object -First 1 + } + + foreach ($dependencyName in @( + 'Microsoft.Extensions.DependencyInjection.Abstractions.dll', + 'Microsoft.Extensions.Logging.Abstractions.dll', + 'Microsoft.Extensions.Options.dll' + )) { + if ($frameworkDirectory) { + $dependencyPath = Join-Path $frameworkDirectory.FullName $dependencyName + if (Test-Path -LiteralPath $dependencyPath -PathType Leaf) { + Add-Type -Path $dependencyPath + } + } + } + + Add-Type -Path $AssemblyPath +} + +function Test-KurrentDbWriteActivity { + [CmdletBinding()] + param([Parameter(Mandatory)] [object] $Context) + + if ($Context.PSObject.Properties['ConfigurationError'] -and $Context.ConfigurationError) { + return Get-CheckResult -Name 'KurrentDB Write Activity' -Status Failed -Summary 'KurrentDB write-activity configuration could not be loaded.' -Details $Context.ConfigurationError -ErrorMessage $Context.ConfigurationError + } + + if (-not $Context.Configuration.PSObject.Properties['KurrentDbWriteActivity'] -or -not [bool] $Context.Configuration.KurrentDbWriteActivity.Enabled) { + return Get-CheckResult -Name 'KurrentDB Write Activity' -Status Passed -Summary 'KurrentDB write-activity check is disabled by configuration.' -Details @() + } + + try { + $settings = $Context.Configuration.KurrentDbWriteActivity + $streamConfigurations = if ($settings.PSObject.Properties['Streams']) { + @($settings.Streams) + } + elseif ($settings.PSObject.Properties['StreamName']) { + @([pscustomobject]@{ Name = $settings.StreamName; EventCount = $settings.EventCount; MaxLatestEventAgeMinutes = $settings.MaxLatestEventAgeMinutes }) + } + else { + @() + } + if ($streamConfigurations.Count -eq 0) { + return Get-CheckResult -Name 'KurrentDB Write Activity' -Status Warning -Summary 'No KurrentDB streams are configured for write-activity checking.' -Details @() + } + + $timeoutSeconds = if ($settings.PSObject.Properties['TimeoutSeconds']) { [int] $settings.TimeoutSeconds } else { 10 } + $assemblyPath = if ($settings.PSObject.Properties['ClientAssemblyPath'] -and -not [string]::IsNullOrWhiteSpace([string] $settings.ClientAssemblyPath)) { + [string] $settings.ClientAssemblyPath + } + else { + Join-Path $PSScriptRoot '..\..\StreamManagementTool\bin\Debug\net10.0\KurrentDB.Client.dll' + } + if (-not [IO.Path]::IsPathRooted($assemblyPath)) { + $assemblyPath = [IO.Path]::GetFullPath((Join-Path (Split-Path -Parent $Context.ConfigPath) $assemblyPath)) + } + + $provider = { + param($indexName, $requestCount) + Import-KurrentDbClientAssembly -AssemblyPath $assemblyPath + + $baseUri = [uri] $settings.BaseUrl + $port = if ($baseUri.IsDefaultPort) { 2113 } else { $baseUri.Port } + $connectionString = if ($settings.PSObject.Properties['ConnectionString'] -and -not [string]::IsNullOrWhiteSpace([string] $settings.ConnectionString)) { + [string] $settings.ConnectionString + } + else { + "esdb://$($baseUri.Host):${port}?tls=false&tlsVerifyCert=false" + } + + $client = [KurrentDB.Client.KurrentDBClient]::new([KurrentDB.Client.KurrentDBClientSettings]::Create($connectionString)) + $cancellationSource = [Threading.CancellationTokenSource]::new() + $cancellationSource.CancelAfter([timespan]::FromSeconds($timeoutSeconds)) + $userCredentials = $null + if ($settings.PSObject.Properties['Username'] -and -not [string]::IsNullOrWhiteSpace([string] $settings.Username)) { + $passwordEnvironmentVariable = [string] $settings.PasswordEnvironmentVariable + $password = [Environment]::GetEnvironmentVariable($passwordEnvironmentVariable) + if ([string]::IsNullOrWhiteSpace($password)) { + throw "KurrentDB password environment variable '$passwordEnvironmentVariable' is not set." + } + $userCredentials = [KurrentDB.Client.UserCredentials]::new([string] $settings.Username, $password) + } + + try { + $filter = [KurrentDB.Client.StreamFilter]::Prefix($indexName) + $read = $client.ReadAllAsync( + [KurrentDB.Client.Direction]::Backwards, + [KurrentDB.Client.Position]::End, + $filter, + $requestCount, + $false, + [timespan]::FromSeconds($timeoutSeconds), + $userCredentials, + $cancellationSource.Token + ) + $enumerator = $read.GetAsyncEnumerator() + $events = [System.Collections.Generic.List[object]]::new() + try { + while ($enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) { + $currentEvent = $enumerator.Current.Event + $events.Add([pscustomobject]@{ + EventId = [string] $currentEvent.EventId + EventType = [string] $currentEvent.EventType + Timestamp = $currentEvent.Created.ToUniversalTime().ToString('o') + Title = [string] $currentEvent.EventStreamId + }) + } + } + finally { + $enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult() | Out-Null + } + + [pscustomobject]@{ events = @($events) } + } + finally { + $cancellationSource.Dispose() + $client.Dispose() + } + } + if ($Context.PSObject.Properties['KurrentDbWriteActivityProvider']) { + $provider = $Context.KurrentDbWriteActivityProvider + } + + $details = foreach ($stream in $streamConfigurations) { + $streamName = [string] $stream.Name + $eventCount = [int] $stream.EventCount + try { + $response = & $provider $streamName $eventCount + $entries = if ($response.PSObject.Properties['events']) { @($response.events) } elseif ($response.PSObject.Properties['entries']) { @($response.entries) } else { @($response) } + $events = foreach ($entry in $entries) { + $updated = $null + $timestampValue = if ($entry.PSObject.Properties['Timestamp']) { $entry.Timestamp } elseif ($entry.PSObject.Properties['Created']) { $entry.Created } elseif ($entry.PSObject.Properties['updated']) { $entry.updated } else { $null } + if ($null -ne $timestampValue -and -not [string]::IsNullOrWhiteSpace([string] $timestampValue)) { + $updated = [datetime]::Parse([string] $timestampValue).ToUniversalTime() + } + [pscustomobject]@{ + EventId = if ($entry.PSObject.Properties['EventId']) { [string] $entry.EventId } elseif ($entry.PSObject.Properties['id']) { [string] $entry.id } else { [string] $entry.title } + EventType = if ($entry.PSObject.Properties['EventType']) { [string] $entry.EventType } elseif ($entry.PSObject.Properties['summary']) { [string] $entry.summary } else { $null } + Timestamp = $updated + Title = if ($entry.PSObject.Properties['Title']) { [string] $entry.Title } elseif ($entry.PSObject.Properties['title']) { [string] $entry.title } else { $null } + } + } + $latest = @($events | Where-Object { $null -ne $_.Timestamp } | Sort-Object Timestamp -Descending | Select-Object -First 1) + $status = 'Passed' + $failureReason = $null + if ($events.Count -eq 0) { + $status = 'Failed'; $failureReason = 'No events were found.' + } + elseif ($latest.Count -eq 0) { + $status = 'Failed'; $failureReason = 'Events did not contain timestamps.' + } + elseif ($stream.PSObject.Properties['MaxLatestEventAgeMinutes'] -and $latest[0].Timestamp -lt [datetime]::UtcNow.AddMinutes(-[double] $stream.MaxLatestEventAgeMinutes)) { + $status = 'Failed'; $failureReason = 'Latest event is older than the configured age limit.' + } + elseif ($events.Count -lt $eventCount) { + $status = 'Warning'; $failureReason = "$($events.Count) event(s) returned; $eventCount requested." + } + + [pscustomobject]@{ StreamName = $streamName; Status = $status; EventCount = $events.Count; LatestEvent = $latest | Select-Object -First 1; FailureReason = $failureReason; Events = @($events) } + } + catch { + [pscustomobject]@{ StreamName = $streamName; Status = 'Failed'; EventCount = 0; LatestEvent = $null; FailureReason = $_.Exception.Message; Events = @() } + } + } + + $failed = @($details | Where-Object Status -eq 'Failed') + $warnings = @($details | Where-Object Status -eq 'Warning') + if ($failed.Count -gt 0) { + $summary = ($failed | ForEach-Object { "$($_.StreamName) [$($_.FailureReason)]" }) -join ', ' + return Get-CheckResult -Name 'KurrentDB Write Activity' -Status Failed -Summary "KurrentDB stream(s) require attention: $summary" -Details @($details) -ErrorMessage $summary + } + if ($warnings.Count -gt 0) { + $summary = ($warnings | ForEach-Object { "$($_.StreamName) [$($_.FailureReason)]" }) -join ', ' + return Get-CheckResult -Name 'KurrentDB Write Activity' -Status Warning -Summary "KurrentDB stream(s) returned fewer events than requested: $summary" -Details @($details) + } + + Get-CheckResult -Name 'KurrentDB Write Activity' -Status Passed -Summary "All $($details.Count) configured KurrentDB stream(s) contain recent events." -Details @($details) + } + catch { + Get-CheckResult -Name 'KurrentDB Write Activity' -Status Failed -Summary 'KurrentDB write-activity query failed.' -Details $_.Exception.Message -ErrorMessage $_.Exception.ToString() + } +} + +function Get-SupportCheckDefinitions { + @( + [pscustomobject]@{ Name = 'PowerShell Runtime'; Action = ${function:Test-PowerShellRuntime} } + [pscustomobject]@{ Name = 'Report Output Directory'; Action = ${function:Test-ReportOutputDirectory} } + [pscustomobject]@{ Name = 'Disk Space'; Action = ${function:Test-DiskSpace} } + [pscustomobject]@{ Name = 'HealthMonitoring'; Action = ${function:Test-HealthMonitoring} } + [pscustomobject]@{ Name = 'Subscription Service'; Action = ${function:Test-SubscriptionService} } + [pscustomobject]@{ Name = 'KurrentDB Projections'; Action = ${function:Test-KurrentDbProjections} } + [pscustomobject]@{ Name = 'Scheduled Tasks'; Action = ${function:Test-ScheduledTasks} } + [pscustomobject]@{ Name = 'KurrentDB Write Activity'; Action = ${function:Test-KurrentDbWriteActivity} } + [pscustomobject]@{ Name = 'Template Configuration'; Action = ${function:Test-TemplateConfiguration} } + ) +} + +function Get-SupportReport { + [OutputType([pscustomobject])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [object[]] $Results + ) + + $completedAt = [datetime]::UtcNow + $status = Get-OverallStatus -Results $Results + [pscustomobject]@{ + ReportName = 'Daily Support Check' + RunId = [guid]::NewGuid().ToString() + StartedAt = $StartedAt.ToUniversalTime().ToString('o') + CompletedAt = $completedAt.ToUniversalTime().ToString('o') + DurationMs = [math]::Round(($completedAt - $StartedAt).TotalMilliseconds, 2) + OverallStatus = $status + Counts = [pscustomobject]@{ + Passed = @($Results | Where-Object Status -eq 'Passed').Count + Warning = @($Results | Where-Object Status -eq 'Warning').Count + Failed = @($Results | Where-Object Status -eq 'Failed').Count + } + Checks = @($Results) + } +} + +function ConvertTo-ReportText { + param([object] $Value) + + if ($null -eq $Value) { return '' } + if ($Value -is [string]) { return $Value } + return ($Value | ConvertTo-Json -Depth 8 -Compress) +} + +function Write-SupportReports { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object] $Report, + [Parameter(Mandatory)] [string] $OutputPath, + [int] $RetentionDays = 7 + ) + + if ($RetentionDays -lt 1) { + throw 'RetentionDays must be greater than zero.' + } + + New-Item -ItemType Directory -Path $OutputPath -Force | Out-Null + $stamp = [datetime]::UtcNow.ToString('yyyyMMdd-HHmmss-fff') + $baseName = "daily-support-check-$stamp" + $jsonPath = Join-Path $OutputPath "$baseName.json" + $htmlPath = Join-Path $OutputPath "$baseName.html" + + $Report | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $jsonPath -Encoding UTF8 + + $rows = [System.Text.StringBuilder]::new() + foreach ($check in @($Report.Checks)) { + $status = [System.Net.WebUtility]::HtmlEncode([string] $check.Status) + $name = [System.Net.WebUtility]::HtmlEncode([string] $check.Name) + $summaryText = [string] $check.Summary + if ($summaryText.Length -gt 160) { + $summaryText = $summaryText.Substring(0, 157).TrimEnd() + '…' + } + $summary = [System.Net.WebUtility]::HtmlEncode($summaryText) + $statusIcon = switch ([string] $check.Status) { + 'Passed' { '🟢' } + 'Warning' { '🟠' } + 'Failed' { '🔴' } + default { '⚪' } + } + [void] $rows.AppendLine("$name$statusIcon $status$summary") + } + + $overallStatus = [System.Net.WebUtility]::HtmlEncode([string] $Report.OverallStatus) + $html = @" + + +Daily Support Check + + +

Daily Support Check

Overall status: $overallStatus

+

Started: $([System.Net.WebUtility]::HtmlEncode([string] $Report.StartedAt))
Completed: $([System.Net.WebUtility]::HtmlEncode([string] $Report.CompletedAt))

+$rows
CheckStatusSummary
+ +"@ + $html | Set-Content -LiteralPath $htmlPath -Encoding UTF8 + + $cutoffUtc = [datetime]::UtcNow.AddDays(-$RetentionDays) + Get-ChildItem -LiteralPath $OutputPath -File | Where-Object { + $_.Name -like 'daily-support-check-*.json' -or $_.Name -like 'daily-support-check-*.html' + } | Where-Object { $_.LastWriteTimeUtc -lt $cutoffUtc } | Remove-Item -Force + + [pscustomobject]@{ JsonPath = $jsonPath; HtmlPath = $htmlPath } +} + +function Send-SupportReport { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object] $Report, + [Parameter(Mandatory)] [object] $ReportPaths, + [object] $Configuration, + [scriptblock] $EmailProvider + ) + + $email = if ($Configuration -and $Configuration.PSObject.Properties['Email']) { $Configuration.Email } else { $null } + if ($null -eq $email -or -not [bool] $email.Enabled) { + return [pscustomobject]@{ + Transport = 'LocalFiles' + Status = 'Ready' + Paths = $ReportPaths + Message = 'Reports are available as local files. Email delivery is disabled.' + } + } + + try { + $providerName = if ([string]::IsNullOrWhiteSpace([string] $email.Provider)) { 'Brevo' } else { [string] $email.Provider } + if ($providerName -ne 'Brevo') { + throw "Unsupported email provider '$providerName'." + } + + foreach ($required in @('ApiUrl', 'ApiKey', 'From')) { + if ([string]::IsNullOrWhiteSpace([string] $email.$required)) { + throw "Email.$required must be configured when email delivery is enabled." + } + } + + $recipients = @($email.To | ForEach-Object { [string] $_ } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($recipients.Count -eq 0) { + throw 'Email.To must contain at least one recipient when email delivery is enabled.' + } + + $counts = if ($Report.Counts) { $Report.Counts } else { [pscustomobject]@{ Passed = 0; Warning = 0; Failed = 0 } } + $textLines = @( + 'Daily Support Check' + "Overall status: $($Report.OverallStatus)" + "Passed: $($counts.Passed); Warning: $($counts.Warning); Failed: $($counts.Failed)" + ) + foreach ($check in @($Report.Checks | Where-Object { $_.Status -ne 'Passed' })) { + $textLines += "[$($check.Status)] $($check.Name): $($check.Summary)" + if ($check.Error) { $textLines += "Error: $($check.Error)" } + } + $textBody = $textLines -join [Environment]::NewLine + + $htmlBody = $null + if ($ReportPaths.HtmlPath -and (Test-Path -LiteralPath $ReportPaths.HtmlPath)) { + $htmlBody = Get-Content -LiteralPath $ReportPaths.HtmlPath -Raw + } + if ([string]::IsNullOrWhiteSpace($htmlBody)) { + $htmlBody = "

Daily Support Check

Overall status: $([System.Net.WebUtility]::HtmlEncode([string] $Report.OverallStatus))

" + } + + $attachments = @() + $attachHtmlReport = if ($email.PSObject.Properties['AttachHtmlReport']) { [bool] $email.AttachHtmlReport } else { $false } + $attachJsonReport = if ($email.PSObject.Properties['AttachJsonReport']) { [bool] $email.AttachJsonReport } else { $false } + $attachmentDefinitions = @( + [pscustomobject]@{ Enabled = $attachHtmlReport; Path = $ReportPaths.HtmlPath; MimeType = 'text/html' } + [pscustomobject]@{ Enabled = $attachJsonReport; Path = $ReportPaths.JsonPath; MimeType = 'application/json' } + ) + foreach ($definition in $attachmentDefinitions) { + if ($definition.Enabled -and $definition.Path -and (Test-Path -LiteralPath $definition.Path)) { + $attachments += [pscustomobject]@{ + name = [IO.Path]::GetFileName($definition.Path) + content = [Convert]::ToBase64String([IO.File]::ReadAllBytes($definition.Path)) + } + } + } + + $configuredSubjectPrefix = if ($email.PSObject.Properties['SubjectPrefix']) { [string] $email.SubjectPrefix } else { '' } + $subjectPrefix = if ([string]::IsNullOrWhiteSpace($configuredSubjectPrefix)) { 'Daily Support Check' } else { $configuredSubjectPrefix } + $serverName = if ([string]::IsNullOrWhiteSpace($env:COMPUTERNAME)) { 'Unknown Server' } else { $env:COMPUTERNAME } + $runDate = try { ([datetime]::Parse([string] $Report.StartedAt)).ToString('yyyy-MM-dd') } catch { (Get-Date).ToString('yyyy-MM-dd') } + $payload = [ordered]@{ + sender = [ordered]@{ email = [string] $email.From } + to = @($recipients | ForEach-Object { [ordered]@{ email = $_ } }) + subject = "$subjectPrefix - $($Report.OverallStatus) - $serverName - $runDate" + textContent = $textBody + htmlContent = $htmlBody + } + if ($attachments.Count -gt 0) { $payload.attachments = $attachments } + + if (-not $EmailProvider) { + $EmailProvider = { + param($requestUri, $headers, $requestPayload) + Invoke-RestMethod -Uri $requestUri -Method Post -Headers $headers -ContentType 'application/json' -Body ($requestPayload | ConvertTo-Json -Depth 12) + } + } + + $headers = @{ + 'api-key' = [string] $email.ApiKey + Accept = 'application/json' + 'Content-Type' = 'application/json' + } + $response = & $EmailProvider ([string] $email.ApiUrl) $headers $payload + + [pscustomobject]@{ + Transport = 'Brevo' + Status = 'Sent' + Paths = $ReportPaths + Message = 'Report sent via Brevo.' + } + } + catch { + [pscustomobject]@{ + Transport = 'Brevo' + Status = 'Failed' + Paths = $ReportPaths + Message = $_.Exception.Message + } + } +} + +function Invoke-DailySupportCheck { + [CmdletBinding()] + param( + [string] $OutputPath = (Join-Path $PSScriptRoot 'Reports'), + [string] $ConfigPath = (Join-Path $PSScriptRoot 'daily-support-check.json'), + [string[]] $CheckName, + [switch] $PassThru, + [switch] $Strict + ) + + $startedAt = [datetime]::UtcNow + $configuration = $null + $configurationError = $null + try { + $configuration = Get-SupportConfiguration -Path $ConfigPath + } + catch { + $configurationError = $_.Exception.Message + $configuration = [pscustomobject]@{ + ReportRetentionDays = 7 + DiskSpace = [pscustomobject]@{ + Enabled = $false + DefaultMinimumFreePercent = 15 + DriveOverrides = [pscustomobject]@{} + } + } + } + + $context = [pscustomobject]@{ OutputPath = $OutputPath; ConfigPath = $ConfigPath; Configuration = $configuration; ConfigurationError = $configurationError; StartedAt = $startedAt } + $definitions = @(Get-SupportCheckDefinitions) + if ($CheckName) { + $definitions = @($definitions | Where-Object Name -in $CheckName) + $unknown = @($CheckName | Where-Object { $_ -notin $definitions.Name }) + if ($unknown.Count -gt 0) { + throw "Unknown check name(s): $($unknown -join ', ')" + } + } + + $results = foreach ($definition in $definitions) { + Invoke-SupportCheck -Name $definition.Name -Action $definition.Action -Context $context + } + + $report = Get-SupportReport -StartedAt $startedAt -Results @($results) + $paths = Write-SupportReports -Report $report -OutputPath $OutputPath -RetentionDays ([int] $configuration.ReportRetentionDays) + $transport = Send-SupportReport -Report $report -ReportPaths $paths -Configuration $configuration + $report | Add-Member -NotePropertyName ReportPaths -NotePropertyValue $paths + $report | Add-Member -NotePropertyName Transport -NotePropertyValue $transport + + Write-Information "Daily Support Check: $($report.OverallStatus) ($($report.Checks.Count) checks)" -InformationAction Continue + Write-Information "JSON report: $($paths.JsonPath)" -InformationAction Continue + Write-Information "HTML report: $($paths.HtmlPath)" -InformationAction Continue + + if ($PassThru) { return $report } +} + +$isDotSourced = $MyInvocation.InvocationName -eq '.' +if (-not $isDotSourced) { + try { + $report = Invoke-DailySupportCheck -OutputPath $OutputPath -ConfigPath $ConfigPath -CheckName $CheckName -PassThru -Strict:$Strict + if ($report.OverallStatus -eq 'Failed' -or ($Strict -and $report.OverallStatus -eq 'Warning')) { + exit 1 + } + exit 0 + } + catch { + Write-Error $_ + exit 1 + } +} diff --git a/ScheduledTasks/DailySupportCheck/README.md b/ScheduledTasks/DailySupportCheck/README.md new file mode 100644 index 0000000..2f58bb3 --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/README.md @@ -0,0 +1,199 @@ +# Daily Support Check + +`Invoke-DailySupportCheck.ps1` is a PowerShell template for a daily operational support run. It executes independent checks, reads operational settings from JSON, writes a structured JSON report and a readable HTML report, and leaves report delivery behind a replaceable transport function. + +## Run it interactively + +From the repository root: + +```powershell +pwsh -NoProfile -File .\ScheduledTasks\DailySupportCheck\Invoke-DailySupportCheck.ps1 +``` + +Choose an output directory, configuration file, or subset of checks: + +```powershell +pwsh -NoProfile -File .\ScheduledTasks\DailySupportCheck\Invoke-DailySupportCheck.ps1 ` + -OutputPath 'C:\SupportReports\Daily' ` + -ConfigPath 'C:\SupportConfig\daily-support-check.json' ` + -CheckName 'PowerShell Runtime', 'Report Output Directory' +``` + +Use `-PassThru` when calling the script from another PowerShell script and you need the report object. Use `-Strict` when warnings should produce a non-zero process exit code. Failed checks always produce a non-zero exit code. + +The default configuration file is `daily-support-check.json` beside the script: + +```json +{ + "DiskSpace": { + "Enabled": true, + "DefaultMinimumFreePercent": 15, + "DriveOverrides": { + "C:": 10, + "D:": 20 + } + }, + "HealthMonitoring": { + "Enabled": true, + "BaseUrl": "http://localhost:9620", + "ServicesPath": "/api/services", + "TimeoutSeconds": 10 + }, + "SubscriptionService": { + "Enabled": true, + "BaseUrl": "http://localhost:8080", + "StatusPath": "/subscriptions/status", + "TimeoutSeconds": 10 + }, + "KurrentDbProjections": { + "Enabled": true, + "Debug": false, + "BaseUrl": "http://localhost:2113", + "ProjectionsPath": "/projections/any", + "ProjectionNames": [ + "MerchantBalanceProjection" + ], + "TimeoutSeconds": 10 + }, + "ScheduledTasks": { + "Enabled": true, + "Tasks": [ + { + "Name": "Daily Settlement", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Replay Parked Queue", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Scavenge", + "Path": "\\", + "ExpectedRunIntervalMinutes": 10080 + } + ] + }, + "KurrentDbWriteActivity": { + "Enabled": true, + "BaseUrl": "http://localhost:2113", + "ClientAssemblyPath": "..\\..\\StreamManagementTool\\bin\\Debug\\net10.0\\KurrentDB.Client.dll", + "Streams": [ + { + "Name": "$idx-ce-TransactionAggregate", + "EventCount": 10, + "MaxLatestEventAgeMinutes": 15 + } + ] + } +} +``` + +If `daily-support-check.local.json` exists beside the base configuration, it is loaded automatically after the base file and its values override or extend the base configuration. Use this ignored local file for machine-specific settings and secrets. A starter example is provided as `daily-support-check.local.example.json`; copy it to `daily-support-check.local.json` and replace the placeholder values locally. The local file is excluded from source control. + +`ReportRetentionDays` controls how long generated JSON and HTML reports are kept. It defaults to `7` and only matching `daily-support-check-*` report files are removed. `DefaultMinimumFreePercent` is the minimum free-space percentage applied to every filesystem drive. An entry in `DriveOverrides` replaces that threshold for the matching drive. Low free space produces a `Warning`; a configuration or inspection error produces a `Failed` result. + +The `HealthMonitoring` check calls the configured services endpoint. All services must be `Healthy` for the check to pass. `Degraded` produces a `Warning`; `Unhealthy`, `Unknown`, an empty response, or an unavailable endpoint produces a `Failed` result, except that an empty service list is reported as a warning because the endpoint responded but has nothing registered. + +The `Subscription Service` check calls the configured subscription status endpoint. It fails when any subscription is not running, warns when parked messages exist, and passes when all subscriptions are running with no parked messages. The report includes subscription ID, tag, running state, health, parked-event count, operational reason, and runtime failure reason. + +The `KurrentDB Projections` check calls the configured `/projections/any` endpoint and checks only the names in `ProjectionNames`. It passes when every selected projection is `Running` and fails when a selected projection is missing or has another status. Set `Debug` to `true` to include the raw API payload in the report details; it is `false` by default. Authentication is optional for insecure development instances. For secured instances, add `Username` and `PasswordEnvironmentVariable`; the password is read from that environment variable and is not stored in JSON. The check is disabled by default until the projection names have been configured. + +The `Scheduled Tasks` check uses `Get-ScheduledTask` and `Get-ScheduledTaskInfo` to inspect only the configured tasks. It fails for missing, disabled, failed, overdue, or stale tasks, and reports the task state, last result, last run, next run, and failure reason. `ExpectedRunIntervalMinutes` determines how old the last run may be; the older `MaxLastRunAgeHours` property remains supported for compatibility. The current example checks `Daily Settlement` and `Replay Parked Queue` every 24 hours, and `Scavenge` every 7 days. + +The `KurrentDB Write Activity` check reads the latest events backwards from `$all` using a secondary-index prefix filter for each configured index. It reports the event IDs, event types, and timestamps for each index. Each index passes when recent events are found, warns when fewer than its `EventCount` are available, and fails when no events are returned, the query is unavailable, or its latest event is older than its `MaxLatestEventAgeMinutes`. The overall check fails if any index fails and warns if one or more indexes warn. `ClientAssemblyPath` points to `KurrentDB.Client.dll`; it is resolved relative to the configuration file when relative. The check is disabled by default until the correct secondary indexes have been configured. For compatibility, the older single-stream properties (`StreamName`, `EventCount`, and `MaxLatestEventAgeMinutes`) are also accepted. + +The default template checks are: + +1. `PowerShell Runtime` +2. `Report Output Directory` +3. `Disk Space` +4. `HealthMonitoring` +5. `Subscription Service` +6. `KurrentDB Projections` +7. `Scheduled Tasks` +8. `KurrentDB Write Activity` +9. `Template Configuration` + +Replace or extend these with checks that are meaningful for the target environment. Set `DiskSpace.Enabled` to `false` when disk-space monitoring is not required for a particular host. + +## Output + +Each run writes timestamped files to the selected output directory: + +- `.json`: structured run metadata, overall status, counts, and check results. +- `.html`: an operator-friendly report suitable for attaching to a message. + +Each check returns: + +```text +Name, Status, Summary, Details, StartedAt, CompletedAt, DurationMs, Error +``` + +`Status` is one of `Passed`, `Warning`, or `Failed`. Overall status uses the same precedence: `Failed`, then `Warning`, then `Passed`. + +## Add a real check + +Add a function that accepts the shared context and returns `New-CheckResult`: + +```powershell +function Test-QueueBacklog { + param([object] $Context) + + $backlog = Get-YourBacklogCount + if ($backlog -gt 100) { + return New-CheckResult -Name 'Queue Backlog' -Status Warning ` + -Summary "Backlog is $backlog" -Details @{ Backlog = $backlog } + } + + New-CheckResult -Name 'Queue Backlog' -Status Passed ` + -Summary "Backlog is $backlog" -Details @{ Backlog = $backlog } +} +``` + +Register it in `Get-SupportCheckDefinitions` in the desired execution order: + +```powershell +[pscustomobject]@{ Name = 'Queue Backlog'; Action = ${function:Test-QueueBacklog} } +``` + +Add a focused Pester test for the check, especially for its warning and failure thresholds. A check that throws is automatically converted to `Failed`, and later checks still run. + +## Schedule it + +For Windows Task Scheduler, use an action similar to: + +```text +Program: pwsh.exe +Arguments: -NoProfile -File "G:\Git\TransactionProcessing\SupportTools\ScheduledTasks\DailySupportCheck\Invoke-DailySupportCheck.ps1" -OutputPath "C:\SupportReports\Daily" -ConfigPath "C:\SupportConfig\daily-support-check.json" +``` + +Run under an account that can access the systems checked and write to the configured report directory. + +## Configure Brevo report delivery + +Reports are always written locally. Email delivery is optional and is enabled through the ignored local configuration file so the API key is not committed. + +Copy the example file and replace its placeholder values: + +```powershell +Copy-Item .\ScheduledTasks\DailySupportCheck\daily-support-check.local.example.json .\ScheduledTasks\DailySupportCheck\daily-support-check.local.json +``` + +Set `Email.Enabled` to `true`, provide the Brevo API key, a verified sender address, and one or more recipients. `AttachHtmlReport` and `AttachJsonReport` control whether the generated reports are attached. The script sends the HTML report as the email body and records a Brevo failure in the run result if delivery fails; the local files remain available. + +The Brevo API key is sent in the `api-key` header. Create and manage the key in Brevo, and do not add `daily-support-check.local.json` to source control. + +## Tests + +The tests do not require external services: + +```powershell +Import-Module Pester +$config = [PesterConfiguration]::Default +$config.TestRegistry.Enabled = $false +$config.TestDrive.Enabled = $false +$config.Run.Path = @('.\ScheduledTasks\DailySupportCheck\Tests\Invoke-DailySupportCheck.Tests.ps1') +Invoke-Pester -Configuration $config +``` diff --git a/ScheduledTasks/DailySupportCheck/Tests/Invoke-DailySupportCheck.Tests.ps1 b/ScheduledTasks/DailySupportCheck/Tests/Invoke-DailySupportCheck.Tests.ps1 new file mode 100644 index 0000000..62c7ecc --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/Tests/Invoke-DailySupportCheck.Tests.ps1 @@ -0,0 +1,920 @@ +Describe 'Daily support check' { + BeforeAll { + . (Join-Path $PSScriptRoot '..\Invoke-DailySupportCheck.ps1') + } + + It 'converts an environment password to a read-only SecureString' { + $securePassword = Get-EnvironmentPasswordSecureString -PasswordCharacters 'test-password'.ToCharArray() + + $securePassword.IsReadOnly() | Should -BeTrue + [System.Net.NetworkCredential]::new('', $securePassword).Password | Should -Be 'test-password' + } + + BeforeEach { + $testOutputRoot = Join-Path ([IO.Path]::GetTempPath()) (Join-Path 'DailySupportCheckTests' ([guid]::NewGuid().ToString('N'))) + New-Item -ItemType Directory -Path $testOutputRoot -Force | Out-Null + } + + AfterEach { + if (Test-Path $testOutputRoot) { + Remove-Item -LiteralPath $testOutputRoot -Recurse -Force + } + } + + It 'runs all registered checks in order and writes both report formats' { + $outputPath = Join-Path $testOutputRoot 'reports' + $configPath = Join-Path $testOutputRoot 'daily-support-check.json' + @' +{ + "DiskSpace": { + "Enabled": true, + "DefaultMinimumFreePercent": 0, + "DriveOverrides": {} + }, + "HealthMonitoring": { "Enabled": false, "BaseUrl": "http://localhost:9620", "ServicesPath": "/api/services", "TimeoutSeconds": 10 }, + "SubscriptionService": { "Enabled": false, "BaseUrl": "http://localhost:8080", "StatusPath": "/subscriptions/status", "TimeoutSeconds": 10 }, + "KurrentDbProjections": { "Enabled": false, "BaseUrl": "http://localhost:2113", "ProjectionsPath": "/projections/any", "ProjectionNames": [], "TimeoutSeconds": 10 }, + "ScheduledTasks": { "Enabled": false, "Tasks": [] }, + "KurrentDbWriteActivity": { "Enabled": false, "BaseUrl": "http://localhost:2113", "Streams": [] } +} +'@ | Set-Content -LiteralPath $configPath -Encoding UTF8 + + $report = Invoke-DailySupportCheck -OutputPath $outputPath -ConfigPath $configPath -PassThru + + $report.OverallStatus | Should -Be 'Passed' + $report.Checks.Name | Should -Be @( + 'PowerShell Runtime' + 'Report Output Directory' + 'Disk Space' + 'HealthMonitoring' + 'Subscription Service' + 'KurrentDB Projections' + 'Scheduled Tasks' + 'KurrentDB Write Activity' + 'Template Configuration' + ) + @((Get-ChildItem -Path $outputPath -Filter '*.json')).Count | Should -Be 1 + @((Get-ChildItem -Path $outputPath -Filter '*.html')).Count | Should -Be 1 + } + + It 'records a failed check and still runs the following check' { + $script:followingCheckRan = $false + + $results = @( + (Invoke-SupportCheck -Name 'Throwing Check' -Action { + throw 'expected failure' + }), + (Invoke-SupportCheck -Name 'Following Check' -Action { + $script:followingCheckRan = $true + Get-CheckResult -Name 'Following Check' -Status Passed -Summary 'ran' + }) + ) + + $results[0].Status | Should -Be 'Failed' + $results[0].Error | Should -Match 'expected failure' + $results[1].Status | Should -Be 'Passed' + $script:followingCheckRan | Should -BeTrue + } + + It 'calculates status precedence as Failed, then Warning, then Passed' { + Get-OverallStatus -Results @( + (Get-CheckResult -Name 'Pass' -Status Passed -Summary 'ok'), + (Get-CheckResult -Name 'Warning' -Status Warning -Summary 'attention') + ) | Should -Be 'Warning' + + Get-OverallStatus -Results @( + (Get-CheckResult -Name 'Warning' -Status Warning -Summary 'attention'), + (Get-CheckResult -Name 'Fail' -Status Failed -Summary 'broken') + ) | Should -Be 'Failed' + } + + It 'preserves details in JSON and HTML-encodes report content' { + $outputPath = Join-Path $testOutputRoot 'content' + $longSummary = ('failure detail ' * 20).Trim() + $check = Get-CheckResult -Name 'Content Check' -Status Warning -Summary $longSummary -Details 'detail & value' + $check.Error = 'error ' + $report = Get-SupportReport -StartedAt ([datetime]::UtcNow) -Results @($check) + + $paths = Write-SupportReports -Report $report -OutputPath $outputPath + $json = Get-Content -Raw -Path $paths.JsonPath | ConvertFrom-Json + $html = Get-Content -Raw -Path $paths.HtmlPath + + $json.Checks[0].Details | Should -Be 'detail & value' + $html | Should -Match 'CheckStatusSummary' + $html | Should -Match '🟠 Warning' + $html | Should -Match 'failure detail failure detail' + $html | Should -Match '…' + $html | Should -Not -Match 'Details' + $html | Should -Not -Match 'detail & value' + $html | Should -Not -Match 'error <text>' + } + + It 'removes matching reports older than the configured retention period' { + $outputPath = Join-Path $testOutputRoot 'retention' + New-Item -ItemType Directory -Path $outputPath -Force | Out-Null + $oldJsonPath = Join-Path $outputPath 'daily-support-check-old.json' + $oldHtmlPath = Join-Path $outputPath 'daily-support-check-old.html' + $recentJsonPath = Join-Path $outputPath 'daily-support-check-recent.json' + $unrelatedPath = Join-Path $outputPath 'unrelated.json' + 'old' | Set-Content -LiteralPath $oldJsonPath + 'old' | Set-Content -LiteralPath $oldHtmlPath + 'recent' | Set-Content -LiteralPath $recentJsonPath + 'unrelated' | Set-Content -LiteralPath $unrelatedPath + $oldTime = [DateTime]::UtcNow.AddDays(-8) + [IO.File]::SetLastWriteTimeUtc($oldJsonPath, $oldTime) + [IO.File]::SetLastWriteTimeUtc($oldHtmlPath, $oldTime) + [IO.File]::SetLastWriteTimeUtc($unrelatedPath, $oldTime) + + $report = Get-SupportReport -StartedAt ([datetime]::UtcNow) -Results @( + (Get-CheckResult -Name 'Retention Check' -Status Passed -Summary 'ok') + ) + Write-SupportReports -Report $report -OutputPath $outputPath -RetentionDays 7 | Out-Null + + Test-Path -LiteralPath $oldJsonPath | Should -BeFalse + Test-Path -LiteralPath $oldHtmlPath | Should -BeFalse + Test-Path -LiteralPath $recentJsonPath | Should -BeTrue + Test-Path -LiteralPath $unrelatedPath | Should -BeTrue + } + + It 'returns the generated files from the local transport seam' { + $paths = [pscustomobject]@{ JsonPath = 'report.json'; HtmlPath = 'report.html' } + $transport = Send-SupportReport -Report ([pscustomobject]@{ OverallStatus = 'Passed' }) -ReportPaths $paths + + $transport.Transport | Should -Be 'LocalFiles' + $transport.Status | Should -Be 'Ready' + $transport.Paths.JsonPath | Should -Be 'report.json' + $transport.Paths.HtmlPath | Should -Be 'report.html' + } + + It 'sends a Brevo email with the report summary and attachments' { + $jsonPath = Join-Path $testOutputRoot 'report.json' + $htmlPath = Join-Path $testOutputRoot 'report.html' + '{"OverallStatus":"Passed"}' | Set-Content -LiteralPath $jsonPath -Encoding UTF8 + 'Passed' | Set-Content -LiteralPath $htmlPath -Encoding UTF8 + $captured = $null + $configuration = [pscustomobject]@{ + Email = [pscustomobject]@{ + Enabled = $true + Provider = 'Brevo' + ApiUrl = 'https://brevo.test/v3/smtp/email' + ApiKey = 'api-test-key' + From = 'support@example.com' + To = @('recipient@example.com') + SubjectPrefix = 'Daily Support Check' + AttachHtmlReport = $true + AttachJsonReport = $true + } + } + $provider = { + param($requestUri, $headers, $payload) + $script:captured = [pscustomobject]@{ Uri = $requestUri; Headers = $headers; Payload = $payload } + [pscustomobject]@{ messageId = '' } + } + + $transport = Send-SupportReport ` + -Report ([pscustomobject]@{ OverallStatus = 'Passed'; StartedAt = '2026-09-29T15:30:00.0000000Z'; Counts = [pscustomobject]@{ Passed = 1; Warning = 0; Failed = 0 }; Checks = @() }) ` + -ReportPaths ([pscustomobject]@{ JsonPath = $jsonPath; HtmlPath = $htmlPath }) ` + -Configuration $configuration ` + -EmailProvider $provider + + $transport.Transport | Should -Be 'Brevo' + $transport.Status | Should -Be 'Sent' + $script:captured.Uri | Should -Be 'https://brevo.test/v3/smtp/email' + $script:captured.Headers['api-key'] | Should -Be 'api-test-key' + $expectedServerName = if ([string]::IsNullOrWhiteSpace($env:COMPUTERNAME)) { 'Unknown Server' } else { $env:COMPUTERNAME } + $script:captured.Payload.subject | Should -Be "Daily Support Check - Passed - $expectedServerName - 2026-09-29" + $script:captured.Payload.sender.email | Should -Be 'support@example.com' + $script:captured.Payload.to.email | Should -Contain 'recipient@example.com' + $script:captured.Payload.attachments.name | Should -Contain 'report.html' + $script:captured.Payload.attachments.name | Should -Contain 'report.json' + } + + It 'reports a Brevo API failure without exposing the API key' { + $configuration = [pscustomobject]@{ + Email = [pscustomobject]@{ + Enabled = $true + Provider = 'Brevo' + ApiUrl = 'https://brevo.test/v3/smtp/email' + ApiKey = 'api-secret-key' + From = 'support@example.com' + To = @('recipient@example.com') + } + } + $transport = Send-SupportReport ` + -Report ([pscustomobject]@{ OverallStatus = 'Failed'; Counts = [pscustomobject]@{ Passed = 0; Warning = 0; Failed = 1 }; Checks = @() }) ` + -ReportPaths ([pscustomobject]@{ JsonPath = 'report.json'; HtmlPath = 'report.html' }) ` + -Configuration $configuration ` + -EmailProvider { param($requestUri, $headers, $payload) throw 'Brevo rejected the request.' } + + $transport.Transport | Should -Be 'Brevo' + $transport.Status | Should -Be 'Failed' + $transport.Message | Should -Match 'rejected' + $transport.Message | Should -Not -Match 'api-secret-key' + } + + It 'loads disk-space settings from a JSON configuration file' { + $configPath = Join-Path $testOutputRoot 'daily-support-check.json' + @' +{ + "DiskSpace": { + "Enabled": true, + "DefaultMinimumFreePercent": 15, + "DriveOverrides": { + "C:": 10, + "D:": 20 + } + }, + "HealthMonitoring": { + "Enabled": true, + "BaseUrl": "http://localhost:9620", + "ServicesPath": "/api/services", + "TimeoutSeconds": 10 + }, + "SubscriptionService": { + "Enabled": true, + "BaseUrl": "http://localhost:8080", + "StatusPath": "/subscriptions/status", + "TimeoutSeconds": 10 + }, + "KurrentDbProjections": { + "Enabled": true, + "BaseUrl": "http://localhost:2113", + "ProjectionsPath": "/projections/any", + "ProjectionNames": [ + "MerchantBalanceProjection" + ], + "TimeoutSeconds": 10 + }, + "ScheduledTasks": { + "Enabled": true, + "Tasks": [ + { + "Name": "Daily Settlement", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Replay Parked Queue", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Scavenge", + "Path": "\\", + "ExpectedRunIntervalMinutes": 10080 + } + ] + }, + "KurrentDbWriteActivity": { + "Enabled": true, + "BaseUrl": "http://localhost:2113", + "Streams": [ + { + "Name": "$idx-ce-TransactionAggregate", + "EventCount": 10, + "MaxLatestEventAgeMinutes": 15 + } + ] + } +} +'@ | Set-Content -LiteralPath $configPath -Encoding UTF8 + + $configuration = Get-SupportConfiguration -Path $configPath + + $configuration.DiskSpace.Enabled | Should -BeTrue + $configuration.DiskSpace.DefaultMinimumFreePercent | Should -Be 15 + $configuration.DiskSpace.DriveOverrides.'C:' | Should -Be 10 + $configuration.HealthMonitoring.Enabled | Should -BeTrue + $configuration.HealthMonitoring.BaseUrl | Should -Be 'http://localhost:9620' + $configuration.SubscriptionService.Enabled | Should -BeTrue + $configuration.SubscriptionService.BaseUrl | Should -Be 'http://localhost:8080' + $configuration.ReportRetentionDays | Should -Be 7 + $configuration.KurrentDbProjections.Enabled | Should -BeTrue + $configuration.KurrentDbProjections.ProjectionNames | Should -Be @('MerchantBalanceProjection') + $configuration.ScheduledTasks.Enabled | Should -BeTrue + $configuration.ScheduledTasks.Tasks.Name | Should -Be @('Daily Settlement', 'Replay Parked Queue', 'Scavenge') + $configuration.ScheduledTasks.Tasks[0].ExpectedRunIntervalMinutes | Should -Be 1440 + $configuration.ScheduledTasks.Tasks[2].ExpectedRunIntervalMinutes | Should -Be 10080 + $configuration.KurrentDbWriteActivity.Enabled | Should -BeTrue + $configuration.KurrentDbWriteActivity.Streams.Count | Should -Be 1 + $configuration.KurrentDbWriteActivity.Streams[0].Name | Should -Be '$idx-ce-TransactionAggregate' + } + + It 'merges the optional local configuration override without changing the base file' { + $configPath = Join-Path $testOutputRoot 'daily-support-check.json' + $localConfigPath = Join-Path $testOutputRoot 'daily-support-check.local.json' + @' +{ + "DiskSpace": { + "Enabled": false, + "DefaultMinimumFreePercent": 15, + "DriveOverrides": {} + }, + "Email": { + "Enabled": false, + "From": "tracked@example.com" + } +} +'@ | Set-Content -LiteralPath $configPath -Encoding UTF8 + @' +{ + "Email": { + "Enabled": true, + "ApiKey": "local-secret" + } +} +'@ | Set-Content -LiteralPath $localConfigPath -Encoding UTF8 + + $configuration = Get-SupportConfiguration -Path $configPath + + $configuration.Email.Enabled | Should -BeTrue + $configuration.Email.From | Should -Be 'tracked@example.com' + $configuration.Email.ApiKey | Should -Be 'local-secret' + (Get-Content -Raw -LiteralPath $configPath) | Should -Match 'tracked@example.com' + } + + It 'uses a drive override instead of the global disk-space threshold' { + $configuration = [pscustomobject]@{ + DiskSpace = [pscustomobject]@{ + Enabled = $true + DefaultMinimumFreePercent = 15 + DriveOverrides = [pscustomobject]@{ 'C:' = 25 } + } + } + $context = [pscustomobject]@{ + Configuration = $configuration + DiskSpaceProvider = { + @( + [pscustomobject]@{ Drive = 'C:'; SizeBytes = 1000; FreeBytes = 200 } + [pscustomobject]@{ Drive = 'D:'; SizeBytes = 1000; FreeBytes = 200 } + ) + } + } + + $result = Test-DiskSpace -Context $context + + $result.Status | Should -Be 'Warning' + ($result.Details | Where-Object Drive -eq 'C:').ThresholdPercent | Should -Be 25 + ($result.Details | Where-Object Drive -eq 'D:').ThresholdPercent | Should -Be 15 + } + + It 'fails the disk-space check when drive inspection fails' { + $configuration = [pscustomobject]@{ + DiskSpace = [pscustomobject]@{ + Enabled = $true + DefaultMinimumFreePercent = 15 + DriveOverrides = [pscustomobject]@{} + } + } + $context = [pscustomobject]@{ + Configuration = $configuration + DiskSpaceProvider = { throw 'disk query failed' } + } + + $result = Test-DiskSpace -Context $context + + $result.Status | Should -Be 'Failed' + $result.Error | Should -Match 'disk query failed' + } + + It 'passes the HealthMonitoring check when all services are healthy' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + HealthMonitoring = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://health-monitoring' + ServicesPath = '/api/services' + TimeoutSeconds = 10 + } + } + HealthMonitoringProvider = { + @( + [pscustomobject]@{ ServiceId = 'orders'; Name = 'Orders'; Status = 'Healthy'; LastObservedAtUtc = '2026-09-29T12:00:00Z'; LastError = $null } + [pscustomobject]@{ ServiceId = 'payments'; Name = 'Payments'; Status = 'Healthy'; LastObservedAtUtc = '2026-09-29T12:00:00Z'; LastError = $null } + ) + } + } + + $result = Test-HealthMonitoring -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details.Count | Should -Be 2 + $result.Summary | Should -Match '2 monitored service' + } + + It 'returns Warning for degraded services and Failed for unhealthy services' { + $configuration = [pscustomobject]@{ + HealthMonitoring = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://health-monitoring' + ServicesPath = '/api/services' + TimeoutSeconds = 10 + } + } + + $degradedContext = [pscustomobject]@{ + Configuration = $configuration + HealthMonitoringProvider = { @([pscustomobject]@{ ServiceId = 'orders'; Name = 'Orders'; Status = 'Degraded' }) } + } + $unhealthyContext = [pscustomobject]@{ + Configuration = $configuration + HealthMonitoringProvider = { @([pscustomobject]@{ ServiceId = 'payments'; Name = 'Payments'; Status = 'Unhealthy' }) } + } + + (Test-HealthMonitoring -Context $degradedContext).Status | Should -Be 'Warning' + (Test-HealthMonitoring -Context $unhealthyContext).Status | Should -Be 'Failed' + } + + It 'fails the HealthMonitoring check when the endpoint cannot be queried' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + HealthMonitoring = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://health-monitoring' + ServicesPath = '/api/services' + TimeoutSeconds = 10 + } + } + HealthMonitoringProvider = { throw 'health endpoint unavailable' } + } + + $result = Test-HealthMonitoring -Context $context + + $result.Status | Should -Be 'Failed' + $result.Error | Should -Match 'health endpoint unavailable' + } + + It 'passes the subscription check when all subscriptions are running without parked messages' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + SubscriptionService = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://subscription-service' + StatusPath = '/subscriptions/status' + TimeoutSeconds = 10 + } + } + SubscriptionServiceProvider = { + @([pscustomobject]@{ + subscriptionId = 'subscription-1' + tag = 'Main' + isRunning = $true + health = 'Healthy' + parkedEventCount = 0 + operationalReason = $null + runtimeFailureReason = $null + }) + } + } + + $result = Test-SubscriptionService -Context $context + + $result.Status | Should -Be 'Passed' + $result.Summary | Should -Match '1 subscription' + } + + It 'expands a wrapped subscription response into individual subscriptions' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + SubscriptionService = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://subscription-service' + StatusPath = '/subscriptions/status' + TimeoutSeconds = 10 + } + } + SubscriptionServiceProvider = { + [pscustomobject]@{ + subscriptionId = @('subscription-1', 'subscription-2') + tag = @('Main', 'Ordered') + isRunning = @($true, $true) + health = @('Healthy', 'Healthy') + parkedEventCount = @(0, 2) + operationalReason = @($null, $null) + runtimeFailureReason = @($null, $null) + } + } + } + + $result = Test-SubscriptionService -Context $context + + $result.Status | Should -Be 'Warning' + $result.Summary | Should -Match '1 subscription\(s\) have parked messages' + $result.Details.Count | Should -Be 2 + } + + It 'fails when a subscription is stopped and warns when parked messages exist' { + $configuration = [pscustomobject]@{ + SubscriptionService = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://subscription-service' + StatusPath = '/subscriptions/status' + TimeoutSeconds = 10 + } + } + $stoppedContext = [pscustomobject]@{ + Configuration = $configuration + SubscriptionServiceProvider = { @([pscustomobject]@{ subscriptionId = 'stopped'; tag = 'Stopped'; isRunning = $false; health = 'Unhealthy'; parkedEventCount = 0 }) } + } + $parkedContext = [pscustomobject]@{ + Configuration = $configuration + SubscriptionServiceProvider = { @([pscustomobject]@{ subscriptionId = 'parked'; tag = 'Parked'; isRunning = $true; health = 'Healthy'; parkedEventCount = 3 }) } + } + + (Test-SubscriptionService -Context $stoppedContext).Status | Should -Be 'Failed' + (Test-SubscriptionService -Context $parkedContext).Status | Should -Be 'Warning' + } + + It 'fails the subscription check when the status endpoint cannot be queried' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + SubscriptionService = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://subscription-service' + StatusPath = '/subscriptions/status' + TimeoutSeconds = 10 + } + } + SubscriptionServiceProvider = { throw 'subscription endpoint unavailable' } + } + + $result = Test-SubscriptionService -Context $context + + $result.Status | Should -Be 'Failed' + $result.Error | Should -Match 'subscription endpoint unavailable' + } + + It 'passes only the configured KurrentDB projections when they are running' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @('selected') + TimeoutSeconds = 10 + } + } + KurrentDbProjectionProvider = { + @( + [pscustomobject]@{ effectiveName = 'selected'; status = 'Running'; progress = 100; stateReason = $null } + [pscustomobject]@{ effectiveName = 'not-selected'; status = 'Stopped'; progress = 0; stateReason = 'disabled' } + ) + } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details.Count | Should -Be 1 + $result.Details[0].Name | Should -Be 'selected' + } + + It 'includes the raw KurrentDB projection payload when debug is enabled' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + Debug = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @('selected') + TimeoutSeconds = 10 + } + } + KurrentDbProjectionProvider = { + @([pscustomobject]@{ effectiveName = 'selected'; status = 'Running'; progress = 100; stateReason = $null }) + } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details.RawResponse[0].effectiveName | Should -Be 'selected' + $result.Details.Projections[0].Name | Should -Be 'selected' + } + + It 'unwraps the projections property returned by KurrentDB' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @('selected') + TimeoutSeconds = 10 + } + } + KurrentDbProjectionProvider = { + [pscustomobject]@{ + projections = @([pscustomobject]@{ effectiveName = 'selected'; status = 'Running'; progress = 100; stateReason = $null }) + } + } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details[0].Name | Should -Be 'selected' + } + + It 'fails when a configured KurrentDB projection is stopped or missing' { + $configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @('stopped', 'missing') + TimeoutSeconds = 10 + } + } + $context = [pscustomobject]@{ + Configuration = $configuration + KurrentDbProjectionProvider = { + @([pscustomobject]@{ effectiveName = 'stopped'; status = 'Stopped'; progress = 0; stateReason = 'disabled' }) + } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Failed' + $result.Summary | Should -Match 'stopped' + $result.Summary | Should -Match 'missing' + } + + It 'warns when no KurrentDB projections are configured' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @() + TimeoutSeconds = 10 + } + } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Warning' + } + + It 'fails the KurrentDB projection check when the endpoint cannot be queried' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbProjections = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + ProjectionsPath = '/projections/any' + ProjectionNames = @('selected') + TimeoutSeconds = 10 + } + } + KurrentDbProjectionProvider = { throw 'projection endpoint unavailable' } + } + + $result = Test-KurrentDbProjections -Context $context + + $result.Status | Should -Be 'Failed' + $result.Error | Should -Match 'projection endpoint unavailable' + } + + It 'passes when configured scheduled tasks are enabled and have successful recent runs' { + $now = [datetime]::Now + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + ScheduledTasks = [pscustomobject]@{ + Enabled = $true + Tasks = @([pscustomobject]@{ Name = 'Daily Support Check'; Path = '\'; MaxLastRunAgeHours = 24 }) + } + } + ScheduledTaskProvider = { + param($taskConfiguration) + [pscustomobject]@{ + Name = $taskConfiguration.Name + Path = $taskConfiguration.Path + State = 'Ready' + LastTaskResult = 0 + LastRunTime = $now.AddHours(-1) + NextRunTime = $now.AddHours(23) + } + } + } + + $result = Test-ScheduledTasks -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details[0].LastTaskResult | Should -Be 0 + } + + It 'fails for missing, disabled, failed, or overdue scheduled tasks' { + $now = [datetime]::Now + $configuration = [pscustomobject]@{ + ScheduledTasks = [pscustomobject]@{ + Enabled = $true + Tasks = @( + [pscustomobject]@{ Name = 'Missing'; Path = '\'; MaxLastRunAgeHours = 24 } + [pscustomobject]@{ Name = 'Disabled'; Path = '\'; MaxLastRunAgeHours = 24 } + [pscustomobject]@{ Name = 'Failed'; Path = '\'; MaxLastRunAgeHours = 24 } + [pscustomobject]@{ Name = 'Overdue'; Path = '\'; MaxLastRunAgeHours = 24 } + ) + } + } + $context = [pscustomobject]@{ + Configuration = $configuration + ScheduledTaskProvider = { + param($taskConfiguration) + switch ($taskConfiguration.Name) { + 'Missing' { throw 'task not found' } + 'Disabled' { [pscustomobject]@{ Name = 'Disabled'; Path = '\'; State = 'Disabled'; LastTaskResult = 0; LastRunTime = $now.AddHours(-1); NextRunTime = $now.AddHours(23) } } + 'Failed' { [pscustomobject]@{ Name = 'Failed'; Path = '\'; State = 'Ready'; LastTaskResult = 1; LastRunTime = $now.AddHours(-1); NextRunTime = $now.AddHours(23) } } + 'Overdue' { [pscustomobject]@{ Name = 'Overdue'; Path = '\'; State = 'Ready'; LastTaskResult = 0; LastRunTime = $now.AddHours(-25); NextRunTime = $now.AddHours(-1) } } + } + } + } + + $result = Test-ScheduledTasks -Context $context + + $result.Status | Should -Be 'Failed' + $result.Summary | Should -Match 'Missing' + $result.Summary | Should -Match 'Disabled' + $result.Summary | Should -Match 'Failed' + $result.Summary | Should -Match 'Overdue' + } + + It 'uses the expected run interval when checking scheduled-task freshness' { + $now = [datetime]::Now + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + ScheduledTasks = [pscustomobject]@{ + Enabled = $true + Tasks = @([pscustomobject]@{ + Name = 'Daily Settlement' + Path = '\' + ExpectedRunIntervalMinutes = 1440 + }) + } + } + ScheduledTaskProvider = { + [pscustomobject]@{ + Name = 'Daily Settlement' + Path = '\' + State = 'Ready' + LastTaskResult = 0 + LastRunTime = $now.AddMinutes(-1441) + NextRunTime = $now.AddHours(23) + } + } + } + + $result = Test-ScheduledTasks -Context $context + + $result.Status | Should -Be 'Failed' + $result.Summary | Should -Match 'Last run is too old' + } + + It 'warns when no scheduled tasks are configured' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + ScheduledTasks = [pscustomobject]@{ Enabled = $true; Tasks = @() } + } + } + + (Test-ScheduledTasks -Context $context).Status | Should -Be 'Warning' + } + + It 'passes when recent events are found on the configured KurrentDB stream' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbWriteActivity = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + StreamName = '$idx-ce-CallbackMessageAggregate' + EventCount = 2 + MaxLatestEventAgeMinutes = 15 + } + } + KurrentDbWriteActivityProvider = { + param($requestUri, $requestCount) + [pscustomobject]@{ + entries = @( + [pscustomobject]@{ title = '9@stream'; updated = ([datetime]::UtcNow.AddMinutes(-1)).ToString('o'); summary = 'EventType'; id = 'event-9' } + [pscustomobject]@{ title = '8@stream'; updated = ([datetime]::UtcNow.AddMinutes(-2)).ToString('o'); summary = 'EventType'; id = 'event-8' } + ) + } + } + } + + $result = Test-KurrentDbWriteActivity -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details.Count | Should -Be 1 + $result.Details[0].Events.Count | Should -Be 2 + $result.Summary | Should -Match '1 configured KurrentDB stream' + } + + It 'checks multiple configured KurrentDB streams independently' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbWriteActivity = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + Streams = @( + [pscustomobject]@{ Name = 'stream-one'; EventCount = 1; MaxLatestEventAgeMinutes = 15 } + [pscustomobject]@{ Name = 'stream-two'; EventCount = 1; MaxLatestEventAgeMinutes = 15 } + ) + } + } + KurrentDbWriteActivityProvider = { + param($requestUri, $requestCount) + [pscustomobject]@{ + entries = @([pscustomobject]@{ title = "0@$requestUri"; updated = ([datetime]::UtcNow.AddMinutes(-1)).ToString('o'); summary = 'EventType'; id = $requestUri }) + } + } + } + + $result = Test-KurrentDbWriteActivity -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details.Count | Should -Be 2 + $result.Details.StreamName | Should -Contain 'stream-one' + $result.Details.StreamName | Should -Contain 'stream-two' + } + + It 'accepts events returned by the KurrentDB filtered all-stream client' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbWriteActivity = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + Streams = @([pscustomobject]@{ Name = '$idx-ce-TransactionAggregate'; EventCount = 1; MaxLatestEventAgeMinutes = 15 }) + } + } + KurrentDbWriteActivityProvider = { + param($indexName, $requestCount) + [pscustomobject]@{ + events = @([pscustomobject]@{ + EventId = 'event-1' + EventType = 'TransactionHasStartedEvent' + Timestamp = ([datetime]::UtcNow.AddMinutes(-1)).ToString('o') + Title = 'TransactionAggregate-1' + }) + } + } + } + + $result = Test-KurrentDbWriteActivity -Context $context + + $result.Status | Should -Be 'Passed' + $result.Details[0].Events[0].EventType | Should -Be 'TransactionHasStartedEvent' + } + + It 'warns when fewer events are returned and fails when the latest event is stale or absent' { + $configuration = [pscustomobject]@{ + KurrentDbWriteActivity = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + StreamName = '$idx-ce-CallbackMessageAggregate' + EventCount = 10 + MaxLatestEventAgeMinutes = 15 + } + } + $fewContext = [pscustomobject]@{ + Configuration = $configuration + KurrentDbWriteActivityProvider = { [pscustomobject]@{ entries = @([pscustomobject]@{ title = '0@stream'; updated = ([datetime]::UtcNow.AddMinutes(-1)).ToString('o'); summary = 'EventType'; id = 'event-0' }) } } + } + $staleContext = [pscustomobject]@{ + Configuration = $configuration + KurrentDbWriteActivityProvider = { [pscustomobject]@{ entries = @([pscustomobject]@{ title = '0@stream'; updated = ([datetime]::UtcNow.AddMinutes(-30)).ToString('o'); summary = 'EventType'; id = 'event-0' }) } } + } + $emptyContext = [pscustomobject]@{ + Configuration = $configuration + KurrentDbWriteActivityProvider = { [pscustomobject]@{ entries = @() } } + } + + (Test-KurrentDbWriteActivity -Context $fewContext).Status | Should -Be 'Warning' + (Test-KurrentDbWriteActivity -Context $staleContext).Status | Should -Be 'Failed' + (Test-KurrentDbWriteActivity -Context $emptyContext).Status | Should -Be 'Failed' + } + + It 'fails the KurrentDB write-activity check when the stream cannot be queried' { + $context = [pscustomobject]@{ + Configuration = [pscustomobject]@{ + KurrentDbWriteActivity = [pscustomobject]@{ + Enabled = $true + BaseUrl = 'http://kurrentdb' + StreamName = '$idx-ce-CallbackMessageAggregate' + EventCount = 10 + MaxLatestEventAgeMinutes = 15 + } + } + KurrentDbWriteActivityProvider = { throw 'stream endpoint unavailable' } + } + + $result = Test-KurrentDbWriteActivity -Context $context + + $result.Status | Should -Be 'Failed' + $result.Error | Should -Match 'stream endpoint unavailable' + } +} diff --git a/ScheduledTasks/DailySupportCheck/daily-support-check.json b/ScheduledTasks/DailySupportCheck/daily-support-check.json new file mode 100644 index 0000000..95b597a --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/daily-support-check.json @@ -0,0 +1,62 @@ +{ + "ReportRetentionDays": 7, + "DiskSpace": { + "Enabled": true, + "DefaultMinimumFreePercent": 15, + "DriveOverrides": {} + }, + "HealthMonitoring": { + "Enabled": true, + "BaseUrl": "http://localhost:9620", + "ServicesPath": "/api/services", + "TimeoutSeconds": 10 + }, + "SubscriptionService": { + "Enabled": true, + "BaseUrl": "http://localhost:8080", + "StatusPath": "/subscriptions/status", + "TimeoutSeconds": 10 + }, + "KurrentDbProjections": { + "Enabled": true, + "Debug": false, + "BaseUrl": "http://localhost:2113", + "ProjectionsPath": "/projections/any", + "ProjectionNames": [ + "MerchantBalanceProjection" + ], + "TimeoutSeconds": 10 + }, + "ScheduledTasks": { + "Enabled": true, + "Tasks": [ + { + "Name": "Daily Settlement", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Replay Parked Queue", + "Path": "\\", + "ExpectedRunIntervalMinutes": 1440 + }, + { + "Name": "Scavenge", + "Path": "\\", + "ExpectedRunIntervalMinutes": 10080 + } + ] + }, + "KurrentDbWriteActivity": { + "Enabled": true, + "BaseUrl": "http://localhost:2113", + "ClientAssemblyPath": "..\\..\\StreamManagementTool\\bin\\Debug\\net10.0\\KurrentDB.Client.dll", + "Streams": [ + { + "Name": "$idx-ce-TransactionAggregate", + "EventCount": 10, + "MaxLatestEventAgeMinutes": 15 + } + ] + } +} diff --git a/ScheduledTasks/DailySupportCheck/daily-support-check.local.example.json b/ScheduledTasks/DailySupportCheck/daily-support-check.local.example.json new file mode 100644 index 0000000..1119042 --- /dev/null +++ b/ScheduledTasks/DailySupportCheck/daily-support-check.local.example.json @@ -0,0 +1,15 @@ +{ + "Email": { + "Enabled": true, + "Provider": "Brevo", + "ApiUrl": "https://api.brevo.com/v3/smtp/email", + "ApiKey": "api-replace-locally", + "From": "support-check@example.com", + "To": [ + "you@example.com" + ], + "SubjectPrefix": "Daily Support Check", + "AttachHtmlReport": true, + "AttachJsonReport": true + } +} diff --git a/docs/superpowers/plans/2026-09-29-daily-support-check.md b/docs/superpowers/plans/2026-09-29-daily-support-check.md new file mode 100644 index 0000000..4b9be1e --- /dev/null +++ b/docs/superpowers/plans/2026-09-29-daily-support-check.md @@ -0,0 +1,113 @@ +# Daily Support Check Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a reusable PowerShell daily support-check template that runs independent checks, creates JSON/HTML reports, and leaves report delivery configurable. + +**Architecture:** A single PowerShell entry point will contain the orchestration and template checks, with a common result contract and isolated report/transport functions. Pester tests will invoke the script in a temporary output directory and validate aggregation, failure isolation, status precedence, and generated artifacts. + +**Tech Stack:** PowerShell 7-compatible scripting, Pester, JSON serialization, HTML generation, Windows Task Scheduler-compatible parameters. + +**Spec:** `docs/superpowers/specs/2026-09-29-daily-support-check-design.md` + +## Global Constraints + +- The first version is a template, not a production-specific monitoring implementation. +- Do not add external module or package dependencies. +- Checks must return `Passed`, `Warning`, or `Failed` results with the shared fields defined in the spec. +- A failed check must not prevent later checks or report generation. +- The initial transport must only confirm local report files; final transport remains out of scope. + +## Review Focus + +- A check throws unexpectedly: the runner must record `Failed` and continue; covered by `Invoke-DailySupportCheck.Tests.ps1` failure-isolation test. +- Output directory does not exist: the script must create it; covered by report-generation test. +- A check returns a warning alongside a passing check: overall status must be `Warning`; covered by status-precedence test. +- Report content contains structured details or error text: JSON must preserve it and HTML must render it safely; covered by report-content test. +- No explicit check list is supplied: all registered checks must run in stable registry order; covered by default orchestration test. + +--- + +### Task 1: Add the daily support-check script + +**Files:** +- Create: `ScheduledTasks/DailySupportCheck/Invoke-DailySupportCheck.ps1` + +**Interfaces:** +- Produces script parameters `OutputPath`, `CheckName`, and `PassThru`. +- Produces functions `New-CheckResult`, `Invoke-SupportCheck`, `Get-OverallStatus`, `New-SupportReport`, `Write-SupportReports`, `Send-SupportReport`, and `Invoke-DailySupportCheck`. +- The script's final output is a report object when `-PassThru` is supplied; otherwise it writes a concise console summary and returns an exit status through the script flow. + +- [ ] **Step 1: Implement result construction and the three safe template checks** + +Create `New-CheckResult` with the fields `Name`, `Status`, `Summary`, `Details`, `StartedAt`, `CompletedAt`, `DurationMs`, and `Error`. Add placeholder checks for PowerShell/runtime availability, output-directory writability, and template configuration readiness. Each check must return the shared result contract and be easy to replace with a real operational check. + +- [ ] **Step 2: Implement isolated check execution and overall status calculation** + +Implement `Invoke-SupportCheck` so exceptions become `Failed` results and do not escape into the registry loop. Implement `Get-OverallStatus` with precedence `Failed` > `Warning` > `Passed`. Register checks in a stable ordered array and filter it when `CheckName` is supplied. + +- [ ] **Step 3: Implement report model and JSON/HTML writers** + +Implement `New-SupportReport` with run metadata, overall status, status counts, and check results. Implement `Write-SupportReports` to create the output directory and write timestamped `.json` and `.html` files. HTML must HTML-encode check summaries, details, and error text. + +- [ ] **Step 4: Implement local transport and script orchestration** + +Implement `Send-SupportReport` to return a local transport result containing the generated paths. Implement `Invoke-DailySupportCheck` to run selected checks, always write reports, print a summary, and return the report object for `-PassThru`. Preserve report generation when checks fail. Keep warnings non-fatal by default and expose a clear place for future strict behavior. + +- [ ] **Step 5: Run a syntax/import smoke check** + +Run: `pwsh -NoProfile -Command ". ./ScheduledTasks/DailySupportCheck/Invoke-DailySupportCheck.ps1; Get-Command Invoke-DailySupportCheck"` + +Expected: the command resolves successfully and no parser errors are emitted. + +### Task 2: Add Pester coverage + +**Files:** +- Create: `ScheduledTasks/DailySupportCheck/Tests/Invoke-DailySupportCheck.Tests.ps1` + +**Interfaces:** +- Consumes the functions and script parameters from Task 1. +- Produces deterministic tests using temporary directories and no external services. + +- [ ] **Step 1: Add the default orchestration test** + +Invoke the function with a temporary output path and assert that all registered template checks run in registry order, the report is returned, and both JSON and HTML files exist. + +- [ ] **Step 2: Add failure-isolation coverage** + +Invoke `Invoke-SupportCheck` with a deliberately throwing test scriptblock followed by a passing scriptblock. Assert that the first result is `Failed` and the second still runs and is `Passed`. + +- [ ] **Step 3: Add status precedence and content coverage** + +Assert `Failed` beats `Warning` and `Passed`, and `Warning` beats `Passed`. Generate a report containing warning details and an error string, then assert the JSON preserves them and the HTML contains encoded content. + +- [ ] **Step 4: Run the Pester test file** + +Run: `pwsh -NoProfile -Command "Invoke-Pester -Path './ScheduledTasks/DailySupportCheck/Tests/Invoke-DailySupportCheck.Tests.ps1' -Output Detailed"` + +Expected: all tests pass with no external-service requirements. + +### Task 3: Document extension, scheduling, and transport + +**Files:** +- Create: `ScheduledTasks/DailySupportCheck/README.md` + +**Interfaces:** +- Documents the script's parameters, result contract, report locations, exit behavior, and the exact location of the transport seam. + +- [ ] **Step 1: Document normal invocation and scheduling** + +Include examples for an interactive run, selecting checks, and a Task Scheduler action using `pwsh -NoProfile -File`. + +- [ ] **Step 2: Document adding a real check** + +Show the required function shape, result fields, registry entry, and focused Pester test expectation. + +- [ ] **Step 3: Document future transport integration** + +Explain that `Send-SupportReport` currently confirms local files and can later be replaced with email, Teams, webhook, or another approved transport without changing check code. + +- [ ] **Step 4: Run final validation** + +Run the Pester test file and the PowerShell smoke check from Tasks 1 and 2. Expected: all tests pass, the script imports cleanly, and the working tree contains only the intended spec, plan, script, tests, and README changes. + diff --git a/docs/superpowers/specs/2026-09-29-daily-support-check-design.md b/docs/superpowers/specs/2026-09-29-daily-support-check-design.md new file mode 100644 index 0000000..363a6ef --- /dev/null +++ b/docs/superpowers/specs/2026-09-29-daily-support-check-design.md @@ -0,0 +1,85 @@ +# Daily Support Check Script Design + +## Purpose + +Provide a reusable PowerShell template for a daily operational support check. The script will run a configurable set of independent checks, collect consistent results, build a human-readable report, and expose a transport seam for sending the report once the delivery mechanism is confirmed. + +## Scope + +The first version will be a template rather than a production-specific monitoring implementation. It will include safe example checks and clear extension points for adding real checks. + +In scope: + +- A single orchestration script under `ScheduledTasks/DailySupportCheck`. +- Independent check functions with a shared result contract. +- Continued execution when an individual check fails. +- Console summary plus JSON and HTML report files. +- Configurable output directory and check selection. +- A transport function that currently supports local output and can later be adapted for email, Teams, webhook, or another destination. +- Pester tests for orchestration, failure isolation, and report generation. +- README documentation covering extension and scheduling. + +Out of scope: + +- Choosing or configuring the final report transport. +- Implementing environment-specific health checks against production systems. +- Adding external module or package dependencies. + +## Design + +### Entry point and configuration + +`Invoke-DailySupportCheck.ps1` will expose parameters for output path, optional check selection, and optional strict behavior. The default run will execute all registered checks. A check registry will make the execution order and available checks visible in one place. + +### Check contract + +Each check will be a PowerShell function accepting a context object and returning a result object with: + +- `Name`: stable check name. +- `Status`: `Passed`, `Warning`, or `Failed`. +- `Summary`: concise operator-facing result. +- `Details`: optional structured or textual diagnostic information. +- `StartedAt`, `CompletedAt`, and `DurationMs`. +- `Error`: optional error text. + +The runner will catch terminating and non-terminating check failures, convert them to `Failed` results, and continue with subsequent checks. + +### Reporting + +The report model will contain run metadata, overall status, counts by status, and the individual check results. JSON will preserve the structured model. HTML will provide a readable table and detail sections suitable for attaching to an email. Report generation will not change the check results. + +The overall status will be: + +- `Failed` if any check failed. +- `Warning` if no check failed but at least one check warned. +- `Passed` otherwise. + +### Transport seam + +`Send-SupportReport` will receive the completed report and generated file paths. For the initial template it will confirm local files are available and return a transport result. A future transport implementation can replace or extend this function without changing check or report code. + +### Failure handling and exit behavior + +The script will always attempt to write the report, including when checks fail. It will return a non-zero exit code when the overall status is `Failed`; warnings will remain successful by default so scheduled-task operators can choose their own alerting policy. Unexpected orchestration errors will be written to the error stream and return a non-zero exit code. + +## Testing + +Pester tests will verify: + +1. A normal run aggregates registered checks and creates JSON/HTML output. +2. A failing check does not prevent later checks from running. +3. Overall status precedence is correct. +4. The generated report contains check names and statuses. +5. The local transport reports the generated files. + +Tests will use temporary output directories and avoid external services. + +## Extension path + +To add a real support action, an operator will: + +1. Add a function implementing the common result contract. +2. Add the function to the check registry. +3. Add a focused Pester test for its behavior. +4. Update the README with required configuration and operational assumptions. +