diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index ec6dc5588..6e0d266a3 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -175,6 +175,11 @@ function GetDefaultSettings "doNotRunTests" = $false "doNotRunBcptTests" = $false "doNotRunPageScriptingTests" = $false + "testIsolation" = [ordered]@{ + "enabled" = $false + "defaultRunnerCodeunitId" = 0 + "partitions" = @() + } "doNotPublishApps" = $false "doNotSignApps" = $false "configPackages" = @() diff --git a/Actions/.Modules/TestIsolation.psm1 b/Actions/.Modules/TestIsolation.psm1 new file mode 100644 index 000000000..202f3d618 --- /dev/null +++ b/Actions/.Modules/TestIsolation.psm1 @@ -0,0 +1,238 @@ +<# +.SYNOPSIS +Test Isolation module for AL-Go for GitHub. + +.DESCRIPTION +Builds a scriptblock compatible with Run-AlPipeline's -RunTestsInBcContainer +override. The scriptblock runs the tests once per declared partition (each with +its own test runner and codeunit-range filter), plus one trailing call under +the default runner whose filter is the complement of every explicit partition's +filter. +#> + +Import-Module (Join-Path -Path $PSScriptRoot "DebugLogHelper.psm1") + +# Codeunit IDs are positive 32-bit integers +$script:maxCodeunitId = [long] 2147483647 + +function ConvertTo-CodeunitIntervals { + <# + .SYNOPSIS + Parse a partition 'codeunits' filter into a list of [lo, hi] integer intervals. + .DESCRIPTION + Only single IDs and closed ranges joined by '|' are supported + (e.g. '60100|60200..60299'). This is deliberately a subset of the BC + filter grammar: the complement AL-Go computes for the trailing + default-runner call is only well-defined over unions of closed + intervals. The settings schema enforces the same subset; this + function is the defense for settings that bypassed schema validation. + #> + Param( + [Parameter(Mandatory = $true)] + [string] $Filter + ) + + $intervals = @() + foreach ($piece in $Filter.Split('|')) { + $trimmed = $piece.Trim() + if (-not $trimmed) { continue } + if ($trimmed -match '^(\d+)$') { + $lo = [long] $Matches[1] + $hi = $lo + } + elseif ($trimmed -match '^(\d+)\s*\.\.\s*(\d+)$') { + $lo = [long] $Matches[1] + $hi = [long] $Matches[2] + } + else { + throw "Unsupported piece '$trimmed' in testIsolation codeunits filter '$Filter'. Supported syntax is single codeunit IDs and closed ranges joined by '|' (e.g. '60100|60200..60299')." + } + if ($lo -lt 1 -or $hi -gt $script:maxCodeunitId) { + throw "Codeunit IDs in testIsolation codeunits filter '$Filter' must be between 1 and $script:maxCodeunitId." + } + if ($lo -gt $hi) { + throw "Invalid range '$trimmed' in testIsolation codeunits filter '$Filter': lower bound is greater than upper bound." + } + $intervals += , @($lo, $hi) + } + if ($intervals.Count -eq 0) { + throw "testIsolation codeunits filter '$Filter' does not contain any codeunit IDs." + } + return , $intervals +} + +function Get-MergedIntervals { + Param( + [Parameter(Mandatory = $true)] + $Intervals + ) + + $sorted = @($Intervals | Sort-Object -Property @{ Expression = { $_[0] } }, @{ Expression = { $_[1] } }) + $merged = @() + foreach ($interval in $sorted) { + # Also merge adjacent intervals (lo = previous hi + 1) so the complement never contains empty gaps + if ($merged.Count -gt 0 -and $interval[0] -le ($merged[-1][1] + 1)) { + if ($interval[1] -gt $merged[-1][1]) { $merged[-1][1] = $interval[1] } + } + else { + $merged += , @($interval[0], $interval[1]) + } + } + return , $merged +} + +function Get-ComplementFilter { + <# + .SYNOPSIS + Build a BC filter expression matching every codeunit ID NOT covered by the given merged intervals. + .DESCRIPTION + The BC filter grammar has no negation over ranges ('<>' only applies + to single values), so the exclusion filter is expressed as the + complement: a '|'-joined union of the gaps, e.g. excluding 60100 and + 60200..60299 yields '..60099|60101..60199|60300..'. Returns an empty + string when the intervals cover the entire codeunit ID space. + #> + Param( + [Parameter(Mandatory = $true)] + $MergedIntervals + ) + + $pieces = @() + $next = [long] 1 + foreach ($interval in $MergedIntervals) { + if ($interval[0] -gt $next) { + $gapHi = $interval[0] - 1 + if ($next -eq 1) { $pieces += "..$gapHi" } + elseif ($next -eq $gapHi) { $pieces += "$next" } + else { $pieces += "$next..$gapHi" } + } + if (($interval[1] + 1) -gt $next) { $next = $interval[1] + 1 } + } + if ($next -le $script:maxCodeunitId) { + $pieces += "$next.." + } + return ($pieces -join '|') +} + +function New-PartitionedTestRunnerScriptBlock { + <# + .SYNOPSIS + Build a scriptblock for Run-AlPipeline's -RunTestsInBcContainer hook. + .DESCRIPTION + Run-AlPipeline invokes the override once per test app with a hashtable + of parameters (extensionId, containerName, disabledTests, JUnit/XUnit + file, AppendTo*ResultFile, auth context, etc.). The returned + scriptblock loops over the configured partitions and, for each one, + runs the tests with the partition's runner id and codeunit-range + filter. After all partitions, it issues one trailing call under + defaultRunnerCodeunitId whose -testCodeunitRange is the complement of + the union of every explicit partition's filter - so every test + codeunit not matched by an explicit partition runs under the default + runner exactly once. If the partitions cover the entire codeunit ID + space, the trailing call is skipped. + + Result-file appending is preserved because we forward the JUnit/XUnit + file params Run-AlPipeline already configured (AppendTo*ResultFile = $true). + .PARAMETER Settings + The testIsolation settings object with `defaultRunnerCodeunitId` and + `partitions` (array of @{ runnerCodeunitId; codeunits }). Closed over + by the returned scriptblock. + .PARAMETER InnerScriptBlock + The scriptblock each partitioned call is routed through, with the + same contract as Run-AlPipeline's -RunTestsInBcContainer override + (receives a parameter hashtable, returns $true if all tests passed). + Pass a project's existing RunTestsInBcContainer override here so + partitioning wraps it instead of replacing it. Defaults to calling + Run-TestsInBcContainer directly. The scriptblock must splat the + hashtable on to Run-TestsInBcContainer for the partition-specific + testCodeunitRange/testRunnerCodeunitId entries to take effect. + .OUTPUTS + [scriptblock] returning $true if every invocation reported success. + #> + Param( + [Parameter(Mandatory = $true)] + $Settings, + [scriptblock] $InnerScriptBlock + ) + + $capturedPartitions = @($Settings.partitions) + $capturedDefaultRunner = [int] $Settings.defaultRunnerCodeunitId + $capturedInner = $InnerScriptBlock + if (-not $capturedInner) { + $capturedInner = { Param([Hashtable] $parameters) Run-TestsInBcContainer @parameters } + } + + $partitionIntervalSets = @() + foreach ($p in $capturedPartitions) { + $partitionIntervalSets += , (ConvertTo-CodeunitIntervals -Filter ([string] $p.codeunits)) + } + + # Overlapping partitions run the shared codeunits once per matching partition, + # duplicating them in the test results - warn, but leave the config decision to the user + for ($i = 0; $i -lt $partitionIntervalSets.Count; $i++) { + for ($j = $i + 1; $j -lt $partitionIntervalSets.Count; $j++) { + $overlaps = $false + foreach ($a in $partitionIntervalSets[$i]) { + foreach ($b in $partitionIntervalSets[$j]) { + if (($a[0] -le $b[1]) -and ($b[0] -le $a[1])) { $overlaps = $true; break } + } + if ($overlaps) { break } + } + if ($overlaps) { + OutputWarning -message "testIsolation partitions overlap: '$($capturedPartitions[$i].codeunits)' (runner $($capturedPartitions[$i].runnerCodeunitId)) and '$($capturedPartitions[$j].codeunits)' (runner $($capturedPartitions[$j].runnerCodeunitId)). Overlapping codeunits run once per matching partition and appear multiple times in the test results." + } + } + } + + $defaultRangeFilter = '' + $skipDefaultCall = $false + if ($capturedPartitions.Count -gt 0) { + $allIntervals = @() + foreach ($set in $partitionIntervalSets) { $allIntervals += $set } + $defaultRangeFilter = Get-ComplementFilter -MergedIntervals (Get-MergedIntervals -Intervals $allIntervals) + $skipDefaultCall = (-not $defaultRangeFilter) + } + + return { + Param([Hashtable] $parameters) + + $appId = "$($parameters.extensionId)" + $allPassed = $true + $invocations = 0 + + foreach ($p in $capturedPartitions) { + $call = @{} + foreach ($k in $parameters.Keys) { $call[$k] = $parameters[$k] } + $call['testCodeunitRange'] = "$($p.codeunits)" + $call['testRunnerCodeunitId'] = "$([int] $p.runnerCodeunitId)" + + Write-Host "Running partition runner=$($p.runnerCodeunitId) range='$($p.codeunits)' app=$appId" + $invocations++ + + $passed = & $capturedInner $call + if (-not $passed) { $allPassed = $false } + } + + if ($skipDefaultCall) { + Write-Host "Partitions cover the entire codeunit ID space - skipping the default-runner call for app $appId" + } + else { + $defaultCall = @{} + foreach ($k in $parameters.Keys) { $defaultCall[$k] = $parameters[$k] } + if ($defaultRangeFilter) { $defaultCall['testCodeunitRange'] = $defaultRangeFilter } + if ($capturedDefaultRunner -gt 0) { $defaultCall['testRunnerCodeunitId'] = "$capturedDefaultRunner" } + $defaultRunnerDisplay = if ($capturedDefaultRunner -gt 0) { "$capturedDefaultRunner" } else { "BC default" } + + Write-Host "Running default partition runner=$defaultRunnerDisplay range='$defaultRangeFilter' app=$appId" + $invocations++ + + $passed = & $capturedInner $defaultCall + if (-not $passed) { $allPassed = $false } + } + + Write-Host "Partitioned test run for app $appId complete. Invocations: $invocations. All passed: $allPassed" + return $allPassed + }.GetNewClosure() +} + +Export-ModuleMember -Function New-PartitionedTestRunnerScriptBlock diff --git a/Actions/.Modules/settings.schema.json b/Actions/.Modules/settings.schema.json index 8cf29ab0d..7c6e7c19b 100644 --- a/Actions/.Modules/settings.schema.json +++ b/Actions/.Modules/settings.schema.json @@ -323,6 +323,45 @@ "treatTestFailuresAsWarnings": { "type": "boolean" }, + "testIsolation": { + "type": "object", + "additionalProperties": false, + "required": [ "enabled", "defaultRunnerCodeunitId", "partitions" ], + "properties": { + "enabled": { + "type": "boolean", + "description": "Enable partitioned test runs. When true, AL-Go invokes Run-TestsInBcContainer once per entry in 'partitions' (each with its own test runner) plus one fallback call covering everything else under 'defaultRunnerCodeunitId'." + }, + "defaultRunnerCodeunitId": { + "type": "integer", + "minimum": 0, + "description": "Test runner codeunit ID used for codeunits not matched by any 'partitions' entry. 0 = the BcContainerHelper default runner." + }, + "partitions": { + "type": "array", + "description": "Each entry runs the codeunits matching its 'codeunits' BC filter under the configured 'runnerCodeunitId'. Codeunits not matched by any entry fall through to 'defaultRunnerCodeunitId'.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ "runnerCodeunitId", "codeunits" ], + "properties": { + "runnerCodeunitId": { + "type": "integer", + "minimum": 1, + "description": "ID of a Subtype = TestRunner codeunit whose TestIsolation property satisfies the codeunits matched below." + }, + "codeunits": { + "type": "string", + "minLength": 1, + "pattern": "^\\s*\\d+(\\.\\.\\d+)?(\\s*\\|\\s*\\d+(\\.\\.\\d+)?)*\\s*$", + "description": "Codeunit IDs to run under 'runnerCodeunitId': single IDs and closed ranges joined by '|' (e.g. '60100|60200..60299'). Only this subset of the BC filter syntax is supported, because AL-Go derives the complement of all partitions for the default-runner call." + } + } + } + } + }, + "description": "See https://aka.ms/ALGoSettings#testIsolation" + }, "rulesetFile": { "type": "string" }, diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index c51848fdf..9f3444f09 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -467,6 +467,25 @@ try { $runAlPipelineParams["preprocessorsymbols"] = $settings.preprocessorSymbols $runAlPipelineParams["features"] = $settings.features + if ($settings.testIsolation.enabled -and -not $settings.doNotRunTests) { + Import-Module (Join-Path $PSScriptRoot '../.Modules/TestIsolation.psm1' -Resolve) + Write-Host "Test isolation enabled - $($settings.testIsolation.partitions.Count) explicit partition(s) + default runner ($($settings.testIsolation.defaultRunnerCodeunitId))" + + $testIsolationParams = @{ Settings = $settings.testIsolation } + if ($runAlPipelineParams.Keys -contains 'RunTestsInBcContainer') { + Write-Host "Existing RunTestsInBcContainer override detected - test isolation partitioning will wrap it" + $testIsolationParams.InnerScriptBlock = $runAlPipelineParams.RunTestsInBcContainer + } + + $testIsolationTelemetry = [System.Collections.Generic.Dictionary[[System.String], [System.String]]]::new() + Add-TelemetryProperty -Hashtable $testIsolationTelemetry -Key 'PartitionCount' -Value "$($settings.testIsolation.partitions.Count)" + Add-TelemetryProperty -Hashtable $testIsolationTelemetry -Key 'DefaultRunnerCodeunitId' -Value "$($settings.testIsolation.defaultRunnerCodeunitId)" + Add-TelemetryProperty -Hashtable $testIsolationTelemetry -Key 'WrapsCustomOverride' -Value "$($testIsolationParams.ContainsKey('InnerScriptBlock'))" + Trace-Information -Message "Test Isolation enabled" -AdditionalData $testIsolationTelemetry + + $runAlPipelineParams["RunTestsInBcContainer"] = New-PartitionedTestRunnerScriptBlock @testIsolationParams + } + Write-Host "Invoke Run-AlPipeline with buildmode $buildMode" Run-AlPipeline @runAlPipelineParams ` -accept_insiderEula ` diff --git a/README.md b/README.md index 489e9259c..d21e5750d 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth worksh 1. [DeliveryTargets and NuGet/GitHub Packages](Scenarios/DeliveryTargets.md) 1. [Enabling Telemetry for AL-Go workflows and actions](Scenarios/EnablingTelemetry.md) 1. [Add a performance test app to an existing project](Scenarios/AddAPerformanceTestApp.md) +1. [Partition test runs by required isolation](Scenarios/TestIsolation.md) 1. [Publish your app to AppSource](Scenarios/PublishToAppSource.md) 1. [Connect your GitHub repository to Power Platform](Scenarios/SetupPowerPlatform.md) 1. [How to set up Service Principal for Power Platform](Scenarios/SetupServicePrincipalForPowerPlatform.md) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6fbf49591..b0b4f2cc7 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,26 @@ +### Test Isolation - run selected test codeunits under a custom test runner + +Business Central test codeunits can declare a `RequiredTestIsolation` value (BC runtime 16+) that tells the runtime which transactional isolation they need. The standard test runner shipped by BC has a single fixed `TestIsolation` value and cannot satisfy multiple requirements at once. AL-Go for GitHub now supports running tests under multiple runners in a single pipeline pass. + +When `testIsolation.enabled` is set in your settings, AL-Go partitions the test stage: each entry in `partitions` runs the matched codeunits under the configured `runnerCodeunitId`, and everything else runs under `defaultRunnerCodeunitId`. Results are merged back into the same JUnit file downstream reporting already consumes; container lifecycle and `disabledTests.json` handling are unchanged, and an existing `RunTestsInBcContainer.ps1` override is wrapped (each partitioned call is routed through it) rather than replaced. + +Enable it by adding: + +```json +{ + "testIsolation": { + "enabled": true, + "defaultRunnerCodeunitId": 0, + "partitions": [ + { "runnerCodeunitId": 130451, "codeunits": "60200..60299" }, + { "runnerCodeunitId": 130452, "codeunits": "60300|60301" } + ] + } +} +``` + +Requires BC 15+ (the test page must expose the `TestCodeunitRangeFilter` control used by BcContainerHelper for codeunit-range filtering). See the full settings reference and compatibility notes in [Test Isolation](Scenarios/TestIsolation.md). The feature is opt-in - existing projects are unaffected unless they set `testIsolation.enabled = true`. + ## v9.1 ### Resilient Pull Request Status Check for large builds diff --git a/Scenarios/TestIsolation.md b/Scenarios/TestIsolation.md new file mode 100644 index 000000000..ad0685453 --- /dev/null +++ b/Scenarios/TestIsolation.md @@ -0,0 +1,107 @@ +# Partitioning tests by required isolation + +Business Central test codeunits can declare, per codeunit, what transactional isolation they need from the test runner that executes them (`RequiredTestIsolation`, runtime 16+). The standard test runner shipped by BC has a single fixed `TestIsolation` value and cannot satisfy multiple requirements at once. AL-Go for GitHub lets you split a single test stage into multiple runs — each driven by a test runner whose `TestIsolation` matches a chosen group of codeunits — so tests behave the same in CI as they do in the BC Test Tool. + +## Background: three AL properties + +| Property | Applies to | Values | Runtime | +|---|---|---|---| +| `TestIsolation` | Test **runner** codeunit (`Subtype = TestRunner`) | `Disabled` (default), `Codeunit`, `Function` | 1.0 | +| `RequiredTestIsolation` | Test codeunit (`Subtype = Test`) | `None` (default), `Disabled`, `Codeunit`, `Function` | 16.0 (BC 2025 W2+) | +| `TestType` | Test codeunit | `UnitTest` (default), `IntegrationTest`, `Uncategorized`, `AITest` | 16.0 | + +The runner's `TestIsolation` decides actual database rollback behavior after tests execute. The test codeunit's `RequiredTestIsolation` is a declaration of what the codeunit expects. If the runner doesn't satisfy it, the test may fail. See Microsoft's [TestIsolation property](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/properties/devenv-testisolation-property) and [RequiredTestIsolation property](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/properties/devenv-requiredtestisolation-property) docs for full semantics. + +## When to enable this + +Enable `testIsolation` only if some of your test codeunits need a non-default test runner. Projects whose tests all run under the BC default runner do not need this feature. + +You supply the runner codeunits — typically by adopting one of the [BCApps Test Runner](https://github.com/microsoft/BCApps/tree/main/src/Tools/Test%20Framework/Test%20Runner) codeunits, or by authoring your own `Subtype = TestRunner` codeunit with the `TestIsolation` property set to the value you need. Your runner codeunit must be installed in the container at test time (typically as part of a test app or a test-runner app). + +## Configuration + +Add a `testIsolation` block to your `.AL-Go/settings.json` (or any higher-precedence settings location — see [settings.md](settings.md#where-are-the-settings-located)): + +```json +{ + "testIsolation": { + "enabled": true, + "defaultRunnerCodeunitId": 0, + "partitions": [ + { "runnerCodeunitId": 130451, "codeunits": "60200..60299" }, + { "runnerCodeunitId": 130452, "codeunits": "60300|60301" } + ] + } +} +``` + +| Key | Meaning | +|---|---| +| `enabled` | Master switch. `false` (default) — AL-Go uses the standard single-pass test behavior. | +| `defaultRunnerCodeunitId` | Runner used for every test codeunit not matched by an entry in `partitions`. `0` means "let BcContainerHelper pick the BC default runner." | +| `partitions[].runnerCodeunitId` | Codeunit ID of the test runner that will execute the codeunits matched by this entry. Must be a `Subtype = TestRunner` codeunit reachable in the container. | +| `partitions[].codeunits` | The test codeunits to run under that runner: single IDs and closed ranges joined by `\|` — see below. | + +### `codeunits` filter syntax + +`codeunits` accepts single codeunit IDs and closed (two-sided) ranges, joined by `|`: + +| Syntax | Meaning | +|---|---| +| `60100` | exact codeunit ID | +| `60100\|60101\|60102` | enumeration (OR) | +| `60100..60199` | inclusive range | +| `60100\|60200..60299` | combined | + +This is deliberately a subset of the BC filter syntax: AL-Go computes the *complement* of all partitions to route every other codeunit to `defaultRunnerCodeunitId` (the BC filter grammar has no negation over ranges), and that complement is only well-defined for unions of closed intervals. Open-ended ranges (`..60199`, `60200..`) and the `<>`, `&`, `<`, `>` operators are rejected by settings validation. + +## How AL-Go uses this + +When `testIsolation.enabled` is `true`, AL-Go installs a custom `RunTestsInBcContainer` scriptblock into Run-AlPipeline. For each test app, the scriptblock: + +1. Invokes `Run-TestsInBcContainer` once per entry in `partitions`, with `-testRunnerCodeunitId` set to the partition's runner and `-testCodeunitRange` set to the partition's `codeunits` filter. +1. Issues one trailing call under `defaultRunnerCodeunitId` whose `-testCodeunitRange` is the complement of every explicit partition's filter — so every test codeunit not in any partition runs exactly once under the default runner. For the configuration above the complement is `..60199|60302..` (`60300` and `60301` are adjacent to `60200..60299` and merge into one excluded block). + +Container lifecycle, app installation, and `disabledTests.json` discovery continue to be handled by Run-AlPipeline. Results are appended into the same JUnit file that downstream reporting (`AnalyzeTests`) already consumes — no other workflow steps need to change. + +## Combining with a `RunTestsInBcContainer` override + +If your project already ships a [`RunTestsInBcContainer.ps1` override](settings.md#scriptoverrides) in the `.AL-Go` folder, enabling `testIsolation` does not replace it: AL-Go wraps it. Each partitioned call (and the trailing default-runner call) invokes your override with the usual parameter hashtable, extended with the partition's `testCodeunitRange` and `testRunnerCodeunitId` entries. For the partitioning to take effect, your override must forward the hashtable to `Run-TestsInBcContainer` by splatting (`Run-TestsInBcContainer @parameters`) — an override that picks out individual parameters will silently ignore the partition filter and run all tests in every call. + +## Workflow-specific overrides + +The normal AL-Go settings cascade applies. To use partitioning only in your nightly workflow, add a `.AL-Go/.settings.json` containing the `testIsolation` block — the CI workflow runs unchanged. + +## Compatibility + +- **Requires BC 15+ (test page 130455).** Partitioning works by typing a filter expression into the BC test page's `TestCodeunitRangeFilter` control. That control exists on the standard test page used by BC 15 and later. On BC 14 and earlier (test page 130409) the control is not present, BcContainerHelper silently no-ops the filter, and every partition's call would run **all** test codeunits — producing wrong results. If you need this feature on an older BC version, supply a custom `testPage` setting that points to a page exposing the control, or stay on the single-pass behavior. +- **No source changes are required** to use this feature. The relationship between an in-source `RequiredTestIsolation = Function` declaration and a `partitions` entry that routes that codeunit to a `TestIsolation = Function` runner is your responsibility to keep aligned. Future BC tooling may automate this mapping; until then, settings are the source of truth for CI. +- **`disabledTests.json` continues to work transparently.** Run-AlPipeline aggregates the disabled-test list per app and passes it via the scriptblock parameters; AL-Go forwards it to every partition call, so a disabled test is excluded from every runner. +- **Existing projects are not affected** unless they opt in via `testIsolation.enabled = true`. + +## Edge cases + +- **Empty `partitions` with `enabled: true`.** The pipeline issues exactly one `Run-TestsInBcContainer` call per test app under `defaultRunnerCodeunitId` (or BC's default if it is `0`) with no codeunit filter. Use this when you want to swap the standard test runner project-wide without partitioning. +- **A codeunit ID listed in two `partitions` entries.** AL-Go emits a pipeline warning; both entries still fire, so the codeunit runs once per matching partition and appears multiple times in the JUnit results. Make sure your `codeunits` filters do not overlap. +- **A codeunit ID listed in a `partitions` entry but not present in any test app.** BC's filter ignores unmatched IDs — harmless, no error. +- **Partitions covering the entire codeunit ID space.** The trailing default-runner call is skipped — there is nothing left for it to run. +- **Cloud / `useCompilerFolder` runs.** Run-AlPipeline calls the override with `compilerFolder` instead of `containerName`; AL-Go forwards parameters verbatim, so the same partitioning works. +- **BCPT and PageScripting tests** go through different Run-AlPipeline scriptblocks (`-RunBCPTTestsInBcContainer`, `-RunPageScriptingTestsInBcContainer`) and are unaffected by `testIsolation`. + +## Performance + +Each entry in `partitions` adds one `Run-TestsInBcContainer` call per test app, plus one trailing default-runner call. For N partitions and M test apps, that is `(N + 1) * M` invocations — versus 1 invocation per test app today. Each call has fixed per-invocation overhead (BC test page setup, control population). Enable `testIsolation` only when isolation requirements actually demand it. + +The trailing default-runner call fires whenever the partitions leave any part of the codeunit ID space uncovered. If your `partitions` filters happen to cover every test codeunit that exists in the app (but not the whole ID space), the default-runner call still executes with a filter that matches nothing — zero tests run, but the test page setup still costs ~1 extra invocation per test app. AL-Go cannot know which codeunits an app contains, so it cannot skip that call for you; widen a partition range to cover the whole ID space if you want it skipped. + +## Related + +- [TestIsolation property](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/properties/devenv-testisolation-property) +- [RequiredTestIsolation property](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/properties/devenv-requiredtestisolation-property) +- [TestType property](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/properties/devenv-testtype-property) +- [Test Runner codeunits](https://learn.microsoft.com/en-us/dynamics365/business-central/dev-itpro/developer/devenv-testrunner-codeunits) +- [BCApps Test Runner](https://github.com/microsoft/BCApps/tree/main/src/Tools/Test%20Framework/Test%20Runner) + +______________________________________________________________________ + +[back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 6b58c1c37..a76223a76 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -243,6 +243,7 @@ Please read the release notes carefully when installing new versions of AL-Go fo | doNotBuildTests | This setting forces the pipeline to NOT build and run the tests and performance tests in testFolders and bcptTestFolders | false | | doNotRunTests | This setting forces the pipeline to NOT run the tests in testFolders. Tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | | doNotRunBcptTests | This setting forces the pipeline to NOT run the performance tests in testFolders. Performance tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | +| testIsolation | Opts the project into partitioned test runs, so that selected test codeunits are executed by a custom test runner (matching their `RequiredTestIsolation` requirement) and everything else runs under a default runner. Read more at [Test Isolation](TestIsolation.md). | `{ "enabled": false, "defaultRunnerCodeunitId": 0, "partitions": [] }` | | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles (**deprecated**) | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | diff --git a/Tests/ReadSettings.Test.ps1 b/Tests/ReadSettings.Test.ps1 index 569a9bcb6..fc00f4335 100644 --- a/Tests/ReadSettings.Test.ps1 +++ b/Tests/ReadSettings.Test.ps1 @@ -331,6 +331,78 @@ InModuleScope ReadSettings { # Allows testing of private functions Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema | Should -Be $true } + It 'testIsolation default shape is correct' { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.enabled | Should -Be $false + $defaultSettings.testIsolation.defaultRunnerCodeunitId | Should -Be 0 + $defaultSettings.testIsolation.partitions | Should -BeNullOrEmpty + } + + It 'testIsolation accepts a populated partitions array' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.enabled = $true + $defaultSettings.testIsolation.defaultRunnerCodeunitId = 130450 + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 130451; codeunits = '60100|60200..60299' } + [ordered]@{ runnerCodeunitId = 130452; codeunits = '60300' } + ) + Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema | Should -Be $true + } + + It 'testIsolation rejects unknown top-level keys' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.typoKey = 'oops' + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*testIsolation/typoKey*' + } + + It 'testIsolation rejects negative defaultRunnerCodeunitId' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.defaultRunnerCodeunitId = -1 + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*defaultRunnerCodeunitId*' + } + + It 'testIsolation rejects partition entry with runnerCodeunitId = 0' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 0; codeunits = '60100' } + ) + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*runnerCodeunitId*' + } + + It 'testIsolation rejects partition entry with empty codeunits' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 130451; codeunits = '' } + ) + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*codeunits*' + } + + It 'testIsolation rejects codeunits filter syntax outside the supported subset' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + foreach ($invalidFilter in @('<>60100', '60100..', '..60100', '60100&60200')) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 130451; codeunits = $invalidFilter } + ) + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*codeunits*' -Because "'$invalidFilter' should be rejected" + } + } + + It 'testIsolation rejects partition entry missing required keys' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 130451 } + ) + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*codeunits*' + } + + It 'testIsolation rejects unknown keys inside a partition entry' -Skip:($PSVersionTable.PSVersion.Major -lt 7) { + $defaultSettings = GetDefaultSettings + $defaultSettings.testIsolation.partitions = @( + [ordered]@{ runnerCodeunitId = 130451; codeunits = '60100'; extraKey = 'oops' } + ) + { Test-Json -json (ConvertTo-Json $defaultSettings -Depth 99) -schema $schema -ErrorAction Stop } | Should -Throw -ExpectedMessage '*extraKey*' + } + It 'overwriteSettings property resets settings from destination object (simple types)' { $dst = [ordered]@{ setting1 = "value1" diff --git a/Tests/TestIsolation.Test.ps1 b/Tests/TestIsolation.Test.ps1 new file mode 100644 index 000000000..8f2a489de --- /dev/null +++ b/Tests/TestIsolation.Test.ps1 @@ -0,0 +1,366 @@ +Import-Module (Join-Path $PSScriptRoot '../Actions/.Modules/TestIsolation.psm1') -Force + +Describe 'TestIsolation' { + + Context 'New-PartitionedTestRunnerScriptBlock' { + + BeforeAll { + # Installed as a global function (not a Pester Mock) because the + # scriptblock returned by New-PartitionedTestRunnerScriptBlock + # resolves Run-TestsInBcContainer at invocation time through the + # global function table - Mock's command-lookup scope does not + # reach across the .GetNewClosure() boundary. The AfterAll block + # removes the stub so no other *.Test.ps1 sees it. + function global:Run-TestsInBcContainer { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseApprovedVerbs', '', Justification = 'Stub must match the BcContainerHelper function name.')] + [CmdletBinding()] + [OutputType([bool])] + Param( + [string] $testCodeunit, + [string] $testCodeunitRange, + [string] $testRunnerCodeunitId, + [string] $extensionId, + [string] $containerName, + $disabledTests, + [string] $JUnitResultFileName, + [string] $XUnitResultFileName, + [switch] $AppendToJUnitResultFile, + [switch] $AppendToXUnitResultFile, + [switch] $returnTrueIfAllPassed, + [Parameter(ValueFromRemainingArguments = $true)] + $rest + ) + $call = [pscustomobject]@{ + testCodeunitRange = $testCodeunitRange + testRunnerCodeunitId = $testRunnerCodeunitId + extensionId = $extensionId + containerName = $containerName + AppendToJUnitResultFile = [bool] $AppendToJUnitResultFile + hasRangeFilter = $PSBoundParameters.ContainsKey('testCodeunitRange') + hasRunnerId = $PSBoundParameters.ContainsKey('testRunnerCodeunitId') + } + $script:RunTestsInvocations += , $call + if ($script:RunTestsResponse -is [scriptblock]) { + return (& $script:RunTestsResponse $call) + } + return [bool] $script:RunTestsResponse + } + } + + AfterAll { + Remove-Item -Path function:global:Run-TestsInBcContainer -ErrorAction SilentlyContinue + } + + BeforeEach { + $script:RunTestsInvocations = @() + $script:RunTestsResponse = $true + } + + It 'returns a scriptblock' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ defaultRunnerCodeunitId = 0; partitions = @() } + $sb | Should -BeOfType [scriptblock] + } + + It 'with empty partitions and defaultRunnerCodeunitId = 0, invokes default runner once with no filter and no runner id' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @() + } + $result = & $sb @{ extensionId = 'a'; containerName = 'c' } + + $result | Should -Be $true + $script:RunTestsInvocations.Count | Should -Be 1 + $script:RunTestsInvocations[0].hasRangeFilter | Should -Be $false + $script:RunTestsInvocations[0].hasRunnerId | Should -Be $false + } + + It 'with empty partitions and defaultRunnerCodeunitId set, default call uses that runner' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 130450 + partitions = @() + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $script:RunTestsInvocations.Count | Should -Be 1 + $script:RunTestsInvocations[0].testRunnerCodeunitId | Should -Be '130450' + $script:RunTestsInvocations[0].hasRangeFilter | Should -Be $false + } + + It 'invokes one call per partition with that partition runner and exact range string' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60200..60299' } + @{ runnerCodeunitId = 130452; codeunits = '60300|60301' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + # 2 partition calls + 1 default call + $script:RunTestsInvocations.Count | Should -Be 3 + + $script:RunTestsInvocations[0].testRunnerCodeunitId | Should -Be '130451' + $script:RunTestsInvocations[0].testCodeunitRange | Should -Be '60200..60299' + + $script:RunTestsInvocations[1].testRunnerCodeunitId | Should -Be '130452' + $script:RunTestsInvocations[1].testCodeunitRange | Should -Be '60300|60301' + } + + It 'default call uses the complement of every partition filter' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60200..60299' } + @{ runnerCodeunitId = 130452; codeunits = '60300|60301' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + # 60200..60299, 60300 and 60301 are adjacent and merge into 60200..60301 + $defaultCall = $script:RunTestsInvocations[2] + $defaultCall.testCodeunitRange | Should -Be '..60199|60302..' + $defaultCall.hasRunnerId | Should -Be $false + } + + It 'default call complement keeps gaps between non-adjacent partitions' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + @{ runnerCodeunitId = 130452; codeunits = '60200..60299' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $defaultCall = $script:RunTestsInvocations[2] + $defaultCall.testCodeunitRange | Should -Be '..60099|60101..60199|60300..' + } + + It 'default call complement collapses a single-ID gap to a single ID' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100|60102' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $defaultCall = $script:RunTestsInvocations[1] + $defaultCall.testCodeunitRange | Should -Be '..60099|60101|60103..' + } + + It 'default call complement has no leading piece when a partition starts at codeunit ID 1' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '1..60000' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $defaultCall = $script:RunTestsInvocations[1] + $defaultCall.testCodeunitRange | Should -Be '60001..' + } + + It 'skips the default-runner call when partitions cover the entire codeunit ID space' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '1..2147483647' } + ) + } + $result = & $sb @{ extensionId = 'a'; containerName = 'c' } + + $result | Should -Be $true + $script:RunTestsInvocations.Count | Should -Be 1 + $script:RunTestsInvocations[0].testRunnerCodeunitId | Should -Be '130451' + } + + It 'default call uses defaultRunnerCodeunitId together with the complement filter' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 130450 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $defaultCall = $script:RunTestsInvocations[1] + $defaultCall.testRunnerCodeunitId | Should -Be '130450' + $defaultCall.testCodeunitRange | Should -Be '..60099|60101..' + } + + It 'forwards arbitrary Run-AlPipeline parameters to every invocation' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + ) + } + & $sb @{ + extensionId = 'app-a' + containerName = 'mycontainer' + AppendToJUnitResultFile = $true + } | Out-Null + + $script:RunTestsInvocations.Count | Should -Be 2 + $script:RunTestsInvocations | ForEach-Object { + $_.extensionId | Should -Be 'app-a' + $_.containerName | Should -Be 'mycontainer' + $_.AppendToJUnitResultFile | Should -Be $true + } + } + + It 'returns $false if any invocation fails but still runs every call' { + $script:RunTestsResponse = { param($call) $call.testRunnerCodeunitId -ne '130452' } + + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + @{ runnerCodeunitId = 130452; codeunits = '60200' } + ) + } + $result = & $sb @{ extensionId = 'a'; containerName = 'c' } + + $result | Should -Be $false + $script:RunTestsInvocations.Count | Should -Be 3 + } + + It 'trims whitespace in codeunits pieces when building the complement filter' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = ' 60100 | 60200..60299 ' } + ) + } + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $defaultCall = $script:RunTestsInvocations[1] + $defaultCall.testCodeunitRange | Should -Be '..60099|60101..60199|60300..' + } + + It 'works with PSCustomObject partitions (as JSON-deserialised settings would supply)' { + $sb = New-PartitionedTestRunnerScriptBlock -Settings ([pscustomobject]@{ + defaultRunnerCodeunitId = 0 + partitions = @( + [pscustomobject]@{ runnerCodeunitId = 130451; codeunits = '60100' } + ) + }) + & $sb @{ extensionId = 'a'; containerName = 'c' } | Out-Null + + $script:RunTestsInvocations.Count | Should -Be 2 + $script:RunTestsInvocations[0].testRunnerCodeunitId | Should -Be '130451' + $script:RunTestsInvocations[0].testCodeunitRange | Should -Be '60100' + $script:RunTestsInvocations[1].testCodeunitRange | Should -Be '..60099|60101..' + } + + It 'routes every call through a provided InnerScriptBlock instead of Run-TestsInBcContainer' { + $script:innerCalls = @() + $inner = { + Param([Hashtable] $parameters) + $script:innerCalls += , $parameters + return $true + } + $sb = New-PartitionedTestRunnerScriptBlock -InnerScriptBlock $inner -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + ) + } + $result = & $sb @{ extensionId = 'a'; containerName = 'c' } + + $result | Should -Be $true + $script:RunTestsInvocations.Count | Should -Be 0 + $script:innerCalls.Count | Should -Be 2 + $script:innerCalls[0].testRunnerCodeunitId | Should -Be '130451' + $script:innerCalls[0].testCodeunitRange | Should -Be '60100' + $script:innerCalls[1].testCodeunitRange | Should -Be '..60099|60101..' + $script:innerCalls[1].extensionId | Should -Be 'a' + } + + It 'returns $false when the InnerScriptBlock reports a failed partition' { + $inner = { + Param([Hashtable] $parameters) + return ($parameters.testRunnerCodeunitId -ne '130451') + } + $sb = New-PartitionedTestRunnerScriptBlock -InnerScriptBlock $inner -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100' } + ) + } + $result = & $sb @{ extensionId = 'a'; containerName = 'c' } + + $result | Should -Be $false + } + + It 'warns when partition filters overlap' { + Mock -CommandName OutputWarning -ModuleName TestIsolation { } + $null = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100..60200' } + @{ runnerCodeunitId = 130452; codeunits = '60150' } + ) + } + Should -Invoke -CommandName OutputWarning -ModuleName TestIsolation -Times 1 -Exactly + } + + It 'does not warn when partition filters do not overlap' { + Mock -CommandName OutputWarning -ModuleName TestIsolation { } + $null = New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100..60200' } + @{ runnerCodeunitId = 130452; codeunits = '60201' } + ) + } + Should -Invoke -CommandName OutputWarning -ModuleName TestIsolation -Times 0 -Exactly + } + + It 'throws on filter syntax outside the supported subset' { + { + New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '<>60100' } + ) + } + } | Should -Throw -ExpectedMessage "*Unsupported piece '<>60100'*" + } + + It 'throws on open-ended ranges' { + { + New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60100..' } + ) + } + } | Should -Throw -ExpectedMessage "*Unsupported piece '60100..'*" + } + + It 'throws on a reversed range' { + { + New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = '60299..60200' } + ) + } + } | Should -Throw -ExpectedMessage "*lower bound is greater than upper bound*" + } + + It 'throws when a partition filter contains no codeunit IDs' { + { + New-PartitionedTestRunnerScriptBlock -Settings @{ + defaultRunnerCodeunitId = 0 + partitions = @( + @{ runnerCodeunitId = 130451; codeunits = ' | ' } + ) + } + } | Should -Throw -ExpectedMessage "*does not contain any codeunit IDs*" + } + } +}