diff --git a/changelog.md b/changelog.md index 0b6b8ee..78c21ef 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] +### Added +* `Test-AdfCode` warns about enabled triggers without pipeline references, which ADF refuses to start #507 +### Fixed +* Starting and stopping triggers now catches Az errors whatever `$ErrorActionPreference` is, so retries and warnings work; a failed start is a warning and the deployment carries on #508 + ## [1.19.0] - 2026-10-16 ### Added * Stopping a trigger now retries up to 5 times (and waits while the trigger is provisioning), like starting a trigger #404 diff --git a/docs/FEATURES/BUILD_AND_TEST.md b/docs/FEATURES/BUILD_AND_TEST.md index 0e0322f..24083a3 100644 --- a/docs/FEATURES/BUILD_AND_TEST.md +++ b/docs/FEATURES/BUILD_AND_TEST.md @@ -63,7 +63,16 @@ ERROR: Pipeline 'PL_CopyData' references dataset 'DS_Missing' which does not exi ERROR: File 'pipeline\PL_Copy.json' contains object named 'PL_CopyData' (name mismatch) ``` -### 4. Additional Checks +### 4. Trigger Validation +- Warns about enabled triggers (`runtimeState: Started`) without pipeline references +- Azure Data Factory refuses to start such triggers, so the deployment would only report the API error `Missing or invalid pipeline references` + +**Warning Example:** +``` +WARNING: Trigger 'TR_Daily' is enabled (runtimeState: Started) but has no pipeline references. Azure Data Factory will fail to start it. +``` + +### 5. Additional Checks - More validations coming soon... ## Using Test-AdfCode in CI/CD diff --git a/private/Start-Trigger.ps1 b/private/Start-Trigger.ps1 index 8cd37ee..60f1e6f 100644 --- a/private/Start-Trigger.ps1 +++ b/private/Start-Trigger.ps1 @@ -1,5 +1,6 @@ function Start-Trigger { [CmdletBinding()] + [OutputType([bool])] param ( [parameter(Mandatory = $true)] [string] $ResourceGroupName, [parameter(Mandatory = $true)] [string] $DataFactoryName, @@ -10,20 +11,29 @@ function Start-Trigger { Write-Host "- Enabling trigger: $Name" $attempts = 5; $i = 0 + $started = $false while ($i -lt $attempts) { $i++ try { + # Az cmdlets write non-terminating errors, so -ErrorAction Stop is required to catch them Start-AzDataFactoryV2Trigger ` -ResourceGroupName $ResourceGroupName ` -DataFactoryName $DataFactoryName ` -Name $Name ` - -Force | Out-Null + -Force -ErrorAction Stop | Out-Null # If successful, break out of the loop + $started = $true break } catch { $errMsg = $_.Exception.Message + if ($errMsg -like "*Missing or invalid pipeline references*") { + # Permanent error - retry won't help (see Test-AdfCode) + Write-Host "Failed starting trigger." + Write-Warning -Message $errMsg + break + } if ($i -lt $attempts) { if ($errMsg -like "*cannot be updated during provisioning*") { @@ -56,4 +66,5 @@ function Start-Trigger { } Write-Debug "END: Start-Trigger()" + return $started } diff --git a/private/Start-Triggers.ps1 b/private/Start-Triggers.ps1 index 7adedf9..386d223 100644 --- a/private/Start-Triggers.ps1 +++ b/private/Start-Triggers.ps1 @@ -50,12 +50,21 @@ function Start-Triggers { if ($toBeStarted.Count -gt 0) { Write-Host "Starting triggers:" - $toBeStarted | ForEach-Object { - Start-Trigger ` + [System.Collections.ArrayList] $failedToStart = @{} + $toBeStarted | ForEach-Object { + $TrName = $_ + $isStarted = Start-Trigger ` -ResourceGroupName $adf.ResourceGroupName ` -DataFactoryName $adf.Name ` - -Name $_ ` - | Out-Null + -Name $TrName + + if (-not $isStarted) { + $failedToStart.Add($TrName) | Out-Null + } + } + # Deployment carries on: a failed start is reported as a warning only + if ($failedToStart.Count -gt 0) { + Write-Warning "The number of triggers that failed to start: $($failedToStart.Count) ($($failedToStart -join ', '))" } Write-Host "Complete starting triggers." } diff --git a/private/Stop-Trigger.ps1 b/private/Stop-Trigger.ps1 index 33dcc80..d6b2ba9 100644 --- a/private/Stop-Trigger.ps1 +++ b/private/Stop-Trigger.ps1 @@ -16,11 +16,12 @@ function Stop-Trigger { { $i++ try { + # Az cmdlets write non-terminating errors, so -ErrorAction Stop is required to catch them Stop-AzDataFactoryV2Trigger ` -ResourceGroupName $ResourceGroupName ` -DataFactoryName $DataFactoryName ` -Name $Name ` - -Force | Out-Null + -Force -ErrorAction Stop | Out-Null # If successful, break out of the loop $stopped = $true break diff --git a/public/Test-AdfCode.ps1 b/public/Test-AdfCode.ps1 index f1dbcb1..d14a571 100644 --- a/public/Test-AdfCode.ps1 +++ b/public/Test-AdfCode.ps1 @@ -99,6 +99,21 @@ function Test-AdfCode { } } + Write-Host "Checking enabled triggers have pipeline references..." + $adf.Triggers | ForEach-Object { + $props = $_.Body.properties + $propNames = $props.PSObject.Properties.Name + if ('runtimeState' -in $propNames -and $props.runtimeState -eq 'Started') { + # TumblingWindowTrigger uses a single 'pipeline', other trigger types use a 'pipelines' array + $pipelines = @(if ('pipelines' -in $propNames) { $props.pipelines | Where-Object { $_ } }) + $hasPipeline = $pipelines.Count -gt 0 -or ('pipeline' -in $propNames -and $null -ne $props.pipeline) + if (-not $hasPipeline) { + Write-Warning "Trigger '$($_.Name)' is enabled (runtimeState: Started) but has no pipeline references. Azure Data Factory will fail to start it." + $result.WarningCount += 1 + } + } + } + Write-Host "Checking: Global parameter names..." if ($adf.Factories.Count -gt 0) { $gparams = $adf.GlobalFactory.GlobalParameters diff --git a/test/Start-Trigger.Tests.ps1 b/test/Start-Trigger.Tests.ps1 index f14a7a6..d693b1f 100644 --- a/test/Start-Trigger.Tests.ps1 +++ b/test/Start-Trigger.Tests.ps1 @@ -57,6 +57,74 @@ InModuleScope azure.datafactory.tools { } } + + # Az cmdlets report failures as non-terminating errors (they do not throw) #508 + Describe 'Start-Trigger when Az cmdlet writes a non-terminating error #508' -Tag 'Unit' { + + BeforeEach { + $ErrorActionPreference = 'Continue' # default preference, as in most pipelines + Mock Start-Sleep {} + Mock Write-Warning {} + } + + It 'Should return true when started' { + Mock Start-AzDataFactoryV2Trigger { } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue + Should -Invoke Start-AzDataFactoryV2Trigger -Times 1 -Exactly + } + + It 'Should retry a transient error, then warn and return false' { + Mock Start-AzDataFactoryV2Trigger { Write-Error 'Conflict: There are concurrent operations on the trigger' } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse + Should -Invoke Start-AzDataFactoryV2Trigger -Times 5 -Exactly + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*concurrent operations*' } + } + + It 'Should succeed when a retry works' { + $script:startAttempt = 0 + Mock Start-AzDataFactoryV2Trigger { $script:startAttempt++; if ($script:startAttempt -eq 1) { Write-Error 'Conflict: concurrent operations' } } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue + Should -Invoke Start-AzDataFactoryV2Trigger -Times 2 -Exactly + Should -Invoke Write-Warning -Times 0 -Exactly + } + + It 'Should not retry a permanent error (missing pipeline references)' { + Mock Start-AzDataFactoryV2Trigger { Write-Error 'HTTP Status Code: BadRequest Error Code: BadRequest Error Message: Missing or invalid pipeline references for trigger tr1' } + Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse + Should -Invoke Start-AzDataFactoryV2Trigger -Times 1 -Exactly + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*Missing or invalid pipeline references*' } + } + + It 'Should not write the error to the error stream' { + Mock Start-AzDataFactoryV2Trigger { Write-Error 'Missing or invalid pipeline references for trigger tr1' } + $out = Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' 2>&1 + @($out | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] }) | Should -BeNullOrEmpty + } + } + + Describe 'Start-Triggers when a trigger fails to start #508' -Tag 'Unit' { + + BeforeAll { + Mock Start-Trigger { param ($ResourceGroupName, $DataFactoryName, $Name) return ($Name -ne 'tr2') } + Mock Write-Warning {} + + $script:adf = [Adf]::new() + $script:adf.Name = 'adf' + $script:adf.ResourceGroupName = 'rg' + $script:adf.PublishOptions = New-AdfPublishOption + foreach ($n in 'tr1', 'tr2', 'tr3') { + $script:adf.Triggers.Add([PSCustomObject]@{ Name = $n; Body = [PSCustomObject]@{ properties = [PSCustomObject]@{ runtimeState = 'Started' } } }) | Out-Null + } + } + + It 'Should carry on (not throw) and try to start all triggers' { + { Start-Triggers -adf $script:adf } | Should -Not -Throw + Should -Invoke Start-Trigger -Times 3 -Exactly + } + It 'Should warn about triggers which failed to start' { + Should -Invoke Write-Warning -Times 1 -Exactly -Scope Describe -ParameterFilter { $Message -like '*tr2*' -and $Message -notlike '*tr1*' -and $Message -notlike '*tr3*' } + } + } } diff --git a/test/Stop-Trigger.Tests.ps1 b/test/Stop-Trigger.Tests.ps1 index 6f597f4..b7808b2 100644 --- a/test/Stop-Trigger.Tests.ps1 +++ b/test/Stop-Trigger.Tests.ps1 @@ -67,6 +67,36 @@ InModuleScope azure.datafactory.tools { } } + # Az cmdlets report failures as non-terminating errors (they do not throw) #508 + Describe 'Stop-Trigger when Az cmdlet writes a non-terminating error #508' -Tag 'Unit' { + + BeforeEach { + $ErrorActionPreference = 'Continue' # default preference, as in most pipelines + Mock Start-Sleep {} + Mock Write-Warning {} + } + + It 'Should retry a transient error, then warn and return false' { + Mock Stop-AzDataFactoryV2Trigger { Write-Error 'Conflict: There are concurrent operations on the trigger' } + Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse + Should -Invoke Stop-AzDataFactoryV2Trigger -Times 5 -Exactly + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*concurrent operations*' } + } + + It 'Should succeed when a retry works' { + $script:stopAttempt = 0 + Mock Stop-AzDataFactoryV2Trigger { $script:stopAttempt++; if ($script:stopAttempt -eq 1) { Write-Error 'Conflict: concurrent operations' } } + Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue + Should -Invoke Stop-AzDataFactoryV2Trigger -Times 2 -Exactly + } + + It 'Should not write the error to the error stream' { + Mock Stop-AzDataFactoryV2Trigger { Write-Error 'Conflict: concurrent operations' } + $out = Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' 2>&1 + @($out | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] }) | Should -BeNullOrEmpty + } + } + Describe 'Stop-Triggers when a trigger fails to stop' -Tag 'Unit' { BeforeAll { diff --git a/test/Test-AdfCode.Tests.ps1 b/test/Test-AdfCode.Tests.ps1 index dcdda36..04728f1 100644 --- a/test/Test-AdfCode.Tests.ps1 +++ b/test/Test-AdfCode.Tests.ps1 @@ -76,6 +76,51 @@ InModuleScope azure.datafactory.tools { } } + Describe 'Test-AdfCode trigger without pipeline references #507' -Tag 'Unit' { + BeforeEach { + Mock Write-Warning {} + } + + It 'Should warn about enabled trigger TR_without_pipeline in adf2' { + $RootFolder = Join-Path -Path $PSScriptRoot -ChildPath 'adf2' + $script:res = Test-AdfCode -RootFolder $RootFolder + Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*TR_without_pipeline*' } + } + + It 'Should return 0 errors and 1 warning for adf2' { + $script:res.ErrorCount | Should -Be 0 + $script:res.WarningCount | Should -Be 1 + } + + It 'Should for with runtimeState ()' -ForEach @( + @{ case = 'empty pipelines'; type = 'ScheduleTrigger'; state = 'Started'; prop = 'pipelines'; refs = $false; expected = 'warn' } + @{ case = 'no pipelines property'; type = 'ScheduleTrigger'; state = 'Started'; prop = $null; refs = $false; expected = 'warn' } + @{ case = 'empty pipelines'; type = 'BlobEventsTrigger'; state = 'Started'; prop = 'pipelines'; refs = $false; expected = 'warn' } + @{ case = 'no pipeline property'; type = 'TumblingWindowTrigger'; state = 'Started'; prop = $null; refs = $false; expected = 'warn' } + @{ case = 'with pipeline'; type = 'ScheduleTrigger'; state = 'Started'; prop = 'pipelines'; refs = $true; expected = 'not warn' } + @{ case = 'with pipeline'; type = 'TumblingWindowTrigger'; state = 'Started'; prop = 'pipeline'; refs = $true; expected = 'not warn' } + @{ case = 'empty pipelines'; type = 'ScheduleTrigger'; state = 'Stopped'; prop = 'pipelines'; refs = $false; expected = 'not warn' } + @{ case = 'no pipeline property'; type = 'TumblingWindowTrigger'; state = 'Stopped'; prop = $null; refs = $false; expected = 'not warn' } + ) { + $RootFolder = Join-Path -Path $TestDrive -ChildPath "adf435-$([guid]::NewGuid().ToString('N').Substring(0,8))" + New-Item -ItemType Directory -Path "$RootFolder/trigger", "$RootFolder/pipeline" -Force | Out-Null + '{ "name": "PL_Wait", "properties": { "activities": [] } }' | Set-Content "$RootFolder/pipeline/PL_Wait.json" + + $pipelineRef = @{ pipelineReference = @{ referenceName = 'PL_Wait'; type = 'PipelineReference' } } + $properties = [ordered]@{ runtimeState = $state; type = $type; typeProperties = @{} } + switch ($prop) { + 'pipelines' { $properties.pipelines = if ($refs) { @($pipelineRef) } else { @() } } + 'pipeline' { $properties.pipeline = $pipelineRef } + } + @{ name = 'TR_Test'; properties = $properties } | ConvertTo-Json -Depth 10 | Set-Content "$RootFolder/trigger/TR_Test.json" + + Test-AdfCode -RootFolder $RootFolder | Out-Null + + $times = if ($expected -eq 'warn') { 1 } else { 0 } + Should -Invoke Write-Warning -Times $times -Exactly -ParameterFilter { $Message -like '*TR_Test*' } + } + } + Describe 'Test-AdfCode' -Tag 'Unit' { It 'Should not throw an error when Get-ChildItem Path parameter is null' { $DataFactoryName = "adf2" diff --git a/test/Triggers.Tests.ps1 b/test/Triggers.Tests.ps1 index 1f940fd..b98acbb 100644 --- a/test/Triggers.Tests.ps1 +++ b/test/Triggers.Tests.ps1 @@ -46,6 +46,7 @@ InModuleScope azure.datafactory.tools { $o = $script:TargetAdf.GetObjectByFullName("*triggers.$Name") $o.RuntimeState = 'Started' $o.Body.properties.RuntimeState = $o.RuntimeState + return $true } Mock Stop-TargetTrigger { param ($Name, $ResourceGroupName, $DataFactoryName)