Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## [Unreleased]
### Added
* `Test-AdfCode` warns about enabled triggers without pipeline references, which ADF refuses to start #507
### Fixed
* Starting and stopping triggers now catches Az errors whatever `$ErrorActionPreference` is, so retries and warnings work; a failed start is a warning and the deployment carries on #508

## [1.19.0] - 2026-10-16
### Added
* Stopping a trigger now retries up to 5 times (and waits while the trigger is provisioning), like starting a trigger #404
Expand Down
11 changes: 10 additions & 1 deletion docs/FEATURES/BUILD_AND_TEST.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,16 @@ ERROR: Pipeline 'PL_CopyData' references dataset 'DS_Missing' which does not exi
ERROR: File 'pipeline\PL_Copy.json' contains object named 'PL_CopyData' (name mismatch)
```

### 4. Additional Checks
### 4. Trigger Validation
- Warns about enabled triggers (`runtimeState: Started`) without pipeline references
- Azure Data Factory refuses to start such triggers, so the deployment would only report the API error `Missing or invalid pipeline references`

**Warning Example:**
```
WARNING: Trigger 'TR_Daily' is enabled (runtimeState: Started) but has no pipeline references. Azure Data Factory will fail to start it.
```

### 5. Additional Checks
- More validations coming soon...

## Using Test-AdfCode in CI/CD
Expand Down
13 changes: 12 additions & 1 deletion private/Start-Trigger.ps1
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
function Start-Trigger {
[CmdletBinding()]
[OutputType([bool])]
param (
[parameter(Mandatory = $true)] [string] $ResourceGroupName,
[parameter(Mandatory = $true)] [string] $DataFactoryName,
Expand All @@ -10,20 +11,29 @@ function Start-Trigger {
Write-Host "- Enabling trigger: $Name"
$attempts = 5;
$i = 0
$started = $false
while ($i -lt $attempts)
{
$i++
try {
# Az cmdlets write non-terminating errors, so -ErrorAction Stop is required to catch them
Start-AzDataFactoryV2Trigger `
-ResourceGroupName $ResourceGroupName `
-DataFactoryName $DataFactoryName `
-Name $Name `
-Force | Out-Null
-Force -ErrorAction Stop | Out-Null
# If successful, break out of the loop
$started = $true
break
}
catch {
$errMsg = $_.Exception.Message
if ($errMsg -like "*Missing or invalid pipeline references*") {
# Permanent error - retry won't help (see Test-AdfCode)
Write-Host "Failed starting trigger."
Write-Warning -Message $errMsg
break
}
if ($i -lt $attempts)
{
if ($errMsg -like "*cannot be updated during provisioning*") {
Expand Down Expand Up @@ -56,4 +66,5 @@ function Start-Trigger {
}

Write-Debug "END: Start-Trigger()"
return $started
}
17 changes: 13 additions & 4 deletions private/Start-Triggers.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,21 @@ function Start-Triggers {
if ($toBeStarted.Count -gt 0)
{
Write-Host "Starting triggers:"
$toBeStarted | ForEach-Object {
Start-Trigger `
[System.Collections.ArrayList] $failedToStart = @{}
$toBeStarted | ForEach-Object {
$TrName = $_
$isStarted = Start-Trigger `
-ResourceGroupName $adf.ResourceGroupName `
-DataFactoryName $adf.Name `
-Name $_ `
| Out-Null
-Name $TrName

if (-not $isStarted) {
$failedToStart.Add($TrName) | Out-Null
}
}
# Deployment carries on: a failed start is reported as a warning only
if ($failedToStart.Count -gt 0) {
Write-Warning "The number of triggers that failed to start: $($failedToStart.Count) ($($failedToStart -join ', '))"
}
Write-Host "Complete starting triggers."
}
Expand Down
3 changes: 2 additions & 1 deletion private/Stop-Trigger.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ function Stop-Trigger {
{
$i++
try {
# Az cmdlets write non-terminating errors, so -ErrorAction Stop is required to catch them
Stop-AzDataFactoryV2Trigger `
-ResourceGroupName $ResourceGroupName `
-DataFactoryName $DataFactoryName `
-Name $Name `
-Force | Out-Null
-Force -ErrorAction Stop | Out-Null
# If successful, break out of the loop
$stopped = $true
break
Expand Down
15 changes: 15 additions & 0 deletions public/Test-AdfCode.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ function Test-AdfCode {
}
}

Write-Host "Checking enabled triggers have pipeline references..."
$adf.Triggers | ForEach-Object {
$props = $_.Body.properties
$propNames = $props.PSObject.Properties.Name
if ('runtimeState' -in $propNames -and $props.runtimeState -eq 'Started') {
# TumblingWindowTrigger uses a single 'pipeline', other trigger types use a 'pipelines' array
$pipelines = @(if ('pipelines' -in $propNames) { $props.pipelines | Where-Object { $_ } })
$hasPipeline = $pipelines.Count -gt 0 -or ('pipeline' -in $propNames -and $null -ne $props.pipeline)
if (-not $hasPipeline) {
Write-Warning "Trigger '$($_.Name)' is enabled (runtimeState: Started) but has no pipeline references. Azure Data Factory will fail to start it."
$result.WarningCount += 1
}
}
}

Write-Host "Checking: Global parameter names..."
if ($adf.Factories.Count -gt 0) {
$gparams = $adf.GlobalFactory.GlobalParameters
Expand Down
68 changes: 68 additions & 0 deletions test/Start-Trigger.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,74 @@ InModuleScope azure.datafactory.tools {
}

}

# Az cmdlets report failures as non-terminating errors (they do not throw) #508
Describe 'Start-Trigger when Az cmdlet writes a non-terminating error #508' -Tag 'Unit' {

BeforeEach {
$ErrorActionPreference = 'Continue' # default preference, as in most pipelines
Mock Start-Sleep {}
Mock Write-Warning {}
}

It 'Should return true when started' {
Mock Start-AzDataFactoryV2Trigger { }
Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue
Should -Invoke Start-AzDataFactoryV2Trigger -Times 1 -Exactly
}

It 'Should retry a transient error, then warn and return false' {
Mock Start-AzDataFactoryV2Trigger { Write-Error 'Conflict: There are concurrent operations on the trigger' }
Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse
Should -Invoke Start-AzDataFactoryV2Trigger -Times 5 -Exactly
Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*concurrent operations*' }
}

It 'Should succeed when a retry works' {
$script:startAttempt = 0
Mock Start-AzDataFactoryV2Trigger { $script:startAttempt++; if ($script:startAttempt -eq 1) { Write-Error 'Conflict: concurrent operations' } }
Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue
Should -Invoke Start-AzDataFactoryV2Trigger -Times 2 -Exactly
Should -Invoke Write-Warning -Times 0 -Exactly
}

It 'Should not retry a permanent error (missing pipeline references)' {
Mock Start-AzDataFactoryV2Trigger { Write-Error 'HTTP Status Code: BadRequest Error Code: BadRequest Error Message: Missing or invalid pipeline references for trigger tr1' }
Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse
Should -Invoke Start-AzDataFactoryV2Trigger -Times 1 -Exactly
Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*Missing or invalid pipeline references*' }
}

It 'Should not write the error to the error stream' {
Mock Start-AzDataFactoryV2Trigger { Write-Error 'Missing or invalid pipeline references for trigger tr1' }
$out = Start-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' 2>&1
@($out | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] }) | Should -BeNullOrEmpty
}
}

Describe 'Start-Triggers when a trigger fails to start #508' -Tag 'Unit' {

BeforeAll {
Mock Start-Trigger { param ($ResourceGroupName, $DataFactoryName, $Name) return ($Name -ne 'tr2') }
Mock Write-Warning {}

$script:adf = [Adf]::new()
$script:adf.Name = 'adf'
$script:adf.ResourceGroupName = 'rg'
$script:adf.PublishOptions = New-AdfPublishOption
foreach ($n in 'tr1', 'tr2', 'tr3') {
$script:adf.Triggers.Add([PSCustomObject]@{ Name = $n; Body = [PSCustomObject]@{ properties = [PSCustomObject]@{ runtimeState = 'Started' } } }) | Out-Null
}
}

It 'Should carry on (not throw) and try to start all triggers' {
{ Start-Triggers -adf $script:adf } | Should -Not -Throw
Should -Invoke Start-Trigger -Times 3 -Exactly
}
It 'Should warn about triggers which failed to start' {
Should -Invoke Write-Warning -Times 1 -Exactly -Scope Describe -ParameterFilter { $Message -like '*tr2*' -and $Message -notlike '*tr1*' -and $Message -notlike '*tr3*' }
}
}
}


Expand Down
30 changes: 30 additions & 0 deletions test/Stop-Trigger.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,36 @@ InModuleScope azure.datafactory.tools {
}
}

# Az cmdlets report failures as non-terminating errors (they do not throw) #508
Describe 'Stop-Trigger when Az cmdlet writes a non-terminating error #508' -Tag 'Unit' {

BeforeEach {
$ErrorActionPreference = 'Continue' # default preference, as in most pipelines
Mock Start-Sleep {}
Mock Write-Warning {}
}

It 'Should retry a transient error, then warn and return false' {
Mock Stop-AzDataFactoryV2Trigger { Write-Error 'Conflict: There are concurrent operations on the trigger' }
Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeFalse
Should -Invoke Stop-AzDataFactoryV2Trigger -Times 5 -Exactly
Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*concurrent operations*' }
}

It 'Should succeed when a retry works' {
$script:stopAttempt = 0
Mock Stop-AzDataFactoryV2Trigger { $script:stopAttempt++; if ($script:stopAttempt -eq 1) { Write-Error 'Conflict: concurrent operations' } }
Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' | Should -BeTrue
Should -Invoke Stop-AzDataFactoryV2Trigger -Times 2 -Exactly
}

It 'Should not write the error to the error stream' {
Mock Stop-AzDataFactoryV2Trigger { Write-Error 'Conflict: concurrent operations' }
$out = Stop-Trigger -ResourceGroupName 'rg' -DataFactoryName 'adf' -Name 'tr1' 2>&1
@($out | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] }) | Should -BeNullOrEmpty
}
}

Describe 'Stop-Triggers when a trigger fails to stop' -Tag 'Unit' {

BeforeAll {
Expand Down
45 changes: 45 additions & 0 deletions test/Test-AdfCode.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,51 @@ InModuleScope azure.datafactory.tools {
}
}

Describe 'Test-AdfCode trigger without pipeline references #507' -Tag 'Unit' {
BeforeEach {
Mock Write-Warning {}
}

It 'Should warn about enabled trigger TR_without_pipeline in adf2' {
$RootFolder = Join-Path -Path $PSScriptRoot -ChildPath 'adf2'
$script:res = Test-AdfCode -RootFolder $RootFolder
Should -Invoke Write-Warning -Times 1 -Exactly -ParameterFilter { $Message -like '*TR_without_pipeline*' }
}

It 'Should return 0 errors and 1 warning for adf2' {
$script:res.ErrorCount | Should -Be 0
$script:res.WarningCount | Should -Be 1
}

It 'Should <expected> for <type> with runtimeState <state> (<case>)' -ForEach @(
@{ case = 'empty pipelines'; type = 'ScheduleTrigger'; state = 'Started'; prop = 'pipelines'; refs = $false; expected = 'warn' }
@{ case = 'no pipelines property'; type = 'ScheduleTrigger'; state = 'Started'; prop = $null; refs = $false; expected = 'warn' }
@{ case = 'empty pipelines'; type = 'BlobEventsTrigger'; state = 'Started'; prop = 'pipelines'; refs = $false; expected = 'warn' }
@{ case = 'no pipeline property'; type = 'TumblingWindowTrigger'; state = 'Started'; prop = $null; refs = $false; expected = 'warn' }
@{ case = 'with pipeline'; type = 'ScheduleTrigger'; state = 'Started'; prop = 'pipelines'; refs = $true; expected = 'not warn' }
@{ case = 'with pipeline'; type = 'TumblingWindowTrigger'; state = 'Started'; prop = 'pipeline'; refs = $true; expected = 'not warn' }
@{ case = 'empty pipelines'; type = 'ScheduleTrigger'; state = 'Stopped'; prop = 'pipelines'; refs = $false; expected = 'not warn' }
@{ case = 'no pipeline property'; type = 'TumblingWindowTrigger'; state = 'Stopped'; prop = $null; refs = $false; expected = 'not warn' }
) {
$RootFolder = Join-Path -Path $TestDrive -ChildPath "adf435-$([guid]::NewGuid().ToString('N').Substring(0,8))"
New-Item -ItemType Directory -Path "$RootFolder/trigger", "$RootFolder/pipeline" -Force | Out-Null
'{ "name": "PL_Wait", "properties": { "activities": [] } }' | Set-Content "$RootFolder/pipeline/PL_Wait.json"

$pipelineRef = @{ pipelineReference = @{ referenceName = 'PL_Wait'; type = 'PipelineReference' } }
$properties = [ordered]@{ runtimeState = $state; type = $type; typeProperties = @{} }
switch ($prop) {
'pipelines' { $properties.pipelines = if ($refs) { @($pipelineRef) } else { @() } }
'pipeline' { $properties.pipeline = $pipelineRef }
}
@{ name = 'TR_Test'; properties = $properties } | ConvertTo-Json -Depth 10 | Set-Content "$RootFolder/trigger/TR_Test.json"

Test-AdfCode -RootFolder $RootFolder | Out-Null

$times = if ($expected -eq 'warn') { 1 } else { 0 }
Should -Invoke Write-Warning -Times $times -Exactly -ParameterFilter { $Message -like '*TR_Test*' }
}
}

Describe 'Test-AdfCode' -Tag 'Unit' {
It 'Should not throw an error when Get-ChildItem Path parameter is null' {
$DataFactoryName = "adf2"
Expand Down
1 change: 1 addition & 0 deletions test/Triggers.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ InModuleScope azure.datafactory.tools {
$o = $script:TargetAdf.GetObjectByFullName("*triggers.$Name")
$o.RuntimeState = 'Started'
$o.Body.properties.RuntimeState = $o.RuntimeState
return $true
}
Mock Stop-TargetTrigger {
param ($Name, $ResourceGroupName, $DataFactoryName)
Expand Down
Loading