diff --git a/CLAUDE.md b/CLAUDE.md index 6e2b1de..e76a11e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,9 +33,11 @@ ## Tests (Pester) +- Set up a local environment with `./tools/Install-DevPrerequisites.ps1`: it installs the Az modules, Pester 6+ and PSScriptAnalyzer for the current user (only what is missing) and checks Node.js/npm. - Tests require **Pester 6.0 or newer** (enforced by `test/!RunAllTests.ps1`, which installs it on CI runners). - Do **not** use `Assert-MockCalled` or `Assert-VerifiableMock` (removed in Pester 6). Use `Should -Invoke -Times N -Exactly` instead; add `-Scope Describe` when the mocked command was called in `BeforeAll`. - Reason: calling `Assert-MockCalled` makes PowerShell autoload the old built-in Pester 3.4.0 (`C:\Program Files\WindowsPowerShell\Modules`), which takes over `Mock`/`Should` and breaks the whole test run. +- Every test that needs access to Azure (a real factory, storage or any Az call that is not mocked) must be in a `Describe` tagged `Integration`; all other tests must run without `Connect-AzAccount`. Put Azure calls in `BeforeAll`/`It`, never in the file or `Describe` body: that code runs at discovery, before tag filters apply. - **Bug fixes start with a test** that reproduces the bug; run it and see it fail before writing the fix. - Mocks of Az cmdlets must `Write-Error` (non-terminating, like the real cmdlets), not `throw`. Mocks that `throw` hid the bugs in #508 and #295. - CI runs with `$ErrorActionPreference = 'Stop'`. Run new or changed tests under both `Stop` and `Continue`; set `$ErrorActionPreference = 'Continue'` in `BeforeEach` to test the default behaviour. diff --git a/changelog.md b/changelog.md index c8e1f6c..fd57ee9 100644 --- a/changelog.md +++ b/changelog.md @@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). * Deleting objects not in source retries temporary API errors and stops on an unsupported object type whatever `$ErrorActionPreference` is #30 * A failed object deployment now stops `Publish-AdfV2FromJson` whatever `$ErrorActionPreference` is, instead of being skipped and the object marked as deployed #295 * Starting and stopping triggers now catches Az errors whatever `$ErrorActionPreference` is, so retries and warnings work; a failed start is a warning and the deployment carries on #508 +* `DeleteNotInSource`: objects skipped as excluded are no longer counted as deleted, and each skipped object is logged #512 +* `DeleteNotInSource` and `-DryRun`: `@folder` filter rules now match objects that exist only in ADF (folder read via REST API), and a rule without `@folder` matches any folder #513 * `Export-AdfToArmTemplate` calls `npm.cmd` on Windows (avoids npm.ps1 failing on Windows PowerShell 5.1) and stops when an npm command fails #462 ## [1.19.0] - 2026-10-16 diff --git a/en-us/messages_index.md b/en-us/messages_index.md index ff07149..9e13236 100644 --- a/en-us/messages_index.md +++ b/en-us/messages_index.md @@ -38,4 +38,7 @@ ADFT0029 | Unknown object type: *type*. ADFT0030 | AzType '$AzType' is not supported. ADFT0031 | Empty value in config file. Path: [*path*]. Check previous warnings. ADFT0032 | The process is exiting the function. Do fix the issue and run again. -ADFT0033 | Incremental Deployment Option DISABLED as Storage Uri is not provided. \ No newline at end of file +ADFT0033 | Incremental Deployment Option DISABLED as Storage Uri is not provided. +ADFT0034 | *[type].[name]* cannot be deleted since it is referenced by *name*, which is not selected for deletion. +ADFT0035 | Unexpected response code: *code* from REST API when listing *types*. +ADFT0036 | Unable to read folder of *[type].[name]* from ADF service. Status Code: *code* diff --git a/private/!AdfPublishOption.class.ps1 b/private/!AdfPublishOption.class.ps1 index 7d138fa..24e307f 100644 --- a/private/!AdfPublishOption.class.ps1 +++ b/private/!AdfPublishOption.class.ps1 @@ -27,4 +27,12 @@ class AdfPublishOption { [TriggerStopTypes] $TriggerStopMethod = [TriggerStopTypes]::AllEnabled [TriggerStartTypes] $TriggerStartMethod = [TriggerStartTypes]::BasedOnSourceCode [Int32] $MaxRetries = 3 + + [Boolean] HasFolderRules() + { + foreach ($rule in @($this.Includes.Keys) + @($this.Excludes.Keys)) { + if ($rule -like '*@*') { return $true } + } + return $false + } } diff --git a/private/AdfObjectName.class.ps1 b/private/AdfObjectName.class.ps1 index 4b4f4d2..cb7cee0 100644 --- a/private/AdfObjectName.class.ps1 +++ b/private/AdfObjectName.class.ps1 @@ -65,29 +65,36 @@ class AdfObjectName { return $null -ne $r } - [Boolean] IsNameExcluded ([AdfPublishOption] $opt) + # Same rule semantics as AdfObject.IsNameMatch: 'type.name[@folder]', folder is checked only when the rule has one + [Boolean] IsRuleMatch ([string] $rule) { - $fullname = $this.FullNameWithFolder() + $arr = $rule.Split('@') + $r = $this.FullName() -like $arr[0] + if ($arr.Count -gt 1) { + $r = $r -and ($this.Folder -like $arr[1]) + } + return $r + } + [Boolean] IsNameExcluded ([AdfPublishOption] $opt) + { # One can exclude objects by listing them explicitly in Excludes collection, ... - $excPatterns = $opt.Excludes.Keys - $r = $excPatterns | Where-Object { $fullname -like $_ } - if ($null -ne $r) - { + foreach ($rule in $opt.Excludes.Keys) { # Means: object is excluded if matches any item in (Excludes) collection - return $true + if ($this.IsRuleMatch($rule)) { return $true } } # ... or by listing them implicitly in Includes collection: - $incPatterns = $opt.Includes.Keys - if ($incPatterns.Count -eq 0) - { + if ($opt.Includes.Keys.Count -eq 0) + { # If no items = all objects match => object is not excluded - return $false + return $false + } + foreach ($rule in $opt.Includes.Keys) { + if ($this.IsRuleMatch($rule)) { return $false } } - $r = $incPatterns | Where-Object { $fullname -like $_ } # Means: object is excluded if not match any item in (Includes) collection - return ($null -eq $r) + return $true } } diff --git a/private/ApplyExclusionOptions.ps1 b/private/ApplyExclusionOptions.ps1 index be91f8b..4cab8d1 100644 --- a/private/ApplyExclusionOptions.ps1 +++ b/private/ApplyExclusionOptions.ps1 @@ -166,6 +166,9 @@ function Get-DryRunPlan { $oname = [AdfObjectName]::new($fullName) $canDelete = $true if ($adf.PublishOptions.DoNotDeleteExcludedObjects) { + if ($adf.PublishOptions.HasFolderRules()) { + $oname.Folder = Get-AdfObjectFolderFromService -adfInstance $targetAdfInstance -type $oname.Type -name $oname.Name + } $canDelete = !($oname.IsNameExcluded($adf.PublishOptions)) } if ($canDelete) { diff --git a/private/Get-AdfObjectFolderFromService.ps1 b/private/Get-AdfObjectFolderFromService.ps1 new file mode 100644 index 0000000..150ba92 --- /dev/null +++ b/private/Get-AdfObjectFolderFromService.ps1 @@ -0,0 +1,40 @@ +function Get-AdfObjectFolderFromService { + [CmdletBinding()] + [OutputType([string])] + param ( + [parameter(Mandatory = $true)] $adfInstance, + [parameter(Mandatory = $true)] [String] $type, + [parameter(Mandatory = $true)] [String] $name + ) + + Write-Debug "BEGIN: Get-AdfObjectFolderFromService(type=$type, name=$name)" + + # Only these types have folders. Az cmdlets do not return the folder of a pipeline, hence REST API + $typePlural = switch ($type) { + 'Pipeline' { 'pipelines' } + 'Dataset' { 'datasets' } + 'DataFlow' { 'dataflows' } + default { $null } + } + if ($null -eq $typePlural) { + return '' + } + + $url = "$($script:BaseApiUrl)$($adfInstance.Id)/$typePlural/$($name)?api-version=2018-06-01" + $r = Invoke-AdfWithRetry -Activity "Reading folder of [$type].[$name]" -ScriptBlock { + $resp = Invoke-AzRestMethod -Method 'GET' -Uri $url -ErrorAction Stop + if ($resp.StatusCode -ne 200) { + throw "ADFT0036: Unable to read folder of [$type].[$name] from ADF service. Status Code: $($resp.StatusCode)" + } + $resp + } + + $folder = '' + $props = ($r.Content | ConvertFrom-Json).properties + if ($null -ne $props -and 'folder' -in $props.PSObject.Properties.Name) { + $folder = $props.folder.name + } + + Write-Debug "END: Get-AdfObjectFolderFromService() = '$folder'" + return $folder +} diff --git a/private/Remove-AdfObject.ps1 b/private/Remove-AdfObject.ps1 index 5c7792d..fc9bdec 100644 --- a/private/Remove-AdfObject.ps1 +++ b/private/Remove-AdfObject.ps1 @@ -31,13 +31,18 @@ function Remove-AdfObject { if ($null -eq $adfSource) { Write-Host "Removing object: [$simtype].[$name]" } else { - [AdfObjectName] $oname = [AdfObjectName]::new("$simType.$name") + # The folder is only needed (and read from the service) when any filter rule uses '@folder' + $folder = '' + if ($adfSource.PublishOptions.HasFolderRules()) { + $folder = Get-AdfObjectFolderFromService -adfInstance $adfInstance -type $simtype -name $name + } + [AdfObjectName] $oname = [AdfObjectName]::new($name, $simtype, $folder) $IsExcluded = $oname.IsNameExcluded($adfSource.PublishOptions) if (-not $IsExcluded) { Write-Host "Removing object: [$simtype].[$name]" } else { if ($adfSource.PublishOptions.DoNotDeleteExcludedObjects) { - Write-Verbose "Object [$simtype].[$name] won't be deleted as publish option 'DoNotDeleteExcludedObjects' = true." + Write-Host "Skipping excluded object: [$simtype].[$name] as publish option 'DoNotDeleteExcludedObjects' = true." $action = "DoNothing" } else { Write-Host "Removing excluded object: [$simtype].[$name] as publish option 'DoNotDeleteExcludedObjects' = false." @@ -141,8 +146,9 @@ function Remove-AdfObject { } elseif ($null -ne $ErrorMessage) { #Rethrow exception throw $ErrorMessage - } elseif ($null -ne $Deleted -and $action -ne 'DoNothing') { - $Deleted.Add($obj) | Out-Null + } elseif ($action -ne 'DoNothing') { + if ($null -ne $Deleted) { $Deleted.Add($obj) | Out-Null } + if ($null -ne $adfSource) { $adfSource.DeletedObjectNames.Add("$simtype.$name") | Out-Null } } Write-Debug "END: Remove-AdfObject()" diff --git a/private/Remove-AdfObjectIfNotInSource.ps1 b/private/Remove-AdfObjectIfNotInSource.ps1 index 3eaaec8..0cbb7ed 100644 --- a/private/Remove-AdfObjectIfNotInSource.ps1 +++ b/private/Remove-AdfObjectIfNotInSource.ps1 @@ -15,11 +15,15 @@ function Remove-AdfObjectIfNotInSource { $simtype = ConvertTo-AdfType -AzType $adfTargetObj.Type } $src = Get-AdfObjectByName -adf $adfSource -name $name -type $simtype - if (!$src) + if ($adfSource.IsObjectDeleted("$simtype.$name")) + { + Write-Verbose "Object [$simtype].[$name] has already been deleted (it referenced another deleted object)." + } + elseif (!$src) { Write-Verbose "Object [$simtype].[$name] hasn't been found in the source - to be deleted." + # Remove-AdfObject adds the object to DeletedObjectNames only when it has really been deleted Remove-AdfObject -adfSource $adfSource -obj $adfTargetObj -adfInstance $adfInstance - $adfSource.DeletedObjectNames.Add("$simtype.$name") } else { Write-Verbose "Object [$simtype].[$name] is in the source - won't be delete." diff --git a/test/Incremental-Deployment.Tests.ps1 b/test/Incremental-Deployment.Tests.ps1 index 9a0e23e..78bad3e 100644 --- a/test/Incremental-Deployment.Tests.ps1 +++ b/test/Incremental-Deployment.Tests.ps1 @@ -127,10 +127,10 @@ InModuleScope azure.datafactory.tools { return $script:TargetAdf } Mock Set-AzDataFactoryV2 { - param ($ResourceGroupName, $DataFactoryName, $Location) + param ($ResourceGroupName, $Name, $Location) if ($null -eq $script:TargetAdf) { $script:TargetAdf = CreateTargetAdf - $script:TargetAdf.Name = $DataFactoryName + $script:TargetAdf.Name = $Name $script:TargetAdf.ResourceGroupName = $ResourceGroupName $script:TargetAdf.Location = $Location } diff --git a/test/Integration.Tests.ps1 b/test/Integration.Tests.ps1 index 8c84c06..a470073 100644 --- a/test/Integration.Tests.ps1 +++ b/test/Integration.Tests.ps1 @@ -12,8 +12,7 @@ InModuleScope azure.datafactory.tools { # Variables for use in tests $script:ResourceGroupName = 'rg-devops-factory' $script:Stage = 'UAT' - $c = Get-AzContext - $script:guid = $c.Subscription.Id.Substring(0,8) + $script:guid = Get-TestSubscriptionPrefix $script:DataFactoryOrigName = 'BigFactorySample2' $script:DataFactoryName = $script:DataFactoryOrigName + "-$guid" $script:SrcFolder = "$PSScriptRoot\$($script:DataFactoryOrigName)" diff --git a/test/Publish-AdfV2FromJson-1.Tests.ps1 b/test/Publish-AdfV2FromJson-1.Tests.ps1 index 5969225..113bfa2 100644 --- a/test/Publish-AdfV2FromJson-1.Tests.ps1 +++ b/test/Publish-AdfV2FromJson-1.Tests.ps1 @@ -23,7 +23,7 @@ InModuleScope azure.datafactory.tools { } } - Describe 'Publish-AdfV2FromJson' { + Describe 'Publish-AdfV2FromJson' -Tag 'Integration' { It 'adf2 Should skip referenced Synapse notebook in a pipeline' { $script:RootFolder = Join-Path $PSScriptRoot "adf2" $o = New-AdfPublishOption diff --git a/test/Publish-AdfV2FromJson-2.Tests.ps1 b/test/Publish-AdfV2FromJson-2.Tests.ps1 index 34f7306..e6710ab 100644 --- a/test/Publish-AdfV2FromJson-2.Tests.ps1 +++ b/test/Publish-AdfV2FromJson-2.Tests.ps1 @@ -16,7 +16,6 @@ InModuleScope azure.datafactory.tools { $script:DataFactoryOrigName = $t.DataFactoryOrigName $script:DataFactoryName = $t.DataFactoryName $script:Location = $t.Location - $script:adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" #$script:RootFolder = Join-Path $PSScriptRoot "BigFactorySample2" $script:opt = New-AdfPublishOption @@ -35,12 +34,21 @@ InModuleScope azure.datafactory.tools { It 'Should exist' { { Get-Command -Name Get-AzDFV2Credential -ErrorAction Stop } | Should -Not -Throw } + } + + Describe 'Get-AzDFV2Credential' -Tag 'Integration' { + BeforeAll { + $script:adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" + } It 'Should run successfully' { { Get-AzDFV2Credential -adfi $adfi | ToArray } | Should -Not -Throw } - } + } - Describe 'Publish-AdfV2FromJson' { + Describe 'Publish-AdfV2FromJson' -Tag 'Integration' { + BeforeAll { + $script:adfi = Get-AzDataFactoryV2 -ResourceGroupName "$ResourceGroupName" -Name "$DataFactoryName" + } It 'BigFactorySample2 should deploy credential object successfully' { $o = New-AdfPublishOption $o.StopStartTriggers = $false diff --git a/test/Remove-AdfObject.Tests.ps1 b/test/Remove-AdfObject.Tests.ps1 index bb7e1c9..e30ca56 100644 --- a/test/Remove-AdfObject.Tests.ps1 +++ b/test/Remove-AdfObject.Tests.ps1 @@ -18,21 +18,24 @@ InModuleScope azure.datafactory.tools { $script:ResourceGroupName = $t.ResourceGroupName $script:Stage = 'UAT' $script:RootFolder = Join-Path $PSScriptRoot $t.DataFactoryOrigName - $script:adf = Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder - $adf.ResourceGroupName = $t.ResourceGroupName - $adf.Region = $t.Location - $o = New-AdfPublishOption - $o.StopStartTriggers = $false - $adf.PublishOptions = $o - Publish-AdfV2FromJson -RootFolder $RootFolder -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -Option $o -Location $Location - $script:adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName - - Describe 'Remove-AdfObject' { + Describe 'Remove-AdfObject' -Tag 'Unit' { It 'Should exist' { { Get-Command -Name 'Remove-AdfObject' -ErrorAction Stop } | Should -Not -Throw } + } + Describe 'Remove-AdfObject' -Tag 'Integration' { + BeforeAll { + $script:adf = Import-AdfFromFolder -FactoryName $script:DataFactoryName -RootFolder $script:RootFolder + $adf.ResourceGroupName = $script:ResourceGroupName + $adf.Region = $script:Location + $o = New-AdfPublishOption + $o.StopStartTriggers = $false + $adf.PublishOptions = $o + Publish-AdfV2FromJson -RootFolder $RootFolder -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName -Option $o -Location $Location + $script:adfIns = Get-AdfFromService -FactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName + } Context 'When called for dep object but it is excluded' { It 'Should not delete any object' { diff --git a/test/Remove-AdfObjectIfNotInSource.Tests.ps1 b/test/Remove-AdfObjectIfNotInSource.Tests.ps1 new file mode 100644 index 0000000..46b27e0 --- /dev/null +++ b/test/Remove-AdfObjectIfNotInSource.Tests.ps1 @@ -0,0 +1,187 @@ +BeforeDiscovery { + $ModuleRootPath = $PSScriptRoot | Split-Path -Parent + $moduleManifestName = 'azure.datafactory.tools.psd1' + $moduleManifestPath = Join-Path -Path $ModuleRootPath -ChildPath $moduleManifestName + Import-Module -Name $moduleManifestPath -Force -Verbose:$false +} + +InModuleScope azure.datafactory.tools { + + # ------------------------------------------------------------------------- + # Issue #440 - DeleteNotInSource with -FilterFilePath + # #512: objects skipped as excluded were reported as deleted + # #513: @folder rules never matched objects that exist only in ADF + # ------------------------------------------------------------------------- + Describe 'Remove-AdfObjectIfNotInSource (Issue #440)' -Tag 'Unit' { + + BeforeAll { + function New-TestSource { + param ([string[]] $Includes = @(), [string[]] $Excludes = @()) + $adf = New-Object -TypeName Adf + $adf.Name = 'adf-test' + $adf.ResourceGroupName = 'rg-test' + $opt = New-AdfPublishOption + $opt.DeleteNotInSource = $true + $Includes | ForEach-Object { $opt.Includes.Add($_, '') } + $Excludes | ForEach-Object { $opt.Excludes.Add($_, '') } + $adf.PublishOptions = $opt + return $adf + } + + function New-TestTarget { + $ins = New-Object -TypeName AdfInstance + $ins.Id = '/subscriptions/sub-id/resourceGroups/rg-test/providers/Microsoft.DataFactory/factories/adf-test' + $ins.Name = 'adf-test' + $ins.ResourceGroupName = 'rg-test' + return $ins + } + + # Folders of the objects in the target ADF, as returned by the REST API + $script:ServiceFolders = @{ + 'pipelines/PL_Sales_Old' = 'Sales/Daily' + 'pipelines/PL_HR_Old' = 'HR' + 'datasets/DS_Sales_Old' = 'Sales' + } + } + + BeforeEach { + $ErrorActionPreference = 'Continue' + Mock Write-Host { } + Mock Remove-AzDataFactoryV2Pipeline { } + Mock Remove-AzDataFactoryV2Dataset { } + Mock Remove-AzDataFactoryV2Trigger { } + Mock Stop-AzDataFactoryV2Trigger { } + Mock Invoke-AzRestMethod { + $key = ($Uri -split '/factories/adf-test/')[1].Split('?')[0] + $folder = $script:ServiceFolders[$key] + $props = if ($folder) { @{ folder = @{ name = $folder } } } else { @{} } + [PSCustomObject]@{ + StatusCode = 200 + Content = (@{ name = $key.Split('/')[1]; properties = $props } | ConvertTo-Json -Depth 5) + } + } + } + + Context 'When the object is excluded and DoNotDeleteExcludedObjects = true (#512)' { + + It 'Should not delete the object' { + $adf = New-TestSource -Includes 'pipeline.PL_Keep*' + $obj = [AdfPSPipeline]::new('PL_Other') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 0 -Exactly + } + + It 'Should not count the object as deleted' { + $adf = New-TestSource -Includes 'pipeline.PL_Keep*' + $obj = [AdfPSPipeline]::new('PL_Other') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + $adf.DeletedObjectNames.Count | Should -Be 0 + } + + It 'Should log the skipped object with Write-Host' { + $adf = New-TestSource -Includes 'pipeline.PL_Keep*' + $obj = [AdfPSPipeline]::new('PL_Other') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Write-Host -Times 1 -Exactly -ParameterFilter { "$Object" -like '*PL_Other*DoNotDeleteExcludedObjects*' } + } + } + + Context 'When the object is not excluded (#512)' { + + It 'Should delete the pipeline and count it as deleted' { + $adf = New-TestSource + $obj = [AdfPSPipeline]::new('PL_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 1 -Exactly -ParameterFilter { + $Name -eq 'PL_Old' -and $ResourceGroupName -eq 'rg-test' -and $DataFactoryName -eq 'adf-test' + } + $adf.DeletedObjectNames | Should -Be @('Pipeline.PL_Old') + } + + It 'Should stop and delete a started trigger listed in the filter file' { + $adf = New-TestSource -Includes 'trigger.TR_Removed' + $obj = [AdfPSTrigger]::new('TR_Removed', 'Started') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Stop-AzDataFactoryV2Trigger -Times 1 -Exactly -ParameterFilter { $Name -eq 'TR_Removed' } + Should -Invoke Remove-AzDataFactoryV2Trigger -Times 1 -Exactly -ParameterFilter { $Name -eq 'TR_Removed' } + $adf.DeletedObjectNames | Should -Be @('Trigger.TR_Removed') + } + + It 'Should not count the object as deleted when the removal fails' { + Mock Remove-AzDataFactoryV2Pipeline { Write-Error 'Simulated: AuthorizationFailed' } + $adf = New-TestSource + $obj = [AdfPSPipeline]::new('PL_Old') + { Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) } | Should -Throw '*AuthorizationFailed*' + $adf.DeletedObjectNames.Count | Should -Be 0 + } + } + + Context 'When filter rules use @folder (#513)' { + + It 'Should delete a pipeline in a folder matched by an include rule' { + $adf = New-TestSource -Includes 'pipeline.*@Sales*' + $obj = [AdfPSPipeline]::new('PL_Sales_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 1 -Exactly -ParameterFilter { $Name -eq 'PL_Sales_Old' } + $adf.DeletedObjectNames | Should -Be @('Pipeline.PL_Sales_Old') + } + + It 'Should delete a dataset in a folder matched by an include rule' { + $adf = New-TestSource -Includes 'dataset.*@Sales' + $obj = [AdfPSDataset]::new('DS_Sales_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Dataset -Times 1 -Exactly -ParameterFilter { $Name -eq 'DS_Sales_Old' } + } + + It 'Should not delete a pipeline in a folder not matched by any include rule' { + $adf = New-TestSource -Includes 'pipeline.*@Sales*' + $obj = [AdfPSPipeline]::new('PL_HR_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 0 -Exactly + $adf.DeletedObjectNames.Count | Should -Be 0 + } + + It 'Should not delete a pipeline in a folder matched by an exclude rule' { + $adf = New-TestSource -Excludes 'pipeline.*@HR' + $obj = [AdfPSPipeline]::new('PL_HR_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 0 -Exactly + $adf.DeletedObjectNames.Count | Should -Be 0 + } + + It 'Should still match a rule without folder for a pipeline in a folder' { + $adf = New-TestSource -Includes 'pipeline.PL_Sales_Old', 'pipeline.*@Other' + $obj = [AdfPSPipeline]::new('PL_Sales_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 1 -Exactly -ParameterFilter { $Name -eq 'PL_Sales_Old' } + } + + It 'Should not call the REST API when no rule uses @folder' { + $adf = New-TestSource -Includes 'pipeline.*' + $obj = [AdfPSPipeline]::new('PL_Sales_Old') + Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) + Should -Invoke Invoke-AzRestMethod -Times 0 -Exactly + } + + It 'Should fail when the folder cannot be read' { + Mock Invoke-AzRestMethod { [PSCustomObject]@{ StatusCode = 403; Content = '{}' } } + $adf = New-TestSource -Excludes 'pipeline.*@HR' + $obj = [AdfPSPipeline]::new('PL_HR_Old') + { Remove-AdfObjectIfNotInSource -adfSource $adf -adfTargetObj $obj -adfInstance (New-TestTarget) } | Should -Throw '*403*' + Should -Invoke Remove-AzDataFactoryV2Pipeline -Times 0 -Exactly + } + } + + Context 'DryRun plan with @folder rules (#513)' { + + It 'Should plan to delete only pipelines in folders matched by an include rule' { + $adf = New-TestSource -Includes 'pipeline.*@Sales*' + $ins = New-TestTarget + $ins.Pipelines.Add([AdfPSPipeline]::new('PL_Sales_Old')) | Out-Null + $ins.Pipelines.Add([AdfPSPipeline]::new('PL_HR_Old')) | Out-Null + $plan = Get-DryRunPlan -adf $adf -targetAdfInstance $ins + $plan.Delete | Should -Be @('pipeline.PL_Sales_Old') + } + } + } +} diff --git a/test/Stop-Triggers.Tests.ps1 b/test/Stop-Triggers.Tests.ps1 index 3342a50..b73afcb 100644 --- a/test/Stop-Triggers.Tests.ps1 +++ b/test/Stop-Triggers.Tests.ps1 @@ -13,8 +13,7 @@ InModuleScope azure.datafactory.tools { # Variables for use in tests $script:ResourceGroupName = 'rg-devops-factory' $script:Stage = 'UAT' - $c = Get-AzContext - $script:guid = $c.Subscription.Id.Substring(0,8) + $script:guid = Get-TestSubscriptionPrefix $script:SrcFolder = "$PSScriptRoot\BigFactorySample2" $script:DataFactoryName = (Split-Path -Path $script:SrcFolder -Leaf) + "-$guid" $script:Location = "NorthEurope" diff --git a/test/Test-AdfLinkedService.Tests.ps1 b/test/Test-AdfLinkedService.Tests.ps1 index 9e7c9a6..da83c5b 100644 --- a/test/Test-AdfLinkedService.Tests.ps1 +++ b/test/Test-AdfLinkedService.Tests.ps1 @@ -11,12 +11,11 @@ InModuleScope azure.datafactory.tools { Import-Module -Name $testHelperPath -Force # Variables for use in tests - $azContext = Get-AzContext $script:params = @{ DataFactoryName = 'adf-example-uat' ResourceGroupName = 'rg-example-uat' #SubscriptionID = "{Your-subscriptionId-here}" - SubscriptionID = $azContext.Subscription.Id + SubscriptionID = "00000000-0000-0000-0000-000000000000" } $script:success = @{ diff --git a/test/TestHelper/TestHelper.psm1 b/test/TestHelper/TestHelper.psm1 index afafb6f..993d97a 100644 --- a/test/TestHelper/TestHelper.psm1 +++ b/test/TestHelper/TestHelper.psm1 @@ -221,13 +221,18 @@ function Get-TargetEnv { Location = "UK South" SrcFolder = "$rootPath\$AdfOrigName" } + $target.DataFactoryName = $AdfOrigName + "-" + (Get-TestSubscriptionPrefix) + return $target +} + +function Get-TestSubscriptionPrefix { + # Test files call it at discovery, also for unit tests: without Azure context return a dummy value. + # Integration tests then fail on their first Azure call (run Connect-AzAccount first). $c = Get-AzContext - if ($null -eq $c) { - throw "No active Azure context found. Run Connect-AzAccount before executing integration tests." + if ($null -eq $c -or $null -eq $c.Subscription) { + return '00000000' } - $guid = $c.Subscription.Id.Substring(0,8) - $target.DataFactoryName = $AdfOrigName + "-$guid" - return $target + return $c.Subscription.Id.Substring(0,8) } function IsPesterDebugMode { @@ -253,4 +258,4 @@ Export-ModuleMember -Function ` Get-AdfObjectFromFile, ` Remove-ObjectPropertyFromFile, Edit-TextInFile, Edit-ObjectPropertyInFile, ` Backup-File, Restore-File, ` - Get-RootPath, Get-TargetEnv, IsPesterDebugMode, CreateTargetAdf + Get-RootPath, Get-TargetEnv, Get-TestSubscriptionPrefix, IsPesterDebugMode, CreateTargetAdf diff --git a/test/Triggers.Tests.ps1 b/test/Triggers.Tests.ps1 index b98acbb..d454f6d 100644 --- a/test/Triggers.Tests.ps1 +++ b/test/Triggers.Tests.ps1 @@ -13,8 +13,7 @@ InModuleScope azure.datafactory.tools { # Variables for use in tests $script:ResourceGroupName = 'rg-devops-factory' $script:Stage = 'UAT' - $c = Get-AzContext - $script:guid = $c.Subscription.Id.Substring(0,8) + $script:guid = Get-TestSubscriptionPrefix $script:DataFactoryOrigName = 'BigFactorySample2' $script:DataFactoryName = $script:DataFactoryOrigName + "-$guid" $script:SrcFolder = "$PSScriptRoot\$($script:DataFactoryOrigName)" diff --git a/test/adfObjectName-class.Tests.ps1 b/test/adfObjectName-class.Tests.ps1 index 5828929..2275fc3 100644 --- a/test/adfObjectName-class.Tests.ps1 +++ b/test/adfObjectName-class.Tests.ps1 @@ -74,7 +74,7 @@ InModuleScope azure.datafactory.tools { @{ pattern = '*.*@fold'; expected = $false}, @{ pattern = 'tri*.*@*'; expected = $false}, @{ pattern = '*.*'; expected = $true}, - @{ pattern = 'pipeline.PL_Wait_Dynamic'; expected = $false}, + @{ pattern = 'pipeline.PL_Wait_Dynamic'; expected = $true}, # rule without folder matches any folder (#513) @{ pattern = 'pipeline.PL_Wait_Dynamic@*'; expected = $true}, @{ pattern = 'pipeline.PL_Wait_Dynamic@ExternalError'; expected = $true}, @{ pattern = 'pipeline.PL_*@ExternalError'; expected = $true}, diff --git a/test/zGet-AdfFromService.Tests.ps1 b/test/zGet-AdfFromService.Tests.ps1 index d6f3fe7..b2d0917 100644 --- a/test/zGet-AdfFromService.Tests.ps1 +++ b/test/zGet-AdfFromService.Tests.ps1 @@ -348,10 +348,12 @@ InModuleScope azure.datafactory.tools { # --------------------------------------------------------------------------- Describe 'Get-AdfFromService' -Tag 'Integration' { - # Variables for use in integration tests - $t = Get-TargetEnv 'adf2' - $script:adfName = $t.DataFactoryName - $script:rg = $t.ResourceGroupName + BeforeAll { + # Variables for use in integration tests + $t = Get-TargetEnv 'adf2' + $script:adfName = $t.DataFactoryName + $script:rg = $t.ResourceGroupName + } It 'Should execute' { Get-AdfFromService -FactoryName $adfName -ResourceGroupName $rg diff --git a/tools/Install-DevPrerequisites.ps1 b/tools/Install-DevPrerequisites.ps1 new file mode 100644 index 0000000..3ecd316 --- /dev/null +++ b/tools/Install-DevPrerequisites.ps1 @@ -0,0 +1,82 @@ +<# +.SYNOPSIS +Installs the PowerShell modules needed to develop and test azure.datafactory.tools locally. + +.DESCRIPTION +Installs (for the current user, from PSGallery) every module that is missing or older than the minimum version: +- Az.Accounts, Az.Resources, Az.DataFactory: required by the module (RequiredModules in the manifest) +- Az.Storage: incremental deployment (deployment state in a storage account) +- Pester 6+: test framework (see CLAUDE.md) +- PSScriptAnalyzer: used by some tests +Also checks that Node.js and npm are available (Export-AdfToArmTemplate and its tests need them). +Modules already installed in a sufficient version are skipped, so the script can be run any time. + +.PARAMETER Update +Installs the latest version of each module, even when a sufficient version is already installed. + +.EXAMPLE +./tools/Install-DevPrerequisites.ps1 + +.EXAMPLE +./tools/Install-DevPrerequisites.ps1 -Update +#> +[CmdletBinding()] +param ( + [switch]$Update +) + +$ErrorActionPreference = 'Stop' + +$modules = [ordered]@{ + 'Az.Accounts' = '2.0.0' + 'Az.Resources' = '6.5.0' + 'Az.DataFactory' = '1.16.0' + 'Az.Storage' = '5.0.0' + 'Pester' = '6.0.0' + 'PSScriptAnalyzer' = '1.20.0' +} + +Write-Host "PowerShell $($PSVersionTable.PSVersion) ($($PSVersionTable.PSEdition))" + +# Windows PowerShell 5.1 needs TLS 1.2 and the NuGet provider to install from PSGallery +if ($PSVersionTable.PSEdition -eq 'Desktop') { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + if (-not (Get-PackageProvider -ListAvailable | Where-Object Name -eq 'NuGet')) { + Write-Host "Installing NuGet package provider..." + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Scope CurrentUser -Force | Out-Null + } +} + +foreach ($name in $modules.Keys) { + $minVersion = [version] $modules[$name] + $installed = Get-Module -Name $name -ListAvailable | Sort-Object Version -Descending | Select-Object -First 1 + if ($installed -and $installed.Version -ge $minVersion -and -not $Update) { + Write-Host "OK $name $($installed.Version)" + continue + } + Write-Host "Installing $name (minimum $minVersion)..." + $params = @{ + Name = $name + MinimumVersion = $minVersion + Scope = 'CurrentUser' + Repository = 'PSGallery' + AllowClobber = $true + Force = $true + } + # Pester 6 is signed by a different publisher than the built-in Pester 3.4.0 + if ($name -eq 'Pester') { $params.SkipPublisherCheck = $true } + Install-Module @params + $installed = Get-Module -Name $name -ListAvailable | Sort-Object Version -Descending | Select-Object -First 1 + Write-Host "Installed $name $($installed.Version)" +} + +$npm = if ($IsWindows -or $PSVersionTable.PSEdition -eq 'Desktop') { 'npm.cmd' } else { 'npm' } +if ((Get-Command 'node' -ErrorAction SilentlyContinue) -and (Get-Command $npm -ErrorAction SilentlyContinue)) { + Write-Host "OK Node.js $(& node --version), npm $(& $npm --version)" +} else { + Write-Warning "Node.js/npm not found. Export-AdfToArmTemplate and its tests need them: https://nodejs.org/" +} + +Write-Host "" +Write-Host "Done. Run tests in a clean process, e.g.:" +Write-Host " pwsh -NoProfile -Command `"Import-Module Pester -MinimumVersion 6.0; Import-Module ./azure.datafactory.tools.psd1; Invoke-Pester -Path ./test/Stop-Trigger.Tests.ps1 -Output Detailed`""