From 0a4cff5ba7f8dfcc441bdda3715684d89d721fe1 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:39:43 +0200 Subject: [PATCH 01/23] Use per-test-file repo names in Environments, Secrets, Variables, Releases, Actions, and TEMPLATE tests --- tests/Actions.Tests.ps1 | 2 +- tests/Environments.Tests.ps1 | 2 +- tests/Releases.Tests.ps1 | 2 +- tests/Secrets.Tests.ps1 | 2 +- tests/TEMPLATE.ps1 | 4 ++-- tests/Variables.Tests.ps1 | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/Actions.Tests.ps1 b/tests/Actions.Tests.ps1 index 3bd3223db..748e0fca4 100644 --- a/tests/Actions.Tests.ps1 +++ b/tests/Actions.Tests.ps1 @@ -54,7 +54,7 @@ Describe 'Actions' { Write-Host ($context | Format-List | Out-String) } } - $repoPrefix = "Test-$os-$TokenType" + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" LogGroup "Using Repository - [$repoName]" { diff --git a/tests/Environments.Tests.ps1 b/tests/Environments.Tests.ps1 index 8735581ec..8aa9d2ca6 100644 --- a/tests/Environments.Tests.ps1 +++ b/tests/Environments.Tests.ps1 @@ -43,7 +43,7 @@ Describe 'Environments' { Write-Host ($context | Format-List | Out-String) } } - $repoPrefix = "Test-$os-$TokenType" + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" $environmentName = "$testName-$os-$TokenType-$id" diff --git a/tests/Releases.Tests.ps1 b/tests/Releases.Tests.ps1 index 38cf8cbd1..8a8d75931 100644 --- a/tests/Releases.Tests.ps1 +++ b/tests/Releases.Tests.ps1 @@ -43,7 +43,7 @@ Describe 'Releases' { Write-Host ($context | Format-Table | Out-String) } } - $repoPrefix = "Test-$os-$TokenType" + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" LogGroup "Using Repository - [$repoName]" { diff --git a/tests/Secrets.Tests.ps1 b/tests/Secrets.Tests.ps1 index fee918fd2..ba479730e 100644 --- a/tests/Secrets.Tests.ps1 +++ b/tests/Secrets.Tests.ps1 @@ -43,7 +43,7 @@ Describe 'Secrets' { Write-Host ($context | Format-List | Out-String) } } - $repoPrefix = "Test-$os-$TokenType" + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" $secretPrefix = "$testName`_$os`_$TokenType" $secretName = "$secretPrefix`_$id" diff --git a/tests/TEMPLATE.ps1 b/tests/TEMPLATE.ps1 index afc9a6114..6f52290c1 100644 --- a/tests/TEMPLATE.ps1 +++ b/tests/TEMPLATE.ps1 @@ -41,8 +41,8 @@ Describe 'Template' { } } - # Ensure the shared test repository exists. Set-GitHubRepository is idempotent. - $repoPrefix = "Test-$os-$TokenType" + # Ensure this test file's repository exists. Set-GitHubRepository is idempotent. + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" if ($OwnerType -in ('repository', 'enterprise')) { $repo = $null diff --git a/tests/Variables.Tests.ps1 b/tests/Variables.Tests.ps1 index 904320d51..350529c82 100644 --- a/tests/Variables.Tests.ps1 +++ b/tests/Variables.Tests.ps1 @@ -43,7 +43,7 @@ Describe 'Variables' { Write-Host ($context | Format-List | Out-String) } } - $repoPrefix = "Test-$os-$TokenType" + $repoPrefix = "$testName-$os-$TokenType" $repoName = "$repoPrefix-$id" $variablePrefix = "$testName`_$os`_$TokenType" $variableName = "$variablePrefix`_$id" From c84380cc6d8c78717119e870537c476353415dd9 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:40:34 +0200 Subject: [PATCH 02/23] Provision and tear down per-test-file repositories in global setup and teardown --- tests/AfterAll.ps1 | 41 +++++++++++++---------- tests/BeforeAll.ps1 | 80 +++++++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 53 deletions(-) diff --git a/tests/AfterAll.ps1 b/tests/AfterAll.ps1 index 7bd8e6d24..d15938f19 100644 --- a/tests/AfterAll.ps1 +++ b/tests/AfterAll.ps1 @@ -11,7 +11,6 @@ LogGroup 'AfterAll - Global Test Teardown' { if (-not $env:Settings) { throw 'Settings environment variable is not set. Process-PSModule must populate it with the test suite configuration.' } - $prefix = 'Test' # Derive the list of OS names from the Settings JSON provided by Process-PSModule. try { @@ -30,6 +29,10 @@ LogGroup 'AfterAll - Global Test Teardown' { } Write-Host "Cleaning up test repositories for OSes: $($osNames -join ', ')" + # Test files that own their per-test-file repositories. Mirror BeforeAll.ps1. + $testNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') + $testNamesWithExtraRepos = @('Secrets', 'Variables') + foreach ($authCase in $authCases) { $authCase.GetEnumerator() | ForEach-Object { Set-Variable -Name $_.Key -Value $_.Value } @@ -46,25 +49,27 @@ LogGroup 'AfterAll - Global Test Teardown' { Write-Host ($context | Format-List | Out-String) foreach ($os in $osNames) { - $repoPrefix = "$prefix-$os-$TokenType" - $repoName = "$repoPrefix-$id" + foreach ($testName in $testNames) { + $repoPrefix = "$testName-$os-$TokenType" + $repoName = "$repoPrefix-$id" - LogGroup "Repository cleanup - $AuthType-$TokenType - $os" { - # Use deterministic name lookups instead of listing all repos to reduce API calls. - $cleanupRepoNames = @($repoName) - if ($OwnerType -eq 'organization') { - $cleanupRepoNames += "$repoName-2", "$repoName-3" - } + LogGroup "Repository cleanup - $AuthType-$TokenType - $os - $testName" { + # Use deterministic name lookups instead of listing all repos to reduce API calls. + $cleanupRepoNames = @($repoName) + if ($OwnerType -eq 'organization' -and $testName -in $testNamesWithExtraRepos) { + $cleanupRepoNames += "$repoName-2", "$repoName-3" + } - foreach ($cleanupRepoName in $cleanupRepoNames) { - switch ($OwnerType) { - 'user' { - Get-GitHubRepository -Name $cleanupRepoName -ErrorAction SilentlyContinue | - Remove-GitHubRepository -Confirm:$false - } - 'organization' { - Get-GitHubRepository -Owner $Owner -Name $cleanupRepoName -ErrorAction SilentlyContinue | - Remove-GitHubRepository -Confirm:$false + foreach ($cleanupRepoName in $cleanupRepoNames) { + switch ($OwnerType) { + 'user' { + Get-GitHubRepository -Name $cleanupRepoName -ErrorAction SilentlyContinue | + Remove-GitHubRepository -Confirm:$false + } + 'organization' { + Get-GitHubRepository -Owner $Owner -Name $cleanupRepoName -ErrorAction SilentlyContinue | + Remove-GitHubRepository -Confirm:$false + } } } } diff --git a/tests/BeforeAll.ps1 b/tests/BeforeAll.ps1 index d1f247999..ef201e78a 100644 --- a/tests/BeforeAll.ps1 +++ b/tests/BeforeAll.ps1 @@ -28,6 +28,14 @@ LogGroup 'BeforeAll - Global Test Setup' { } Write-Host "Creating test repositories for OSes: $($osNames -join ', ')" + # Test files that require their own per-test-file repository. + # Each test file's per-context BeforeAll also calls Set-GitHubRepository as a safety net, + # so this list is an optimization rather than a hard dependency. + $testNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') + + # Test files that need companion repositories (-2, -3) for org-scoped SelectedRepository tests. + $testNamesWithExtraRepos = @('Secrets', 'Variables') + foreach ($authCase in $authCases) { $authCase.GetEnumerator() | ForEach-Object { Set-Variable -Name $_.Key -Value $_.Value } @@ -43,47 +51,49 @@ LogGroup 'BeforeAll - Global Test Setup' { Write-Host ($context | Format-List | Out-String) foreach ($os in $osNames) { - $repoPrefix = "Test-$os-$TokenType" - $repoName = "$repoPrefix-$id" + foreach ($testName in $testNames) { + $repoPrefix = "$testName-$os-$TokenType" + $repoName = "$repoPrefix-$id" - LogGroup "Repository setup - $AuthType-$TokenType - $os" { - # Clean up repos from a previous attempt of the same run (re-runs). - # Use deterministic name lookups instead of listing all repos to reduce API calls. - $cleanupRepoNames = @($repoName) - if ($OwnerType -eq 'organization') { - $cleanupRepoNames += "$repoName-2", "$repoName-3" - } + LogGroup "Repository setup - $AuthType-$TokenType - $os - $testName" { + # Clean up repos from a previous attempt of the same run (re-runs). + # Use deterministic name lookups instead of listing all repos to reduce API calls. + $cleanupRepoNames = @($repoName) + if ($OwnerType -eq 'organization' -and $testName -in $testNamesWithExtraRepos) { + $cleanupRepoNames += "$repoName-2", "$repoName-3" + } - foreach ($cleanupRepoName in $cleanupRepoNames) { - switch ($OwnerType) { - 'user' { - Get-GitHubRepository -Name $cleanupRepoName -ErrorAction SilentlyContinue | - Remove-GitHubRepository -Confirm:$false - } - 'organization' { - Get-GitHubRepository -Owner $Owner -Name $cleanupRepoName -ErrorAction SilentlyContinue | - Remove-GitHubRepository -Confirm:$false + foreach ($cleanupRepoName in $cleanupRepoNames) { + switch ($OwnerType) { + 'user' { + Get-GitHubRepository -Name $cleanupRepoName -ErrorAction SilentlyContinue | + Remove-GitHubRepository -Confirm:$false + } + 'organization' { + Get-GitHubRepository -Owner $Owner -Name $cleanupRepoName -ErrorAction SilentlyContinue | + Remove-GitHubRepository -Confirm:$false + } } } - } - # Provision the primary shared repository. - $repoParams = @{ - Name = $repoName - AddReadme = $true - License = 'mit' - Gitignore = 'VisualStudio' - } - switch ($OwnerType) { - 'user' { Set-GitHubRepository @repoParams } - 'organization' { Set-GitHubRepository @repoParams -Organization $Owner } - } + # Provision the primary per-test-file repository. + $repoParams = @{ + Name = $repoName + AddReadme = $true + License = 'mit' + Gitignore = 'VisualStudio' + } + switch ($OwnerType) { + 'user' { Set-GitHubRepository @repoParams } + 'organization' { Set-GitHubRepository @repoParams -Organization $Owner } + } - # Provision extra repositories needed by Secrets/Variables SelectedRepository tests. - # Only organization owners need them — those tests are skipped for user owners. - if ($OwnerType -eq 'organization') { - foreach ($suffix in 2, 3) { - Set-GitHubRepository -Organization $Owner -Name "$repoName-$suffix" + # Provision extra repositories needed by Secrets/Variables SelectedRepository tests. + # Only organization owners need them — those tests are skipped for user owners. + if ($OwnerType -eq 'organization' -and $testName -in $testNamesWithExtraRepos) { + foreach ($suffix in 2, 3) { + Set-GitHubRepository -Organization $Owner -Name "$repoName-$suffix" + } } } } From fe3248a20aec7160a59406c7b0170178bd475ae8 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:41:12 +0200 Subject: [PATCH 03/23] Clean up stale releases in Releases per-context BeforeAll to support partial reruns --- tests/Releases.Tests.ps1 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/Releases.Tests.ps1 b/tests/Releases.Tests.ps1 index 8a8d75931..fad24e4b3 100644 --- a/tests/Releases.Tests.ps1 +++ b/tests/Releases.Tests.ps1 @@ -63,6 +63,22 @@ Describe 'Releases' { } Write-Host ($repo | Select-Object * | Out-String) } + + # Clean up stale releases from prior runs with the same GITHUB_RUN_ID. + # Idempotent setup must not assume a clean repository — partial reruns can leave + # tags like v1.0/v1.1/v1.3 behind, which would cause New-GitHubRelease to fail + # with 422 (already_exists). + if ($repo) { + LogGroup "Pre-test Cleanup - Existing Releases on [$repoName]" { + $existingReleases = Get-GitHubRelease -Owner $Owner -Repository $repoName -AllVersions -ErrorAction SilentlyContinue + if ($existingReleases) { + Write-Host ($existingReleases | Format-Table | Out-String) + $existingReleases | Remove-GitHubRelease -Confirm:$false + } else { + Write-Host 'No existing releases to clean up.' + } + } + } } AfterAll { From 9da0e0aac9d32cf6f85026511bb075a9d388543d Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:41:31 +0200 Subject: [PATCH 04/23] Remove the test environment in Secrets and Variables AfterAll to prevent leakage across test files --- tests/Secrets.Tests.ps1 | 8 ++++++++ tests/Variables.Tests.ps1 | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/Secrets.Tests.ps1 b/tests/Secrets.Tests.ps1 index ba479730e..92fc9047c 100644 --- a/tests/Secrets.Tests.ps1 +++ b/tests/Secrets.Tests.ps1 @@ -98,6 +98,14 @@ Describe 'Secrets' { } } } + # Remove the test environment created on the per-test-file repository so it does + # not leak into other test files or subsequent reruns. + if ($OwnerType -notin ('repository', 'enterprise') -and $repo) { + LogGroup "Environment cleanup - [$environmentName] on [$repoName]" { + Get-GitHubEnvironment -Owner $owner -Repository $repoName -Name $environmentName -ErrorAction SilentlyContinue | + Remove-GitHubEnvironment -Confirm:$false + } + } Get-GitHubContext -ListAvailable | Disconnect-GitHubAccount -Silent Write-Host ('-' * 60) } diff --git a/tests/Variables.Tests.ps1 b/tests/Variables.Tests.ps1 index 350529c82..f046fefce 100644 --- a/tests/Variables.Tests.ps1 +++ b/tests/Variables.Tests.ps1 @@ -97,6 +97,14 @@ Describe 'Variables' { $variablesToRemove | Remove-GitHubVariable } } + # Remove the test environment created on the per-test-file repository so it does + # not leak into other test files or subsequent reruns. + if ($OwnerType -notin ('repository', 'enterprise') -and $repo) { + LogGroup "Environment cleanup - [$environmentName] on [$repoName]" { + Get-GitHubEnvironment -Owner $owner -Repository $repoName -Name $environmentName -ErrorAction SilentlyContinue | + Remove-GitHubEnvironment -Confirm:$false + } + } Get-GitHubContext -ListAvailable | Disconnect-GitHubAccount -Silent Write-Host ('-' * 60) } From 8f94994a6b95e4c3678bac5541d3f6bf7bbe4818 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:41:47 +0200 Subject: [PATCH 05/23] Retry Install-GitHubApp on enterprise organization to absorb propagation delay (#596) --- tests/Organizations.Tests.ps1 | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 94b884ab6..4dd71deae 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -143,7 +143,24 @@ Describe 'Organizations' { } It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { - $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName -ClientID $installationContext.ClientID -RepositorySelection 'all' + # The enterprise organization was just created and may not have propagated to the + # enterprise apps endpoint yet. Retry briefly to absorb propagation delay before + # failing — GitHub returns 404 (rather than 403) when the resource is not yet visible + # to the token, which is indistinguishable from a missing-permission failure on the + # first call. See issue #596. + $installation = $null + $maxAttempts = 5 + $delaySeconds = 3 + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName -ClientID $installationContext.ClientID -RepositorySelection 'all' + break + } catch { + if ($attempt -eq $maxAttempts) { throw } + Write-Host "Install-GitHubApp attempt $attempt failed ($($_.Exception.Message)); retrying in $delaySeconds seconds..." + Start-Sleep -Seconds $delaySeconds + } + } LogGroup 'Installed App' { Write-Host ($installation | Select-Object * | Out-String) } From 2b3cbe8683bfef9b138ba90a412b751715d490b6 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 20:42:03 +0200 Subject: [PATCH 06/23] Document enterprise_organization_installations permission requirement for APP_ENT --- .github/instructions/tests.instructions.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/instructions/tests.instructions.md b/.github/instructions/tests.instructions.md index 5b419265a..4459441f8 100644 --- a/.github/instructions/tests.instructions.md +++ b/.github/instructions/tests.instructions.md @@ -25,6 +25,14 @@ Secrets: Homed in `MSX`. ClientID: `Iv23lieHcDQDwVV3alK1`. Installed on [psmodule-test-org3](https://github.com/orgs/psmodule-test-org3) (enterprise org) with all permissions and push events. +Required enterprise-scoped permissions (configured on the app, homed in `msx`): + +- `enterprise_organization_installations: write` — required by `Install-GitHubApp` on enterprise-owned organizations + ([docs](https://docs.github.com/rest/enterprise-admin/organization-installations#install-a-github-app-on-an-enterprise-owned-organization)). + The Organizations test creates an enterprise organization and then installs the app on it; the + endpoint returns 404 (not 403) when this permission is missing, which makes a missing + permission look like a missing resource. See issue #596. + Secrets: `TEST_APP_ENT_CLIENT_ID`, `TEST_APP_ENT_PRIVATE_KEY` ### APP_ORG — PSModule Organization App From d8def0c2b83e10b586c735b0f848f147cca5baf1 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 21:06:02 +0200 Subject: [PATCH 07/23] Move Remove-GitHubOrganization (enterprise, Should -Throw) to after Install-GitHubApp to prevent accidental org deletion --- tests/Organizations.Tests.ps1 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 4dd71deae..8d466e60c 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -138,10 +138,6 @@ Describe 'Organizations' { { Update-GitHubOrganization -Name $orgName -Location 'New Location' } | Should -Throw } - It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { - { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw - } - It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { # The enterprise organization was just created and may not have propagated to the # enterprise apps endpoint yet. Retry briefly to absorb propagation delay before @@ -181,6 +177,16 @@ Describe 'Organizations' { Update-GitHubOrganization -Name $orgName -Location 'New Location' -Context $orgContext } + # This test verifies that the enterprise IAT cannot delete an org — org-level operations + # require an org installation (shown by the tests above). It is intentionally placed AFTER + # Install-GitHubApp and the org-IAT tests so that if the enterprise app unexpectedly gains + # organization_administration permission and this call succeeds instead of throwing, the + # critical install/connect/update tests have already passed and the org deletion is + # a no-op for those assertions. See issue #596. + It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { + { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw + } + It 'Remove-GitHubOrganization - Removes an organization using organization installation' -Skip:($OwnerType -ne 'enterprise') { $orgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $orgContext From 3362dc56a62ba0c78e8cac4b93cd2a5ca87b0ca4 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 21:09:01 +0200 Subject: [PATCH 08/23] Clarify why enterprise IAT cannot delete an org (endpoint requires org-level administration permission) --- tests/Organizations.Tests.ps1 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 8d466e60c..01eeac22f 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -177,12 +177,11 @@ Describe 'Organizations' { Update-GitHubOrganization -Name $orgName -Location 'New Location' -Context $orgContext } - # This test verifies that the enterprise IAT cannot delete an org — org-level operations - # require an org installation (shown by the tests above). It is intentionally placed AFTER - # Install-GitHubApp and the org-IAT tests so that if the enterprise app unexpectedly gains - # organization_administration permission and this call succeeds instead of throwing, the - # critical install/connect/update tests have already passed and the org deletion is - # a no-op for those assertions. See issue #596. + # GitHub's DELETE /orgs/{org} endpoint requires the app to have the org-level + # `administration: write` permission. The enterprise IAT is enterprise-scoped and does not + # carry org-level permissions, so this call is expected to fail regardless of which + # enterprise permissions the app holds. An org-level IAT (obtained after Install-GitHubApp) + # is required. See issue #596. It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw } From f80b19d45ef28c4a7fbe688f1bdb0426d6f6985a Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 21:15:51 +0200 Subject: [PATCH 09/23] Refactor test cleanup in AfterAll.ps1 and streamline organization removal tests in Organizations.Tests.ps1 Co-authored-by: Copilot --- tests/AfterAll.ps1 | 2 -- tests/Organizations.Tests.ps1 | 32 +++++--------------------------- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/tests/AfterAll.ps1 b/tests/AfterAll.ps1 index d15938f19..06bb715fc 100644 --- a/tests/AfterAll.ps1 +++ b/tests/AfterAll.ps1 @@ -29,10 +29,8 @@ LogGroup 'AfterAll - Global Test Teardown' { } Write-Host "Cleaning up test repositories for OSes: $($osNames -join ', ')" - # Test files that own their per-test-file repositories. Mirror BeforeAll.ps1. $testNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') $testNamesWithExtraRepos = @('Secrets', 'Variables') - foreach ($authCase in $authCases) { $authCase.GetEnumerator() | ForEach-Object { Set-Variable -Name $_.Key -Value $_.Value } diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 01eeac22f..94b884ab6 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -138,25 +138,12 @@ Describe 'Organizations' { { Update-GitHubOrganization -Name $orgName -Location 'New Location' } | Should -Throw } + It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { + { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw + } + It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { - # The enterprise organization was just created and may not have propagated to the - # enterprise apps endpoint yet. Retry briefly to absorb propagation delay before - # failing — GitHub returns 404 (rather than 403) when the resource is not yet visible - # to the token, which is indistinguishable from a missing-permission failure on the - # first call. See issue #596. - $installation = $null - $maxAttempts = 5 - $delaySeconds = 3 - for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { - try { - $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName -ClientID $installationContext.ClientID -RepositorySelection 'all' - break - } catch { - if ($attempt -eq $maxAttempts) { throw } - Write-Host "Install-GitHubApp attempt $attempt failed ($($_.Exception.Message)); retrying in $delaySeconds seconds..." - Start-Sleep -Seconds $delaySeconds - } - } + $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName -ClientID $installationContext.ClientID -RepositorySelection 'all' LogGroup 'Installed App' { Write-Host ($installation | Select-Object * | Out-String) } @@ -177,15 +164,6 @@ Describe 'Organizations' { Update-GitHubOrganization -Name $orgName -Location 'New Location' -Context $orgContext } - # GitHub's DELETE /orgs/{org} endpoint requires the app to have the org-level - # `administration: write` permission. The enterprise IAT is enterprise-scoped and does not - # carry org-level permissions, so this call is expected to fail regardless of which - # enterprise permissions the app holds. An org-level IAT (obtained after Install-GitHubApp) - # is required. See issue #596. - It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { - { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw - } - It 'Remove-GitHubOrganization - Removes an organization using organization installation' -Skip:($OwnerType -ne 'enterprise') { $orgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $orgContext From e1d9609fbd31beda6e7c11eef41e0e5e9074e873 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 21:26:12 +0200 Subject: [PATCH 10/23] Clean up stale enterprise org in BeforeAll and add assertions to New-GitHubOrganization test --- tests/Organizations.Tests.ps1 | 38 +++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 94b884ab6..2b9bf1b44 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -41,15 +41,38 @@ Describe 'Organizations' { $orgName = "$orgPrefix$id" if ($AuthType -eq 'APP') { - LogGroup 'Pre-test Cleanup - App Installations' { - Get-GitHubAppInstallation -Context $context | Where-Object { $_.Target.Name -like "$orgName*" } | - Uninstall-GitHubApp -Confirm:$false - } - $installationContext = Connect-GitHubApp @connectAppParams -PassThru -Default -Silent LogGroup 'Context - Installation' { Write-Host ($installationContext | Select-Object * | Out-String) } + + if ($OwnerType -eq 'enterprise') { + # Clean up a stale enterprise org from a previous run attempt with the same + # GITHUB_RUN_ID. DELETE /orgs/{org} requires org-level administration:write, + # so we install the app first to obtain an org-level IAT, then delete. + LogGroup 'Pre-test Cleanup - Stale Enterprise Organization' { + $staleOrg = Get-GitHubOrganization -Name $orgName -ErrorAction SilentlyContinue + if ($staleOrg -and $staleOrg.Name) { + Write-Host "Stale org [$orgName] found from previous run attempt. Removing..." + try { + $null = Install-GitHubApp -Enterprise $owner -Organization $orgName ` + -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop + $cleanupOrgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent + Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $cleanupOrgContext + Write-Host "Stale org [$orgName] removed." + } catch { + Write-Host "WARNING: Could not remove stale org [$orgName]: $($_.Exception.Message)" + } + } else { + Write-Host "No stale org found for [$orgName]." + } + } + } + + LogGroup 'Pre-test Cleanup - App Installations' { + Get-GitHubAppInstallation -Context $context | Where-Object { $_.Target.Name -like "$orgName*" } | + Uninstall-GitHubApp -Confirm:$false + } } } @@ -128,10 +151,13 @@ Describe 'Organizations' { Owner = 'MariusStorhaug' BillingEmail = 'post@msx.no' } + $org = New-GitHubOrganization @orgParam LogGroup 'Organization' { - $org = New-GitHubOrganization @orgParam Write-Host ($org | Select-Object * | Out-String) } + $org | Should -Not -BeNullOrEmpty + $org | Should -BeOfType 'GitHubOrganization' + $org.Name | Should -Be $orgName } It 'Update-GitHubOrganization - Updates the organization location using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { From 0d7811e3ef2818edd5c27c2541828c4f13121198 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sun, 3 May 2026 21:38:20 +0200 Subject: [PATCH 11/23] Append GITHUB_RUN_ATTEMPT to org name on reruns to avoid the 90-day org name hold --- tests/Organizations.Tests.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 2b9bf1b44..16ccb35ff 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -26,6 +26,13 @@ BeforeAll { if (-not $id) { throw 'GITHUB_RUN_ID is required to safely scope pre-test cleanup in Organizations.Tests.ps1.' } + # GITHUB_RUN_ATTEMPT increments on each rerun (1, 2, 3...). Enterprise org names go on a + # 90-day hold after deletion, so a rerun of the same GITHUB_RUN_ID would collide if we used + # the run ID alone. Appending the attempt number makes each attempt produce a unique org name. + $attempt = $env:GITHUB_RUN_ATTEMPT + if ($attempt -and $attempt -ne '1') { + $id = "$id-$attempt" + } } Describe 'Organizations' { From a84ae9838091bf460a601904e60a562a1bd559c3 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Mon, 4 May 2026 00:25:50 +0200 Subject: [PATCH 12/23] Extract testNames to shared TestRepos.ps1 data file; rethrow stale org cleanup failure --- tests/AfterAll.ps1 | 7 +++++-- tests/BeforeAll.ps1 | 12 +++++------- tests/Data/TestRepos.ps1 | 11 +++++++++++ tests/Organizations.Tests.ps1 | 4 +++- 4 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 tests/Data/TestRepos.ps1 diff --git a/tests/AfterAll.ps1 b/tests/AfterAll.ps1 index 06bb715fc..7719f6538 100644 --- a/tests/AfterAll.ps1 +++ b/tests/AfterAll.ps1 @@ -29,8 +29,11 @@ LogGroup 'AfterAll - Global Test Teardown' { } Write-Host "Cleaning up test repositories for OSes: $($osNames -join ', ')" - $testNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') - $testNamesWithExtraRepos = @('Secrets', 'Variables') + # Source the single authoritative list of test-file repositories so setup and teardown + # always operate on the same set. See tests/Data/TestRepos.ps1. + $testRepos = . "$PSScriptRoot/Data/TestRepos.ps1" + $testNames = $testRepos.TestNames + $testNamesWithExtraRepos = $testRepos.TestNamesWithExtraRepos foreach ($authCase in $authCases) { $authCase.GetEnumerator() | ForEach-Object { Set-Variable -Name $_.Key -Value $_.Value } diff --git a/tests/BeforeAll.ps1 b/tests/BeforeAll.ps1 index ef201e78a..62aa9edec 100644 --- a/tests/BeforeAll.ps1 +++ b/tests/BeforeAll.ps1 @@ -28,13 +28,11 @@ LogGroup 'BeforeAll - Global Test Setup' { } Write-Host "Creating test repositories for OSes: $($osNames -join ', ')" - # Test files that require their own per-test-file repository. - # Each test file's per-context BeforeAll also calls Set-GitHubRepository as a safety net, - # so this list is an optimization rather than a hard dependency. - $testNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') - - # Test files that need companion repositories (-2, -3) for org-scoped SelectedRepository tests. - $testNamesWithExtraRepos = @('Secrets', 'Variables') + # Source the single authoritative list of test-file repositories so setup and teardown + # always operate on the same set. See tests/Data/TestRepos.ps1. + $testRepos = . "$PSScriptRoot/Data/TestRepos.ps1" + $testNames = $testRepos.TestNames + $testNamesWithExtraRepos = $testRepos.TestNamesWithExtraRepos foreach ($authCase in $authCases) { $authCase.GetEnumerator() | ForEach-Object { Set-Variable -Name $_.Key -Value $_.Value } diff --git a/tests/Data/TestRepos.ps1 b/tests/Data/TestRepos.ps1 new file mode 100644 index 000000000..a2b072da2 --- /dev/null +++ b/tests/Data/TestRepos.ps1 @@ -0,0 +1,11 @@ +# Test files that require their own per-test-file repository. +# Each test file's per-context BeforeAll also calls Set-GitHubRepository as a safety net, +# so this list is an optimization rather than a hard dependency. BeforeAll.ps1 and +# AfterAll.ps1 both source this file so setup and teardown always operate on the same set. +@{ + # Test files that each need a primary repository. + TestNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') + + # Subset that also need companion -2/-3 repositories for org-scoped SelectedRepository tests. + TestNamesWithExtraRepos = @('Secrets', 'Variables') +} diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 16ccb35ff..fc22612f7 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -68,7 +68,9 @@ Describe 'Organizations' { Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $cleanupOrgContext Write-Host "Stale org [$orgName] removed." } catch { - Write-Host "WARNING: Could not remove stale org [$orgName]: $($_.Exception.Message)" + # Rethrow — if the org exists but we can't remove it, New-GitHubOrganization + # will fail anyway. Failing here gives a clearer root-cause message. + throw "Could not remove stale org [$orgName]: $($_.Exception.Message)" } } else { Write-Host "No stale org found for [$orgName]." From bec6182212afa8e0338220a945e4d558b39bb58b Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Mon, 4 May 2026 02:46:19 +0200 Subject: [PATCH 13/23] Address Copilot review: retry Install-GitHubApp (threads #3179013796), add -Confirm:false to org secret/variable cleanup (threads #3179013812, #3179013820), fix MSX/msx inconsistency in test instructions (thread #3179013826) --- .github/instructions/tests.instructions.md | 4 +-- tests/Organizations.Tests.ps1 | 40 ++++++++++++++++++++-- tests/Secrets.Tests.ps1 | 2 +- tests/Variables.Tests.ps1 | 2 +- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/.github/instructions/tests.instructions.md b/.github/instructions/tests.instructions.md index 4459441f8..ccd6e293b 100644 --- a/.github/instructions/tests.instructions.md +++ b/.github/instructions/tests.instructions.md @@ -22,10 +22,10 @@ Secrets: ### APP_ENT — PSModule Enterprise App -Homed in `MSX`. ClientID: `Iv23lieHcDQDwVV3alK1`. +Homed in `MSX` (enterprise slug: `msx`). ClientID: `Iv23lieHcDQDwVV3alK1`. Installed on [psmodule-test-org3](https://github.com/orgs/psmodule-test-org3) (enterprise org) with all permissions and push events. -Required enterprise-scoped permissions (configured on the app, homed in `msx`): +Required enterprise-scoped permissions (configured on the app): - `enterprise_organization_installations: write` — required by `Install-GitHubApp` on enterprise-owned organizations ([docs](https://docs.github.com/rest/enterprise-admin/organization-installations#install-a-github-app-on-an-enterprise-owned-organization)). diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index fc22612f7..7c71d318a 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -62,8 +62,24 @@ Describe 'Organizations' { if ($staleOrg -and $staleOrg.Name) { Write-Host "Stale org [$orgName] found from previous run attempt. Removing..." try { - $null = Install-GitHubApp -Enterprise $owner -Organization $orgName ` - -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop + # Retry Install-GitHubApp: the enterprise apps endpoint can return 404 + # for a short time after the org was originally created. + $maxAttempts = 5 + $retryDelay = 3 + for ($retryAttempt = 1; $retryAttempt -le $maxAttempts; $retryAttempt++) { + try { + $null = Install-GitHubApp -Enterprise $owner -Organization $orgName ` + -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop + break + } catch { + if ($retryAttempt -lt $maxAttempts) { + Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $($_.Exception.Message). Retrying in ${retryDelay}s..." + Start-Sleep -Seconds $retryDelay + } else { + throw + } + } + } $cleanupOrgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $cleanupOrgContext Write-Host "Stale org [$orgName] removed." @@ -178,7 +194,25 @@ Describe 'Organizations' { } It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { - $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName -ClientID $installationContext.ClientID -RepositorySelection 'all' + # Retry: the enterprise apps endpoint can return 404 transiently right after + # New-GitHubOrganization, before the new org has propagated. + $maxAttempts = 5 + $retryDelay = 3 + $installation = $null + for ($retryAttempt = 1; $retryAttempt -le $maxAttempts; $retryAttempt++) { + try { + $installation = Install-GitHubApp -Enterprise $owner -Organization $orgName ` + -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop + break + } catch { + if ($retryAttempt -lt $maxAttempts) { + Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $($_.Exception.Message). Retrying in ${retryDelay}s..." + Start-Sleep -Seconds $retryDelay + } else { + throw + } + } + } LogGroup 'Installed App' { Write-Host ($installation | Select-Object * | Out-String) } diff --git a/tests/Secrets.Tests.ps1 b/tests/Secrets.Tests.ps1 index 92fc9047c..6acd3fff3 100644 --- a/tests/Secrets.Tests.ps1 +++ b/tests/Secrets.Tests.ps1 @@ -94,7 +94,7 @@ Describe 'Secrets' { LogGroup 'Secrets to remove' { $orgSecrets = Get-GitHubSecret -Owner $owner | Where-Object { $_.Name -like "$secretName*" } Write-Host "$($orgSecrets | Format-List | Out-String)" - $orgSecrets | Remove-GitHubSecret + $orgSecrets | Remove-GitHubSecret -Confirm:$false } } } diff --git a/tests/Variables.Tests.ps1 b/tests/Variables.Tests.ps1 index f046fefce..ae95c05cd 100644 --- a/tests/Variables.Tests.ps1 +++ b/tests/Variables.Tests.ps1 @@ -94,7 +94,7 @@ Describe 'Variables' { LogGroup 'Variables to remove' { Write-Host "$($variablesToRemove | Format-List | Out-String)" } - $variablesToRemove | Remove-GitHubVariable + $variablesToRemove | Remove-GitHubVariable -Confirm:$false } } # Remove the test environment created on the per-test-file repository so it does From b22c2aad8cf4af83ec2f0238162bbac9fc45bf97 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 13:01:15 +0200 Subject: [PATCH 14/23] Fix unresolved review threads: proper stale org cleanup and test ordering - Keep base GITHUB_RUN_ID separate from attempt-versioned ID to properly clean up orphaned orgs from all previous attempts (not just current attempt) - Search for and remove any stale orgs matching the base run prefix pattern to prevent resource leakage across reruns - Move 'Remove-GitHubOrganization ... Should -Throw' test to after 'Install-GitHubApp' to prevent accidentally deleting the org before the install step completes --- tests/Organizations.Tests.ps1 | 86 +++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 7c71d318a..bc5d9ec2c 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -22,16 +22,17 @@ param() BeforeAll { $testName = 'Organizations' $os = $env:RUNNER_OS - $id = $env:GITHUB_RUN_ID - if (-not $id) { + $runId = $env:GITHUB_RUN_ID + if (-not $runId) { throw 'GITHUB_RUN_ID is required to safely scope pre-test cleanup in Organizations.Tests.ps1.' } # GITHUB_RUN_ATTEMPT increments on each rerun (1, 2, 3...). Enterprise org names go on a # 90-day hold after deletion, so a rerun of the same GITHUB_RUN_ID would collide if we used # the run ID alone. Appending the attempt number makes each attempt produce a unique org name. $attempt = $env:GITHUB_RUN_ATTEMPT + $id = $runId if ($attempt -and $attempt -ne '1') { - $id = "$id-$attempt" + $id = "$runId-$attempt" } } @@ -58,38 +59,55 @@ Describe 'Organizations' { # GITHUB_RUN_ID. DELETE /orgs/{org} requires org-level administration:write, # so we install the app first to obtain an org-level IAT, then delete. LogGroup 'Pre-test Cleanup - Stale Enterprise Organization' { - $staleOrg = Get-GitHubOrganization -Name $orgName -ErrorAction SilentlyContinue - if ($staleOrg -and $staleOrg.Name) { - Write-Host "Stale org [$orgName] found from previous run attempt. Removing..." - try { - # Retry Install-GitHubApp: the enterprise apps endpoint can return 404 - # for a short time after the org was originally created. - $maxAttempts = 5 - $retryDelay = 3 - for ($retryAttempt = 1; $retryAttempt -le $maxAttempts; $retryAttempt++) { - try { - $null = Install-GitHubApp -Enterprise $owner -Organization $orgName ` - -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop - break - } catch { - if ($retryAttempt -lt $maxAttempts) { - Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $($_.Exception.Message). Retrying in ${retryDelay}s..." - Start-Sleep -Seconds $retryDelay - } else { - throw + # On reruns, clean up any orgs matching the base run prefix (e.g., ...-1234, + # ...-1234-2, etc.) before creating a new one. This prevents orphaned orgs + # from failed previous attempts. + $orgPrefix = "$testName-$os-$runId" + Write-Host "Searching for stale orgs matching prefix: $orgPrefix*" + + # Collect all orgs that match the base run prefix pattern + $staleOrgs = Get-GitHubOrganization -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "$orgPrefix*" -and $_.Name -ne $orgName } + + # Also check for the current org name in case it exists from a failed attempt + $currentOrg = Get-GitHubOrganization -Name $orgName -ErrorAction SilentlyContinue + if ($currentOrg -and $currentOrg.Name) { + $staleOrgs += $currentOrg + } + + if ($staleOrgs.Count -gt 0) { + foreach ($staleOrg in $staleOrgs) { + Write-Host "Stale org [$($staleOrg.Name)] found from previous run. Removing..." + try { + # Retry Install-GitHubApp: the enterprise apps endpoint can return 404 + # for a short time after the org was originally created. + $maxAttempts = 5 + $retryDelay = 3 + for ($retryAttempt = 1; $retryAttempt -le $maxAttempts; $retryAttempt++) { + try { + $null = Install-GitHubApp -Enterprise $owner -Organization $staleOrg.Name ` + -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop + break + } catch { + if ($retryAttempt -lt $maxAttempts) { + Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $($_.Exception.Message). Retrying in ${retryDelay}s..." + Start-Sleep -Seconds $retryDelay + } else { + throw + } } } + $cleanupOrgContext = Connect-GitHubApp -Organization $staleOrg.Name -Context $context -PassThru -Silent + Remove-GitHubOrganization -Name $staleOrg.Name -Confirm:$false -Context $cleanupOrgContext + Write-Host "Stale org [$($staleOrg.Name)] removed." + } catch { + # Rethrow — if the org exists but we can't remove it, New-GitHubOrganization + # will fail anyway. Failing here gives a clearer root-cause message. + throw "Could not remove stale org [$($staleOrg.Name)]: $($_.Exception.Message)" } - $cleanupOrgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent - Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $cleanupOrgContext - Write-Host "Stale org [$orgName] removed." - } catch { - # Rethrow — if the org exists but we can't remove it, New-GitHubOrganization - # will fail anyway. Failing here gives a clearer root-cause message. - throw "Could not remove stale org [$orgName]: $($_.Exception.Message)" } } else { - Write-Host "No stale org found for [$orgName]." + Write-Host "No stale orgs found matching prefix: $orgPrefix*" } } } @@ -189,10 +207,6 @@ Describe 'Organizations' { { Update-GitHubOrganization -Name $orgName -Location 'New Location' } | Should -Throw } - It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { - { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw - } - It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { # Retry: the enterprise apps endpoint can return 404 transiently right after # New-GitHubOrganization, before the new org has propagated. @@ -233,6 +247,10 @@ Describe 'Organizations' { Update-GitHubOrganization -Name $orgName -Location 'New Location' -Context $orgContext } + It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { + { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw + } + It 'Remove-GitHubOrganization - Removes an organization using organization installation' -Skip:($OwnerType -ne 'enterprise') { $orgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $orgContext From 916f0b0198ccc9bfebdc9e4f8a7ddad2112081d1 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 13:12:42 +0200 Subject: [PATCH 15/23] Address review feedback: separate org prefix variables and use -Enterprise for discovery - Use separate variable \ for the run-scoped prefix in cleanup block to avoid mutating \ which is used later in tests - Add -Enterprise \ parameter to Get-GitHubOrganization calls to properly enumerate enterprise-owned orgs for discovery and current org lookup --- tests/Organizations.Tests.ps1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index bc5d9ec2c..9dcf69c9b 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -62,15 +62,15 @@ Describe 'Organizations' { # On reruns, clean up any orgs matching the base run prefix (e.g., ...-1234, # ...-1234-2, etc.) before creating a new one. This prevents orphaned orgs # from failed previous attempts. - $orgPrefix = "$testName-$os-$runId" - Write-Host "Searching for stale orgs matching prefix: $orgPrefix*" + $orgRunPrefix = "$testName-$os-$runId" + Write-Host "Searching for stale orgs matching prefix: $orgRunPrefix*" - # Collect all orgs that match the base run prefix pattern - $staleOrgs = Get-GitHubOrganization -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "$orgPrefix*" -and $_.Name -ne $orgName } + # Collect all orgs that match the base run prefix pattern (scoped to this enterprise) + $staleOrgs = Get-GitHubOrganization -Enterprise $owner -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "$orgRunPrefix*" -and $_.Name -ne $orgName } # Also check for the current org name in case it exists from a failed attempt - $currentOrg = Get-GitHubOrganization -Name $orgName -ErrorAction SilentlyContinue + $currentOrg = Get-GitHubOrganization -Enterprise $owner -Name $orgName -ErrorAction SilentlyContinue if ($currentOrg -and $currentOrg.Name) { $staleOrgs += $currentOrg } @@ -107,7 +107,7 @@ Describe 'Organizations' { } } } else { - Write-Host "No stale orgs found matching prefix: $orgPrefix*" + Write-Host "No stale orgs found matching prefix: $orgRunPrefix*" } } } From 87addeef69eaed98a0894ee7ed0d339b29a7048f Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 14:07:28 +0200 Subject: [PATCH 16/23] Optimize stale-org cleanup: use deterministic lookups instead of enumerating all enterprise orgs Replace inefficient enumeration of all enterprise organizations (which can burn API quota on large enterprises) with deterministic lookups of specific candidate org names. The cleanup now checks only the expected org names from the current run and previous attempts, reducing API calls and improving performance without functional change. --- tests/Organizations.Tests.ps1 | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 9dcf69c9b..a4e2b5460 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -59,20 +59,30 @@ Describe 'Organizations' { # GITHUB_RUN_ID. DELETE /orgs/{org} requires org-level administration:write, # so we install the app first to obtain an org-level IAT, then delete. LogGroup 'Pre-test Cleanup - Stale Enterprise Organization' { - # On reruns, clean up any orgs matching the base run prefix (e.g., ...-1234, - # ...-1234-2, etc.) before creating a new one. This prevents orphaned orgs - # from failed previous attempts. - $orgRunPrefix = "$testName-$os-$runId" - Write-Host "Searching for stale orgs matching prefix: $orgRunPrefix*" + # On reruns, clean up any orgs from the current and previous attempts. Deleted + # GitHub organizations are unavailable for 90 days, so rerun attempts must use + # unique org names to avoid collisions. Deterministically check for stale orgs + # from the base run (attempt 1) and any previous rerun attempts (2, 3, etc.). + # Use direct lookups by name instead of enumerating all enterprise orgs to avoid + # API quota burn on enterprises with many organizations. - # Collect all orgs that match the base run prefix pattern (scoped to this enterprise) - $staleOrgs = Get-GitHubOrganization -Enterprise $owner -ErrorAction SilentlyContinue | - Where-Object { $_.Name -like "$orgRunPrefix*" -and $_.Name -ne $orgName } + # Build deterministic list of org names to check: base run + previous attempts + $orgNamesToCheck = @("$testName-$os-$runId") # Attempt 1 + if ($attempt -and $attempt -ne '1') { + for ($attemptNum = 2; $attemptNum -le [int]$attempt; $attemptNum++) { + $orgNamesToCheck += "$testName-$os-$runId-$attemptNum" + } + } - # Also check for the current org name in case it exists from a failed attempt - $currentOrg = Get-GitHubOrganization -Enterprise $owner -Name $orgName -ErrorAction SilentlyContinue - if ($currentOrg -and $currentOrg.Name) { - $staleOrgs += $currentOrg + # Check each expected org name; collect any that exist and differ from current org + $staleOrgs = @() + foreach ($candidateName in $orgNamesToCheck) { + if ($candidateName -ne $orgName) { # Skip the current org we're about to create + $candidateOrg = Get-GitHubOrganization -Enterprise $owner -Name $candidateName -ErrorAction SilentlyContinue + if ($candidateOrg -and $candidateOrg.Name) { + $staleOrgs += $candidateOrg + } + } } if ($staleOrgs.Count -gt 0) { From 2fe8af36fb66a2c840ea47687576494af0ca51f3 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 14:17:19 +0200 Subject: [PATCH 17/23] Address review feedback: use idiomatic if (\) over .Count; reword environment cleanup comments to reflect per-test-file repo isolation --- tests/Organizations.Tests.ps1 | 11 ++++++----- tests/Secrets.Tests.ps1 | 4 ++-- tests/Variables.Tests.ps1 | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index a4e2b5460..9bfb685ab 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -65,7 +65,7 @@ Describe 'Organizations' { # from the base run (attempt 1) and any previous rerun attempts (2, 3, etc.). # Use direct lookups by name instead of enumerating all enterprise orgs to avoid # API quota burn on enterprises with many organizations. - + # Build deterministic list of org names to check: base run + previous attempts $orgNamesToCheck = @("$testName-$os-$runId") # Attempt 1 if ($attempt -and $attempt -ne '1') { @@ -73,19 +73,20 @@ Describe 'Organizations' { $orgNamesToCheck += "$testName-$os-$runId-$attemptNum" } } - + # Check each expected org name; collect any that exist and differ from current org $staleOrgs = @() foreach ($candidateName in $orgNamesToCheck) { - if ($candidateName -ne $orgName) { # Skip the current org we're about to create + if ($candidateName -ne $orgName) { + # Skip the current org we're about to create $candidateOrg = Get-GitHubOrganization -Enterprise $owner -Name $candidateName -ErrorAction SilentlyContinue if ($candidateOrg -and $candidateOrg.Name) { $staleOrgs += $candidateOrg } } } - - if ($staleOrgs.Count -gt 0) { + + if ($staleOrgs) { foreach ($staleOrg in $staleOrgs) { Write-Host "Stale org [$($staleOrg.Name)] found from previous run. Removing..." try { diff --git a/tests/Secrets.Tests.ps1 b/tests/Secrets.Tests.ps1 index 6acd3fff3..2d3e149a2 100644 --- a/tests/Secrets.Tests.ps1 +++ b/tests/Secrets.Tests.ps1 @@ -98,8 +98,8 @@ Describe 'Secrets' { } } } - # Remove the test environment created on the per-test-file repository so it does - # not leak into other test files or subsequent reruns. + # Remove the test environment created on the per-test-file repository as a + # defense-in-depth measure to keep the repo clean across reruns. if ($OwnerType -notin ('repository', 'enterprise') -and $repo) { LogGroup "Environment cleanup - [$environmentName] on [$repoName]" { Get-GitHubEnvironment -Owner $owner -Repository $repoName -Name $environmentName -ErrorAction SilentlyContinue | diff --git a/tests/Variables.Tests.ps1 b/tests/Variables.Tests.ps1 index ae95c05cd..91545daa2 100644 --- a/tests/Variables.Tests.ps1 +++ b/tests/Variables.Tests.ps1 @@ -97,8 +97,8 @@ Describe 'Variables' { $variablesToRemove | Remove-GitHubVariable -Confirm:$false } } - # Remove the test environment created on the per-test-file repository so it does - # not leak into other test files or subsequent reruns. + # Remove the test environment created on the per-test-file repository as a + # defense-in-depth measure to keep the repo clean across reruns. if ($OwnerType -notin ('repository', 'enterprise') -and $repo) { LogGroup "Environment cleanup - [$environmentName] on [$repoName]" { Get-GitHubEnvironment -Owner $owner -Repository $repoName -Name $environmentName -ErrorAction SilentlyContinue | From e06d31dc77da93e4aae6e93684512b656245bc7b Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 14:48:09 +0200 Subject: [PATCH 18/23] Fix Organizations test review findings on stale-org lookup and logging --- tests/Organizations.Tests.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 9bfb685ab..8c1cdb003 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -79,7 +79,7 @@ Describe 'Organizations' { foreach ($candidateName in $orgNamesToCheck) { if ($candidateName -ne $orgName) { # Skip the current org we're about to create - $candidateOrg = Get-GitHubOrganization -Enterprise $owner -Name $candidateName -ErrorAction SilentlyContinue + $candidateOrg = Get-GitHubOrganization -Name $candidateName -ErrorAction SilentlyContinue if ($candidateOrg -and $candidateOrg.Name) { $staleOrgs += $candidateOrg } @@ -118,7 +118,7 @@ Describe 'Organizations' { } } } else { - Write-Host "No stale orgs found matching prefix: $orgRunPrefix*" + Write-Host "No stale orgs found matching prefix: $orgPrefix*" } } } From c5551d054ba9de22eaeec52448dd128ee7ffff08 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Wed, 13 May 2026 14:58:39 +0200 Subject: [PATCH 19/23] Harden stale enterprise org cleanup in Organizations tests --- tests/Organizations.Tests.ps1 | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 8c1cdb003..48069882d 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -74,15 +74,14 @@ Describe 'Organizations' { } } - # Check each expected org name; collect any that exist and differ from current org + # Check each expected org name; collect any that currently exist. + # Include the current attempt name as well so reruns of the same + # GITHUB_RUN_ATTEMPT remain idempotent. $staleOrgs = @() foreach ($candidateName in $orgNamesToCheck) { - if ($candidateName -ne $orgName) { - # Skip the current org we're about to create - $candidateOrg = Get-GitHubOrganization -Name $candidateName -ErrorAction SilentlyContinue - if ($candidateOrg -and $candidateOrg.Name) { - $staleOrgs += $candidateOrg - } + $candidateOrg = Get-GitHubOrganization -Name $candidateName -ErrorAction SilentlyContinue + if ($candidateOrg -and $candidateOrg.Name) { + $staleOrgs += $candidateOrg } } @@ -100,8 +99,13 @@ Describe 'Organizations' { -ClientID $installationContext.ClientID -RepositorySelection 'all' -ErrorAction Stop break } catch { + $message = $_.Exception.Message + if ($message -match 'already\s+installed') { + Write-Host "App is already installed on stale org [$($staleOrg.Name)]; continuing with org-level cleanup context." + break + } if ($retryAttempt -lt $maxAttempts) { - Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $($_.Exception.Message). Retrying in ${retryDelay}s..." + Write-Host "Install-GitHubApp attempt $retryAttempt/$maxAttempts failed: $message. Retrying in ${retryDelay}s..." Start-Sleep -Seconds $retryDelay } else { throw From faa656f1e65c804b2716e094009d401bc097521c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 14 May 2026 09:24:55 +0200 Subject: [PATCH 20/23] Clarify stale org cleanup log with checked candidate names --- tests/Organizations.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index 48069882d..b4aa27019 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -122,7 +122,7 @@ Describe 'Organizations' { } } } else { - Write-Host "No stale orgs found matching prefix: $orgPrefix*" + Write-Host "No stale orgs found among candidates: $($orgNamesToCheck -join ', ')" } } } From 299352dc42ef66b1bd2cf6581bdb7904f8e8dd9e Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 14 May 2026 11:55:00 +0200 Subject: [PATCH 21/23] Fix Environments rerun cleanup per review --- .github/instructions/tests.instructions.md | 10 +++++----- tests/Environments.Tests.ps1 | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/instructions/tests.instructions.md b/.github/instructions/tests.instructions.md index ccd6e293b..8f191e5a0 100644 --- a/.github/instructions/tests.instructions.md +++ b/.github/instructions/tests.instructions.md @@ -83,11 +83,11 @@ Runs once before all parallel test files. For each auth case (except `GITHUB_TOK 4. For `organization` owners only, provisions extra repositories (`-2`, `-3` suffix) for test files that need companion repos (e.g., Secrets/Variables `SelectedRepository` tests) -`Set-GitHubRepository` is idempotent — if the repository already exists it updates it in place (issuing a -PATCH), and if it does not exist it creates it. Because the same parameters are passed each time, the -end-state is identical regardless of how many times the setup runs. The extra PATCH on the happy path is -a deliberate trade-off for simplicity: one call handles both first-run and partial-rerun scenarios without -branching logic. +Global setup deliberately removes any matching deterministic repositories before provisioning them again. +That gives workflow reruns a clean repository state for resources such as releases, tags, environments, +secrets, and variables. Each individual test file still calls `Set-GitHubRepository` in its per-context +`BeforeAll` as an idempotent safety net, so a single test file or auth context can be rerun independently +even when the global setup step did not run first. ### `AfterAll.ps1` — global teardown diff --git a/tests/Environments.Tests.ps1 b/tests/Environments.Tests.ps1 index 8aa9d2ca6..336a34a79 100644 --- a/tests/Environments.Tests.ps1 +++ b/tests/Environments.Tests.ps1 @@ -64,6 +64,20 @@ Describe 'Environments' { } Write-Host ($repo | Select-Object * | Out-String) } + + # Clean up stale environments from prior runs with the same GITHUB_RUN_ID. + # This keeps isolated reruns self-contained even when the global BeforeAll did not reset the repository. + if ($repo) { + LogGroup "Pre-test Cleanup - Existing Environments on [$repoName]" { + $existingEnvironments = Get-GitHubEnvironment -Owner $owner -Repository $repoName -ErrorAction SilentlyContinue + if ($existingEnvironments) { + Write-Host ($existingEnvironments | Format-Table | Out-String) + $existingEnvironments | Remove-GitHubEnvironment -Confirm:$false + } else { + Write-Host 'No existing environments to clean up.' + } + } + } } AfterAll { From 35fd9be8345501a9a6041a4e0b741bdfc532ba92 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 14 May 2026 12:31:16 +0200 Subject: [PATCH 22/23] Fix PSAlignAssignmentStatement: align TestNames and TestNamesWithExtraRepos assignment operators --- tests/Data/TestRepos.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Data/TestRepos.ps1 b/tests/Data/TestRepos.ps1 index a2b072da2..92f222624 100644 --- a/tests/Data/TestRepos.ps1 +++ b/tests/Data/TestRepos.ps1 @@ -4,8 +4,8 @@ # AfterAll.ps1 both source this file so setup and teardown always operate on the same set. @{ # Test files that each need a primary repository. - TestNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') + TestNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') # Subset that also need companion -2/-3 repositories for org-scoped SelectedRepository tests. - TestNamesWithExtraRepos = @('Secrets', 'Variables') + TestNamesWithExtraRepos = @('Secrets', 'Variables') } From 859838dcccebe8d6d311792f1c3f08dd4ec61a02 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 14 May 2026 12:46:42 +0200 Subject: [PATCH 23/23] Refactor test repository definitions for consistency and clarity --- tests/Data/TestRepos.ps1 | 4 ++-- tests/Organizations.Tests.ps1 | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/Data/TestRepos.ps1 b/tests/Data/TestRepos.ps1 index 92f222624..53edd365d 100644 --- a/tests/Data/TestRepos.ps1 +++ b/tests/Data/TestRepos.ps1 @@ -4,8 +4,8 @@ # AfterAll.ps1 both source this file so setup and teardown always operate on the same set. @{ # Test files that each need a primary repository. - TestNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') + TestNames = @('Environments', 'Secrets', 'Variables', 'Releases', 'Actions') # Subset that also need companion -2/-3 repositories for org-scoped SelectedRepository tests. - TestNamesWithExtraRepos = @('Secrets', 'Variables') + TestNamesWithExtraRepos = @('Secrets', 'Variables') } diff --git a/tests/Organizations.Tests.ps1 b/tests/Organizations.Tests.ps1 index b4aa27019..69ac6653f 100644 --- a/tests/Organizations.Tests.ps1 +++ b/tests/Organizations.Tests.ps1 @@ -222,6 +222,10 @@ Describe 'Organizations' { { Update-GitHubOrganization -Name $orgName -Location 'New Location' } | Should -Throw } + It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { + { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw + } + It 'Install-GitHubApp - Installs a GitHub App to an organization' -Skip:($OwnerType -ne 'enterprise') { # Retry: the enterprise apps endpoint can return 404 transiently right after # New-GitHubOrganization, before the new org has propagated. @@ -262,10 +266,6 @@ Describe 'Organizations' { Update-GitHubOrganization -Name $orgName -Location 'New Location' -Context $orgContext } - It 'Remove-GitHubOrganization - Removes an organization using enterprise installation' -Skip:($OwnerType -ne 'enterprise') { - { Remove-GitHubOrganization -Name $orgName -Confirm:$false } | Should -Throw - } - It 'Remove-GitHubOrganization - Removes an organization using organization installation' -Skip:($OwnerType -ne 'enterprise') { $orgContext = Connect-GitHubApp -Organization $orgName -Context $context -PassThru -Silent Remove-GitHubOrganization -Name $orgName -Confirm:$false -Context $orgContext