From 71f27874eca12e82921e86456f5cb59ffc4219ad Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:24:48 +0200 Subject: [PATCH] test(pwsh): add opt-in contract tests against a disposable repository Every other test mocks at the CLI wrapper boundary, which proves the module's logic but says nothing about whether GitHub accepts the payloads it builds or returns the shapes it parses. Two of the four bugs found in review were exactly that gap: assumptions about API shapes that the mocks confirmed because the fixtures were written from the same assumptions. These drive the real audit/remediate cycle against a throwaway repository and assert against the live API - reading with `gh api` directly rather than through the module, so a symmetrical bug in audit and remediate cannot hide: - settings converge, and a second run applies nothing - the ruleset is repository-owned, targets branch, covers the default branch, and keeps the admin bypass - require_code_owner_review and friends survive a merge-method remediation (the regression the mocks could not catch) - a same-named tag ruleset is left untouched - the environment ends up pinned to the default branch and nothing else, with the secret in the environment rather than repo-level - a broader branch policy is removed rather than left beside the pin Destructive, so off by default. Skipped unless DOTFILES_CONTRACT_TEST=1; CI runs Pester with no tag filter, so a tag alone would not have been enough. Verified that Pester does not execute the file's BeforeAll while every test in it is skipped, which is what keeps a privileged token from creating orphan repositories on an ordinary run. The repository is named zz-delete-me-dotfiles-contract- so an orphan left by a hard kill is obviously safe to delete, and AfterAll confirms the deletion rather than assuming it. Not yet executed: the token in use cannot create repositories. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/github-repo-config.md | 48 +++ .../GitHubRepoConfig.Contract.Tests.ps1 | 291 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 tests/powershell/GitHubRepoConfig.Contract.Tests.ps1 diff --git a/docs/github-repo-config.md b/docs/github-repo-config.md index 239c8a44..0ca74ddf 100644 --- a/docs/github-repo-config.md +++ b/docs/github-repo-config.md @@ -471,3 +471,51 @@ the suite runnable without them installed. Every external dependency is mocked at the module's own CLI wrappers, so the suite runs on any platform without `gh` or `op` installed and never touches a real repository. + +### Contract tests + +Mocking at the wrapper boundary proves the module's logic but not that GitHub +accepts the payloads it builds or returns the shapes it parses. Two of the four +bugs found in review were exactly that — assumptions the mocks confirmed because +the fixtures were written from the same assumptions. + +`GitHubRepoConfig.Contract.Tests.ps1` closes that gap by driving the real +audit/remediate cycle against a throwaway repository. It verifies, against the +live API: + +- settings converge and are idempotent on a second run +- the created ruleset is a repository-owned branch ruleset covering the default + branch, with the admin bypass intact +- `require_code_owner_review` and friends survive a merge-method remediation +- a same-named **tag** ruleset is left untouched +- the `production` environment ends up pinned to the default branch and nothing + else, with the secret in the environment rather than at repository level + +!!! danger "Destructive, and off by default" + + These tests create and delete a repository. They are skipped unless + `DOTFILES_CONTRACT_TEST=1`, so a normal suite or a CI run can never trigger + them — Pester does not even execute the file's setup block while every test + in it is skipped. + +Running them needs a token beyond the everyday one — it must be able to create +and delete repositories: + +| Permission | Why | +| ------------------------------------- | ---------------------------------------- | +| Repository creation (account level) | Create the disposable repo | +| `delete_repo` (classic) or equivalent | Delete it again | +| Administration: read & write | Settings, rulesets, Actions permissions | +| Secrets / Variables: read & write | Credential placement | +| Environments: read & write | Environment creation and branch policies | +| Contents: read | Pages-workflow detection | + +```powershell +$env:DOTFILES_CONTRACT_TEST = '1' +./tests/powershell/Invoke-PesterTests.ps1 -Tag Contract +``` + +The repository is named `zz-delete-me-dotfiles-contract-` so an orphan +left by a hard kill is obviously safe to remove, and it is deleted in `AfterAll` +even when a test fails. It is created **public** because environments and +deployment branch policies are public-repository-only on the Free plan. diff --git a/tests/powershell/GitHubRepoConfig.Contract.Tests.ps1 b/tests/powershell/GitHubRepoConfig.Contract.Tests.ps1 new file mode 100644 index 00000000..d4216b9f --- /dev/null +++ b/tests/powershell/GitHubRepoConfig.Contract.Tests.ps1 @@ -0,0 +1,291 @@ +#Requires -Version 5.1 +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Placeholder credential for a throwaway repository; not a real key.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingWriteHost', '', Justification = 'Destructive opt-in test; announcing the repository it creates and deletes is the point.')] +param() +<# +.SYNOPSIS + Opt-in contract tests for the GitHub repository configuration tooling, + run against a real, disposable GitHub repository. + +.DESCRIPTION + Every other test in this suite mocks at the CLI wrapper boundary, which + proves the module's own logic but says nothing about whether the payloads + it builds are ones the GitHub API actually accepts, or whether the shapes + it parses are ones the API actually returns. Two of the four bugs found in + review were exactly that: assumptions about API shapes that the mocks + happily confirmed because the fixtures were written from the same + assumptions. + + These tests close that gap. They create a throwaway repository, drive the + real audit/remediate cycle against it, and delete it again. + + DESTRUCTIVE and NOT run by default. They are skipped unless + DOTFILES_CONTRACT_TEST=1, so a stray CI run or a normal local suite can + never create repositories. + + Requirements: + - DOTFILES_CONTRACT_TEST=1 + - `gh` authenticated with a token that can create and delete + repositories, and administer them: + Repository creation (account level) + Administration: read & write (settings, rulesets, Actions perms) + Secrets / Variables: read & write + Environments: read & write + Contents: read + - Optional: DOTFILES_CONTRACT_OWNER to place the repository somewhere + other than the authenticated user. + + The repository is created public because GitHub environments and + deployment branch policies are public-repository-only on the Free plan, + and the environment pinning behaviour is one of the things worth + verifying. It is empty, exists for the duration of the run, and is + deleted in AfterAll even when a test fails. +#> + +BeforeDiscovery { + $script:ContractEnabled = $env:DOTFILES_CONTRACT_TEST -eq '1' +} + +BeforeAll { + $script:RepoRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + $modulePath = Join-Path $script:RepoRoot 'home/dot_config/powershell/modules/DotfilesHelpers' + Get-Module DotfilesHelpers -All | Remove-Module -Force -ErrorAction SilentlyContinue + Import-Module $modulePath -Force -DisableNameChecking + + $script:Owner = if ($env:DOTFILES_CONTRACT_OWNER) { + $env:DOTFILES_CONTRACT_OWNER + } + else { + (& gh api user --jq '.login' | Out-String).Trim() + } + + # Name carries its own explanation, so an orphan left by a hard kill is + # obviously safe to delete. + $script:RepoName = "zz-delete-me-dotfiles-contract-$(Get-Random -Minimum 100000 -Maximum 999999)" + $script:Target = "$($script:Owner)/$($script:RepoName)" + + Write-Host "[contract] creating disposable repository $($script:Target)" -ForegroundColor Cyan + $created = & gh api -X POST user/repos ` + -f name=$script:RepoName ` + -f description='Disposable repository for dotfiles contract tests. Safe to delete.' ` + -F private=false ` + -F auto_init=true 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Could not create $($script:Target): $created" + } + + # auto_init commits a README, so there is a default branch to protect. + $script:DefaultBranch = (& gh api "repos/$script:Target" --jq '.default_branch' | Out-String).Trim() +} + +AfterAll { + if ($script:Target) { + Write-Host "[contract] deleting $($script:Target)" -ForegroundColor Cyan + $null = & gh api -X DELETE "repos/$script:Target" 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Warning "Could not delete $($script:Target). Delete it by hand: gh repo delete $($script:Target) --yes" + } + else { + # Deletion is the one thing that must not silently fail, so confirm. + $null = & gh api "repos/$script:Target" 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Warning "$($script:Target) still exists after delete; remove it by hand." + } + } + } +} + +Describe "Repository settings round-trip" -Tag "Contract" -Skip:(-not $script:ContractEnabled) { + It "reports drift on a freshly created repository" { + $result = Get-GitHubRepoConfig -Repository $script:Target -Check Settings + $result.DriftCount | Should -BeGreaterThan 0 -Because 'a new repository uses GitHub defaults, not the baseline' + } + + It "converges to compliant after remediation" { + Get-GitHubRepoConfig -Repository $script:Target -Check Settings | + Set-GitHubRepoConfig -Category Settings -Confirm:$false + + $after = Get-GitHubRepoConfig -Repository $script:Target -Check Settings + $after.DriftCount | Should -Be 0 + $after.IsCompliant | Should -BeTrue + } + + It "is idempotent: a second run writes nothing" { + $result = Get-GitHubRepoConfig -Repository $script:Target -Check Settings | + Set-GitHubRepoConfig -Category Settings -Confirm:$false -PassThru + @($result.Applied).Count | Should -Be 0 + } + + It "actually applied the baseline values, not just its own idea of them" { + # Read straight from the API rather than through the module, so a + # symmetrical bug in audit and remediate cannot hide. + $repo = & gh api "repos/$script:Target" | ConvertFrom-Json + $repo.allow_squash_merge | Should -BeTrue + $repo.allow_merge_commit | Should -BeFalse + $repo.allow_rebase_merge | Should -BeFalse + $repo.delete_branch_on_merge | Should -BeTrue + $repo.squash_merge_commit_title | Should -Be 'PR_TITLE' + $repo.squash_merge_commit_message | Should -Be 'BLANK' + } +} + +Describe "Ruleset round-trip" -Tag "Contract" -Skip:(-not $script:ContractEnabled) { + It "creates a ruleset the API accepts" { + Get-GitHubRepoConfig -Repository $script:Target -Check Ruleset | + Set-GitHubRepoConfig -Category Ruleset -Confirm:$false + + $after = Get-GitHubRepoConfig -Repository $script:Target -Check Ruleset + $after.DriftCount | Should -Be 0 + } + + It "produced a repository-owned branch ruleset covering the default branch" { + $rulesets = & gh api "repos/$script:Target/rulesets" | ConvertFrom-Json + $default = @($rulesets | Where-Object { $_.name -eq 'Default' })[0] + $default | Should -Not -BeNullOrEmpty + $default.target | Should -Be 'branch' + + $detail = & gh api "repos/$script:Target/rulesets/$($default.id)" | ConvertFrom-Json + $detail.conditions.ref_name.include | Should -Contain '~DEFAULT_BRANCH' + @($detail.rules | ForEach-Object { $_.type }) | Should -Contain 'pull_request' + } + + It "kept the admin bypass so the owner is not locked out" { + $rulesets = & gh api "repos/$script:Target/rulesets" | ConvertFrom-Json + $default = @($rulesets | Where-Object { $_.name -eq 'Default' })[0] + $detail = & gh api "repos/$script:Target/rulesets/$($default.id)" | ConvertFrom-Json + + @($detail.bypass_actors | Where-Object { + $_.actor_type -eq 'RepositoryRole' -and $_.actor_id -eq 5 + }).Count | Should -Be 1 + } + + It "preserves pull_request parameters the baseline does not own" { + # The regression the mocks could not catch: set a stricter review + # setting through the real API, force a merge-method drift, remediate, + # and confirm the stricter setting survived. + $rulesets = & gh api "repos/$script:Target/rulesets" | ConvertFrom-Json + $default = @($rulesets | Where-Object { $_.name -eq 'Default' })[0] + $detail = & gh api "repos/$script:Target/rulesets/$($default.id)" | ConvertFrom-Json + + $rules = @($detail.rules | Where-Object { $_.type -ne 'pull_request' }) + $rules += @{ + type = 'pull_request' + parameters = @{ + allowed_merge_methods = @('merge', 'squash', 'rebase') + required_approving_review_count = 0 + require_code_owner_review = $true + dismiss_stale_reviews_on_push = $true + require_last_push_approval = $false + required_review_thread_resolution = $true + } + } + + $body = @{ + name = $detail.name + target = 'branch' + enforcement = 'active' + bypass_actors = @($detail.bypass_actors) + conditions = @{ ref_name = @{ include = @('~DEFAULT_BRANCH'); exclude = @() } } + rules = $rules + } | ConvertTo-Json -Depth 10 -Compress + + $null = $body | & gh api "repos/$script:Target/rulesets/$($default.id)" --method PUT --input - 2>&1 + $LASTEXITCODE | Should -Be 0 -Because 'the seeding call itself must succeed for this test to mean anything' + + Get-GitHubRepoConfig -Repository $script:Target -Check Ruleset | + Set-GitHubRepoConfig -Category Ruleset -Confirm:$false + + $final = & gh api "repos/$script:Target/rulesets/$($default.id)" | ConvertFrom-Json + $pr = @($final.rules | Where-Object { $_.type -eq 'pull_request' })[0] + + $pr.parameters.require_code_owner_review | Should -BeTrue -Because 'remediating merge methods must not weaken review requirements' + $pr.parameters.dismiss_stale_reviews_on_push | Should -BeTrue + $pr.parameters.required_review_thread_resolution | Should -BeTrue + $pr.parameters.allowed_merge_methods | Should -Be @('squash') + } + + It "leaves a same-named tag ruleset untouched" { + $tagBody = @{ + name = 'Default' + target = 'tag' + enforcement = 'active' + conditions = @{ ref_name = @{ include = @('refs/tags/v*'); exclude = @() } } + rules = @(@{ type = 'deletion' }) + } | ConvertTo-Json -Depth 10 -Compress + + $created = $tagBody | & gh api "repos/$script:Target/rulesets" --method POST --input - 2>&1 + $LASTEXITCODE | Should -Be 0 + $tagId = ($created | ConvertFrom-Json).id + + Get-GitHubRepoConfig -Repository $script:Target -Check Ruleset | + Set-GitHubRepoConfig -Category Ruleset -Confirm:$false + + $tag = & gh api "repos/$script:Target/rulesets/$tagId" | ConvertFrom-Json + $tag.target | Should -Be 'tag' -Because 'a tag ruleset sharing the name must never be overwritten with a branch payload' + @($tag.rules | ForEach-Object { $_.type }) | Should -Be @('deletion') + } +} + +Describe "Environment credential placement" -Tag "Contract" -Skip:(-not $script:ContractEnabled) { + BeforeAll { + # A placeholder, not a real key: this exercises placement against the + # live API without needing 1Password or a genuine App credential. + $script:FakeCredential = [PSCustomObject]@{ + PSTypeName = 'Dotfiles.GitHubAppCredential' + AppId = '000000' + PrivateKey = (ConvertTo-SecureString "-----BEGIN RSA PRIVATE KEY-----`nCONTRACTTEST`n-----END RSA PRIVATE KEY-----" -AsPlainText -Force) + } + } + + It "creates the environment, pins it, and stores the credential" { + Get-GitHubRepoConfig -Repository $script:Target -Check AppCredential | + Set-GitHubRepoConfig -Category AppCredential -AppCredential $script:FakeCredential -Confirm:$false + + $after = Get-GitHubRepoConfig -Repository $script:Target -Check AppCredential + $after.DriftCount | Should -Be 0 + } + + It "pinned the environment to the default branch and nothing else" { + $policies = & gh api "repos/$script:Target/environments/production/deployment-branch-policies" | ConvertFrom-Json + @($policies.branch_policies).Count | Should -Be 1 + $policies.branch_policies[0].name | Should -Be $script:DefaultBranch + } + + It "stored the secret in the environment, not at repository level" { + $envSecrets = & gh api "repos/$script:Target/environments/production/secrets" | ConvertFrom-Json + @($envSecrets.secrets | Where-Object { $_.name -eq 'AUTOMATION_APP_PRIVATE_KEY' }).Count | Should -Be 1 + + $repoSecrets = & gh api "repos/$script:Target/actions/secrets" | ConvertFrom-Json + @($repoSecrets.secrets | Where-Object { $_.name -eq 'AUTOMATION_APP_PRIVATE_KEY' }).Count | Should -Be 0 + } + + It "removes a broader branch policy rather than leaving it alongside the pin" { + $null = @{ name = 'feature/*'; type = 'branch' } | ConvertTo-Json -Compress | + & gh api "repos/$script:Target/environments/production/deployment-branch-policies" --method POST --input - 2>&1 + $LASTEXITCODE | Should -Be 0 + + $audit = Get-GitHubRepoConfig -Repository $script:Target -Check AppCredential + @($audit.Drift | Where-Object { $_.Setting -eq 'environment_pinned_to_default_branch' }).Count | + Should -Be 1 -Because 'main plus feature/* is not pinned' + + $audit | Set-GitHubRepoConfig -Category AppCredential -AppCredential $script:FakeCredential -Confirm:$false + + $policies = & gh api "repos/$script:Target/environments/production/deployment-branch-policies" | ConvertFrom-Json + @($policies.branch_policies).Count | Should -Be 1 + $policies.branch_policies[0].name | Should -Be $script:DefaultBranch + } +} + +Describe "Actions permissions round-trip" -Tag "Contract" -Skip:(-not $script:ContractEnabled) { + It "applies and converges" { + Get-GitHubRepoConfig -Repository $script:Target -Check Actions | + Set-GitHubRepoConfig -Category Actions -Confirm:$false + + $after = Get-GitHubRepoConfig -Repository $script:Target -Check Actions + $after.DriftCount | Should -Be 0 + + $perms = & gh api "repos/$script:Target/actions/permissions/workflow" | ConvertFrom-Json + $perms.default_workflow_permissions | Should -Be 'read' + $perms.can_approve_pull_request_reviews | Should -BeFalse + } +}