Adds site-with-errors test back, only checks PRs daily (#156)

lindseywild · web-flow · commit b826e2651401 · 2026-03-13T12:05:39.000-04:00
Adds back the site-with-errors tests to run on every PR, but will only check issues. PR creation validation will be run at a daily cadence so we can add enough of a timeout to ensure the Copilot PRs are created (sometimes they take several minutes). Also adds a check in the workflow if the test _does_ fail, so we can address it. Current [site-with-errors test added back passes](https://github.com/github/accessibility-scanner/actions/runs/23054537722/job/66964182473?pr=156), excluding the PR check 🎉 I'll have to run the `test-copilot-pr-creation.yml` workflow once this PR is merged but can make any follow-up changes necessary!
diff --git a/.github/workflows/report-failure.yml b/.github/workflows/report-failure.yml
@@ -0,0 +1,16 @@
+name: Report failure
+
+on: workflow_call
+
+permissions:
+  issues: write
+
+jobs:
+  report-failure:
+    runs-on: ubuntu-latest
+    name: Report failure
+    steps:
+      - uses: actions/github-script@v8
+        with:
+          script: |
+            github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: "`${{ github.workflow }}` workflow failed", body: "The workflow, `${{ github.workflow }}`, failed to run. See [Action run output](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more information. cc: @github/accessibility-reviewers." })
diff --git a/.github/workflows/test-copilot-pr-creation.yml b/.github/workflows/test-copilot-pr-creation.yml
@@ -0,0 +1,136 @@
+name: Test Copilot PR creation
+on:
+  schedule:
+    - cron: "0 6 * * *" # Every day at 06:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+env:
+  TESTING_REPOSITORY: github/accessibility-scanner-testing
+
+jobs:
+  test_pull_request_creation:
+    name: Test (with PR creation)
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        site: ["sites/site-with-errors"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c
+        with:
+          ruby-version: "3.4"
+          bundler-cache: true
+          working-directory: ${{ matrix.site }}
+
+      - name: Build Jekyll site (${{ matrix.site }})
+        shell: bash
+        working-directory: ${{ matrix.site }}
+        env:
+          JEKYLL_ENV: production
+        run: bundle exec jekyll build
+
+      - name: Start Puma (${{ matrix.site }})
+        shell: bash
+        working-directory: ${{ matrix.site }}
+        env:
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+          TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }}
+        run: |
+          set -euo pipefail
+          bundle exec puma -b tcp://127.0.0.1:4000 &
+          echo "Starting Puma on port 4000"
+          curl -fsS --retry 25 --retry-delay 1 --retry-all-errors -u "${TEST_USERNAME}:${TEST_PASSWORD}" "http://127.0.0.1:4000/" > /dev/null
+          echo "Puma has started"
+
+      - name: Generate cache key
+        id: cache_key
+        shell: bash
+        run: |
+          echo "cache_key=$(printf 'cached_results-%s-%s.json' "${{ matrix.site }}" "daily" | tr -cs 'A-Za-z0-9._-' '_')" >> $GITHUB_OUTPUT
+
+      - name: Scan site (${{ matrix.site }})
+        uses: ./
+        with:
+          urls: |
+            http://127.0.0.1:4000/
+            http://127.0.0.1:4000/jekyll/update/2025/07/30/welcome-to-jekyll.html
+            http://127.0.0.1:4000/about/
+            http://127.0.0.1:4000/404.html
+          login_url: http://127.0.0.1:4000/
+          username: ${{ secrets.TEST_USERNAME }}
+          password: ${{ secrets.TEST_PASSWORD }}
+          repository: ${{ env.TESTING_REPOSITORY }}
+          token: ${{ secrets.GH_TOKEN }}
+          cache_key: ${{ steps.cache_key.outputs.cache_key }}
+
+      - name: Retrieve cached results
+        uses: ./.github/actions/gh-cache/restore
+        with:
+          path: ${{ steps.cache_key.outputs.cache_key }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Validate scan results (${{ matrix.site }})
+        run: |
+          npm ci
+          npm run test
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          CACHE_PATH: ${{ steps.cache_key.outputs.cache_key }}
+          WAIT_FOR_PULL_REQUESTS: "true"
+
+      - name: Clean up issues and pull requests
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ ! -f "${{ steps.cache_key.outputs.cache_key }}" ]]; then
+            echo "Skipping 'Clean up issues and pull requests' (no cached results)."
+            exit 0
+          fi
+          jq -r '
+            (if type=="string" then fromjson else . end)
+            | .[] | .issue.url, .pullRequest.url
+            | select(. != null)
+          ' "${{ steps.cache_key.outputs.cache_key }}" \
+            | while read -r URL; do
+                if [[ "$URL" == *"/pull/"* ]]; then
+                  echo "Closing pull request: $URL"
+                  gh pr close "$URL" || echo "Failed to close pull request: $URL"
+                  branch="$(gh pr view "$URL" --json headRefName -q .headRefName || true)"
+                  if [[ -n "$branch" ]]; then
+                    echo "Deleting branch: $branch"
+                    gh api -X DELETE "repos/${{ env.TESTING_REPOSITORY }}/git/refs/heads/$branch" || echo "Failed to delete branch: $branch"
+                  fi
+                elif [[ "$URL" == *"/issues/"* ]]; then
+                  echo "Closing issue: $URL"
+                  gh issue close "$URL" || echo "Failed to close issue: $URL"
+                else
+                  echo "Skipping unrecognized url: $URL"
+                fi
+              done
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - name: Clean up cached results
+        if: ${{ always() }}
+        uses: ./.github/actions/gh-cache/delete
+        with:
+          path: ${{ steps.cache_key.outputs.cache_key }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  notify-of-failures:
+    if: always() && github.ref_name == 'main' && needs.test_pull_request_creation.result == 'failure'
+    needs: [test_pull_request_creation]
+    uses: github/accessibility-scorecard/.github/workflows/report-failure.yml@main
diff --git a/tests/site-with-errors.test.ts b/tests/site-with-errors.test.ts
@@ -6,7 +6,58 @@ import {Octokit} from '@octokit/core'
 import {throttling} from '@octokit/plugin-throttling'
 const OctokitWithThrottling = Octokit.plugin(throttling)
 
-describe.skip('site-with-errors', () => {
+const WAIT_FOR_PULL_REQUESTS = !!process.env.WAIT_FOR_PULL_REQUESTS
+const POLL_INTERVAL_MS = 30_000 // 30 seconds
+const POLL_TIMEOUT_MS = 15 * 60 * 1000 // 15 minutes
+
+/**
+ * Repeatedly calls `fn` until `predicate(result)` returns `true`, or until the timeout is exceeded.
+ * Errors thrown by `fn` (e.g. HTTP 404) are swallowed while polling continues.
+ */
+async function pollUntil<T>(
+  fn: () => Promise<T>,
+  predicate: (result: T) => boolean,
+  {intervalMs, timeoutMs}: {intervalMs: number; timeoutMs: number},
+): Promise<T> {
+  const deadline = Date.now() + timeoutMs
+  let lastError: unknown
+  while (true) {
+    try {
+      const result = await fn()
+      if (predicate(result)) return result
+    } catch (error) {
+      lastError = error
+    }
+    if (Date.now() >= deadline) {
+      throw lastError ?? new Error(`Timed out after ${timeoutMs}ms waiting for condition`)
+    }
+    await new Promise(resolve => setTimeout(resolve, intervalMs))
+  }
+}
+
+function createOctokit(): InstanceType<typeof OctokitWithThrottling> {
+  return new OctokitWithThrottling({
+    auth: process.env.GITHUB_TOKEN,
+    throttle: {
+      onRateLimit: (retryAfter, options, octokit, retryCount) => {
+        octokit.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`)
+        if (retryCount < 3) {
+          octokit.log.info(`Retrying after ${retryAfter} seconds!`)
+          return true
+        }
+      },
+      onSecondaryRateLimit: (retryAfter, options, octokit, retryCount) => {
+        octokit.log.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`)
+        if (retryCount < 3) {
+          octokit.log.info(`Retrying after ${retryAfter} seconds!`)
+          return true
+        }
+      },
+    },
+  })
+}
+
+describe('site-with-errors', () => {
   let results: Result[]
 
   beforeAll(() => {
@@ -16,11 +67,13 @@ describe.skip('site-with-errors', () => {
   })
 
   it('cache has expected results', () => {
-    const actual = results.map(({issue: {url: issueUrl}, pullRequest: {url: pullRequestUrl}, findings}) => {
+    const actual = results.map(({issue: {url: issueUrl}, pullRequest, findings}) => {
       const {problemUrl, solutionLong, screenshotId, ...finding} = findings[0]
       // Check volatile fields for existence only
       expect(issueUrl).toBeDefined()
-      expect(pullRequestUrl).toBeDefined()
+      if (WAIT_FOR_PULL_REQUESTS) {
+        expect(pullRequest?.url).toBeDefined()
+      }
       expect(problemUrl).toBeDefined()
       expect(solutionLong).toBeDefined()
       // Check `problemUrl`, ignoring axe version
@@ -100,32 +153,11 @@ describe.skip('site-with-errors', () => {
     expect(process.env.GITHUB_TOKEN).toBeDefined()
   })
 
-  describe.runIf(!!process.env.GITHUB_TOKEN)('—', () => {
-    let octokit: Octokit
+  describe.runIf(!!process.env.GITHUB_TOKEN)('issues', () => {
     let issues: Endpoints['GET /repos/{owner}/{repo}/issues/{issue_number}']['response']['data'][]
-    let pullRequests: Endpoints['GET /repos/{owner}/{repo}/pulls/{pull_number}']['response']['data'][]
 
     beforeAll(async () => {
-      octokit = new OctokitWithThrottling({
-        auth: process.env.GITHUB_TOKEN,
-        throttle: {
-          onRateLimit: (retryAfter, options, octokit, retryCount) => {
-            octokit.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`)
-            if (retryCount < 3) {
-              octokit.log.info(`Retrying after ${retryAfter} seconds!`)
-              return true
-            }
-          },
-          onSecondaryRateLimit: (retryAfter, options, octokit, retryCount) => {
-            octokit.log.warn(`Secondary rate limit hit for request ${options.method} ${options.url}`)
-            if (retryCount < 3) {
-              octokit.log.info(`Retrying after ${retryAfter} seconds!`)
-              return true
-            }
-          },
-        },
-      })
-      // Fetch issues referenced in the cache file
+      const octokit = createOctokit()
       issues = await Promise.all(
         results.map(async ({issue: {url: issueUrl}}) => {
           expect(issueUrl).toBeDefined()
@@ -142,23 +174,6 @@ describe.skip('site-with-errors', () => {
           return issue
         }),
       )
-      // Fetch pull requests referenced in the findings file
-      pullRequests = await Promise.all(
-        results.map(async ({pullRequest: {url: pullRequestUrl}}) => {
-          expect(pullRequestUrl).toBeDefined()
-          const {owner, repo, pullNumber} =
-            /https:\/\/github\.com\/(?<owner>[^/]+)\/(?<repo>[^/]+)\/pull\/(?<pullNumber>\d+)/.exec(
-              pullRequestUrl!,
-            )!.groups!
-          const {data: pullRequest} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
-            owner,
-            repo,
-            pull_number: parseInt(pullNumber, 10),
-          })
-          expect(pullRequest).toBeDefined()
-          return pullRequest
-        }),
-      )
     })
 
     it('issues exist and have expected title, state, and assignee', async () => {
@@ -179,6 +194,35 @@ describe.skip('site-with-errors', () => {
         expect(issue.assignees!.some(a => a.login === 'Copilot')).toBe(true)
       }
     })
+  })
+
+  describe.runIf(!!process.env.GITHUB_TOKEN && WAIT_FOR_PULL_REQUESTS)('pull requests', () => {
+    let pullRequests: Endpoints['GET /repos/{owner}/{repo}/pulls/{pull_number}']['response']['data'][]
+
+    beforeAll(async () => {
+      const octokit = createOctokit()
+      pullRequests = await Promise.all(
+        results.map(async ({pullRequest: {url: pullRequestUrl}}) => {
+          expect(pullRequestUrl).toBeDefined()
+          const {owner, repo, pullNumber} =
+            /https:\/\/github\.com\/(?<owner>[^/]+)\/(?<repo>[^/]+)\/pull\/(?<pullNumber>\d+)/.exec(
+              pullRequestUrl!,
+            )!.groups!
+          return pollUntil(
+            async () => {
+              const {data} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
+                owner,
+                repo,
+                pull_number: parseInt(pullNumber, 10),
+              })
+              return data
+            },
+            pr => pr.state === 'open',
+            {intervalMs: POLL_INTERVAL_MS, timeoutMs: POLL_TIMEOUT_MS},
+          )
+        }),
+      )
+    }, POLL_TIMEOUT_MS + 60_000)
 
     it('pull requests exist and have expected author, state, and assignee', async () => {
       for (const pullRequest of pullRequests) {
