Fix Copilot PR timing: increase retry limit and make tests resilient to missing PRs

Co-authored-by: lindseywild <35239154+lindseywild@users.noreply.github.com>

Author: Copilot

.github/actions/fix/src/retry.ts

@@ -16,7 +16,7 @@ function sleep(ms: number): Promise<void> {
  */
 export async function retry<T>(
   fn: () => Promise<T | null | undefined> | T | null | undefined,
-  maxAttempts = 6,
+  maxAttempts = 10,
   baseDelay = 2000,
   attempt = 1,
 ): Promise<T | undefined> {

tests/site-with-errors.test.ts

@@ -16,12 +16,10 @@ describe('site-with-errors', () => {
   })
 
   it('cache has expected results', () => {
-    const actual = results.map(({issue: {url: issueUrl}, pullRequest, findings}) => {
-      const pullRequestUrl = pullRequest?.url
+    const actual = results.map(({issue: {url: issueUrl}, findings}) => {
       const {problemUrl, solutionLong, screenshotId, ...finding} = findings[0]
       // Check volatile fields for existence only
       expect(issueUrl).toBeDefined()
-      expect(pullRequestUrl).toBeDefined()
       expect(problemUrl).toBeDefined()
       expect(solutionLong).toBeDefined()
       // Check `problemUrl`, ignoring axe version
@@ -145,21 +143,22 @@ describe('site-with-errors', () => {
       )
       // Fetch pull requests referenced in the findings file
       pullRequests = await Promise.all(
-        results.map(async ({pullRequest}) => {
-          const pullRequestUrl = pullRequest?.url
-          expect(pullRequestUrl).toBeDefined()
-          const {owner, repo, pullNumber} =
-            /https:\/\/github\.com\/(?<owner>[^/]+)\/(?<repo>[^/]+)\/pull\/(?<pullNumber>\d+)/.exec(
-              pullRequestUrl!,
-            )!.groups!
-          const {data: fetchedPullRequest} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
-            owner,
-            repo,
-            pull_number: parseInt(pullNumber, 10),
-          })
-          expect(fetchedPullRequest).toBeDefined()
-          return fetchedPullRequest
-        }),
+        results
+          .filter(({pullRequest}) => !!pullRequest?.url)
+          .map(async ({pullRequest}) => {
+            const pullRequestUrl = pullRequest!.url
+            const {owner, repo, pullNumber} =
+              /https:\/\/github\.com\/(?<owner>[^/]+)\/(?<repo>[^/]+)\/pull\/(?<pullNumber>\d+)/.exec(
+                pullRequestUrl,
+              )!.groups!
+            const {data: fetchedPullRequest} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
+              owner,
+              repo,
+              pull_number: parseInt(pullNumber, 10),
+            })
+            expect(fetchedPullRequest).toBeDefined()
+            return fetchedPullRequest
+          }),
       )
     })
 
@@ -183,6 +182,8 @@ describe('site-with-errors', () => {
     })
 
     it('pull requests exist and have expected author, state, and assignee', async () => {
+      // Verify every result has an associated pull request (not just those that happened to be fetched)
+      expect(pullRequests).toHaveLength(results.length)
       for (const pullRequest of pullRequests) {
         expect(pullRequest.user.login).toBe('Copilot')
         expect(pullRequest.state).toBe('open')

tests/types.d.ts

@@ -26,5 +26,5 @@ export type PullRequest = {
 export type Result = {
   findings: Finding[]
   issue: Issue
-  pullRequest: PullRequest
+  pullRequest?: PullRequest
 }