From 75e732d9873e9f954cac68d861044f81fea171bd Mon Sep 17 00:00:00 2001 From: joshuaterk Date: Thu, 8 Oct 2026 13:01:08 -0400 Subject: [PATCH] fix(ci): retry transient deploy gate requests --- .github/scripts/wait-for-daily-ci.cjs | 161 +++++++++++++++++--- .github/scripts/wait-for-daily-ci.test.cjs | 122 ++++++++++++++- .github/workflows/ci.yml | 15 ++ .github/workflows/deploy.yml | 9 +- README.md | 2 +- docs/daily-deploy-gate-repair-2026-10-08.md | 19 +++ 6 files changed, 301 insertions(+), 27 deletions(-) create mode 100644 docs/daily-deploy-gate-repair-2026-10-08.md diff --git a/.github/scripts/wait-for-daily-ci.cjs b/.github/scripts/wait-for-daily-ci.cjs index a021573..555c6b6 100644 --- a/.github/scripts/wait-for-daily-ci.cjs +++ b/.github/scripts/wait-for-daily-ci.cjs @@ -11,6 +11,22 @@ const DETROIT_DATE_FORMATTER = new Intl.DateTimeFormat('en-US', { const DEFAULT_TIMEOUT_MS = 20 * 60 * 1000; const DEFAULT_POLL_INTERVAL_MS = 20 * 1000; +const MAX_REQUEST_ATTEMPTS = 3; +const RETRYABLE_NETWORK_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'EPIPE', + 'ESOCKETTIMEDOUT', + 'ETIMEDOUT', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_SOCKET', +]); +const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429]); +const RETRY_BASE_DELAY_MS = 1000; +const MAX_RETRY_DELAY_MS = 10000; function detroitCalendarDate(timestamp) { if (typeof timestamp !== 'string') return null; @@ -37,6 +53,96 @@ function repositoryMatches(run, expectedFullName) { && repository.toLowerCase() === expectedFullName.toLowerCase(); } +function errorStatus(error) { + const status = error?.status ?? error?.response?.status; + return Number.isInteger(status) ? status : null; +} + +function errorCode(error) { + let current = error; + for (let depth = 0; current && depth < 4; depth += 1) { + if (typeof current.code === 'string') return current.code; + current = current.cause; + } + + // Octokit may preserve a native network error only in the wrapped message. + const message = String(error?.message ?? ''); + return [...RETRYABLE_NETWORK_CODES].find((candidate) => ( + new RegExp(`\\b${candidate}\\b`).test(message) + )) ?? null; +} + +function responseHeader(error, name) { + const headers = error?.response?.headers ?? error?.headers; + if (!headers) return undefined; + return headers[name.toLowerCase()] ?? headers.get?.(name.toLowerCase()); +} + +function isTransientRequestError(error) { + const status = errorStatus(error); + if ( + RETRYABLE_HTTP_STATUSES.has(status) + || (status !== null && status >= 500) + ) { + return true; + } + + if ( + status === 403 + && (responseHeader(error, 'retry-after') !== undefined + || responseHeader(error, 'x-ratelimit-remaining') === '0') + ) { + return true; + } + + const code = errorCode(error); + if (code && RETRYABLE_NETWORK_CODES.has(code)) return true; + + return false; +} + +function retryDelayMs(error, attempt) { + const retryAfter = responseHeader(error, 'retry-after'); + if (retryAfter !== undefined) { + const seconds = Number(retryAfter); + const requestedDelay = Number.isFinite(seconds) + ? seconds * 1000 + : Date.parse(retryAfter) - Date.now(); + if (Number.isFinite(requestedDelay)) { + return Math.min(MAX_RETRY_DELAY_MS, Math.max(0, requestedDelay)); + } + } + + const resetAt = responseHeader(error, 'x-ratelimit-reset'); + if (resetAt !== undefined && responseHeader(error, 'x-ratelimit-remaining') === '0') { + const requestedDelay = Number(resetAt) * 1000 - Date.now(); + if (Number.isFinite(requestedDelay)) { + return Math.min(MAX_RETRY_DELAY_MS, Math.max(0, requestedDelay)); + } + } + + return Math.min(MAX_RETRY_DELAY_MS, RETRY_BASE_DELAY_MS * (2 ** (attempt - 1))); +} + +async function requestWithRetry(method, request, { sleep, core }) { + for (let attempt = 1; ; attempt += 1) { + try { + return await request(); + } catch (error) { + if (attempt >= MAX_REQUEST_ATTEMPTS || !isTransientRequestError(error)) { + throw error; + } + + const delayMs = retryDelayMs(error, attempt); + const reason = errorStatus(error) ?? errorCode(error) ?? 'transient network error'; + core.info( + `GitHub Actions ${method} failed transiently (${reason}); retrying attempt ${attempt + 1}/${MAX_REQUEST_ATTEMPTS} in ${delayMs}ms.`, + ); + await sleep(delayMs); + } + } +} + function requireValidTiming(timeoutMs, pollIntervalMs) { if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { throw new TypeError('timeoutMs must be a finite non-negative number'); @@ -72,49 +178,62 @@ async function waitForDailyCI({ if (!owner || !repo || !expectedSha || !deploymentRunId) { throw new TypeError('GitHub repository, SHA, and workflow run context are required'); } - if (context.eventName !== 'schedule' || context.ref !== 'refs/heads/main') { - throw new Error('Daily CI wait is only valid for a scheduled run on main'); + const deploymentEvent = context.eventName; + if ( + !['schedule', 'workflow_dispatch'].includes(deploymentEvent) + || context.ref !== 'refs/heads/main' + ) { + throw new Error('CI wait is only valid for a scheduled or manually dispatched run on main'); } const startedAt = now(); - const deploymentResponse = await github.rest.actions.getWorkflowRun({ - owner, - repo, - run_id: deploymentRunId, - }); + const deploymentResponse = await requestWithRetry( + 'getWorkflowRun', + () => github.rest.actions.getWorkflowRun({ + owner, + repo, + run_id: deploymentRunId, + }), + { sleep, core }, + ); const deploymentRun = deploymentResponse?.data; if ( !deploymentRun - || deploymentRun.event !== 'schedule' + || deploymentRun.event !== deploymentEvent || deploymentRun.head_branch !== 'main' || deploymentRun.head_sha !== expectedSha || !repositoryMatches(deploymentRun, expectedFullName) ) { - throw new Error('Could not verify this scheduled deploy run belongs to main in the current repository'); + throw new Error(`Could not verify this ${deploymentEvent} deploy run belongs to main in the current repository`); } const scheduledDate = detroitCalendarDate(deploymentRun.created_at); if (!scheduledDate) { - throw new Error('Scheduled deploy run has no valid created_at timestamp'); + throw new Error('Deploy run has no valid created_at timestamp'); } + const ciRunKind = deploymentEvent === 'schedule' ? 'scheduled' : 'manually dispatched'; while (true) { - const response = await github.rest.actions.listWorkflowRuns({ - owner, - repo, - workflow_id: 'ci.yml', - event: 'schedule', - branch: 'main', - per_page: 100, - }); + const response = await requestWithRetry( + 'listWorkflowRuns', + () => github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: 'ci.yml', + event: deploymentEvent, + branch: 'main', + per_page: 100, + }), + { sleep, core }, + ); const workflowRuns = response?.data?.workflow_runs; if (!Array.isArray(workflowRuns)) { throw new Error('GitHub returned an invalid CI workflow-runs response'); } const matchingRuns = workflowRuns.filter((run) => ( - run?.event === 'schedule' + run?.event === deploymentEvent && run.head_branch === 'main' && run.head_sha === expectedSha && repositoryMatches(run, expectedFullName) @@ -134,7 +253,7 @@ async function waitForDailyCI({ run.status === 'completed' && run.conclusion === 'success' )); if (successfulRun) { - core.info(`Verified scheduled CI run ${successfulRun.id} for ${scheduledDate}.`); + core.info(`Verified ${ciRunKind} CI run ${successfulRun.id} for ${scheduledDate}.`); return { runId: successfulRun.id, scheduledDate, @@ -144,7 +263,7 @@ async function waitForDailyCI({ const remainingMs = timeoutMs - (now() - startedAt); if (remainingMs <= 0) { throw new Error( - `Timed out after ${timeoutMs}ms waiting for scheduled CI on ${scheduledDate}; refusing deployment`, + `Timed out after ${timeoutMs}ms waiting for ${ciRunKind} CI on ${scheduledDate}; refusing deployment`, ); } diff --git a/.github/scripts/wait-for-daily-ci.test.cjs b/.github/scripts/wait-for-daily-ci.test.cjs index 66145be..f132164 100644 --- a/.github/scripts/wait-for-daily-ci.test.cjs +++ b/.github/scripts/wait-for-daily-ci.test.cjs @@ -37,7 +37,12 @@ function makeWorkflowRun(overrides = {}) { }; } -function makeHarness(runResponses, { deployRun, context = makeContext() } = {}) { +function makeHarness(runResponses, { + deployRun, + context = makeContext(), + getErrors = [], + listErrors = [], +} = {}) { let elapsedMs = 0; const listCalls = []; const sleeps = []; @@ -47,12 +52,14 @@ function makeHarness(runResponses, { deployRun, context = makeContext() } = {}) actions: { getWorkflowRun: async (args) => { assert.deepEqual(args, { owner: OWNER, repo: REPO, run_id: context.runId }); + if (getErrors.length > 0) throw getErrors.shift(); return { - data: deployRun ?? makeWorkflowRun({ id: context.runId }), + data: deployRun ?? makeWorkflowRun({ id: context.runId, event: context.eventName }), }; }, listWorkflowRuns: async (args) => { listCalls.push(args); + if (listErrors.length > 0) throw listErrors.shift(); return { data: { workflow_runs: runResponses.shift() ?? [] }, }; @@ -105,6 +112,117 @@ test('waits for the same-day scheduled CI run for the deploy SHA and reports its }); }); +test('retries transient deployment and CI GET errors with bounded backoff', async () => { + const harness = makeHarness( + [[makeWorkflowRun()]], + { + getErrors: [new Error('RequestError getaddrinfo EAI_AGAIN api.github.com')], + listErrors: [ + Object.assign(new Error('temporary upstream failure'), { status: 502 }), + Object.assign(new Error('temporary upstream failure'), { status: 503 }), + ], + }, + ); + + const result = await waitForDailyCI(harness.args); + + assert.deepEqual(result, { runId: 41, scheduledDate: DETROIT_DAY }); + assert.deepEqual(harness.sleeps, [1000, 1000, 2000]); + assert.equal(harness.listCalls.length, 3); + assert.deepEqual(harness.infos.slice(0, 3), [ + 'GitHub Actions getWorkflowRun failed transiently (EAI_AGAIN); retrying attempt 2/3 in 1000ms.', + 'GitHub Actions listWorkflowRuns failed transiently (502); retrying attempt 2/3 in 1000ms.', + 'GitHub Actions listWorkflowRuns failed transiently (503); retrying attempt 3/3 in 2000ms.', + ]); + assert.equal(harness.infos.at(-1), `Verified scheduled CI run 41 for ${DETROIT_DAY}.`); +}); + +test('stops after the bounded retry count when a transient API error persists', async () => { + const persistentError = Object.assign(new Error('temporary upstream failure'), { status: 503 }); + const harness = makeHarness([], { listErrors: [persistentError, persistentError, persistentError] }); + + await assert.rejects(waitForDailyCI(harness.args), (error) => error === persistentError); + + assert.equal(harness.listCalls.length, 3); + assert.deepEqual(harness.sleeps, [1000, 2000]); + assert.deepEqual(harness.infos, [ + 'GitHub Actions listWorkflowRuns failed transiently (503); retrying attempt 2/3 in 1000ms.', + 'GitHub Actions listWorkflowRuns failed transiently (503); retrying attempt 3/3 in 2000ms.', + ]); +}); + +test('does not retry a permanent API error', async () => { + const permanentError = Object.assign(new Error('Not Found'), { status: 404 }); + const harness = makeHarness([], { listErrors: [permanentError] }); + + await assert.rejects(waitForDailyCI(harness.args), (error) => error === permanentError); + + assert.equal(harness.listCalls.length, 1); + assert.deepEqual(harness.sleeps, []); + assert.deepEqual(harness.infos, []); +}); + +test('validates a manual deployment against same-day manually dispatched CI for the same commit', async () => { + const manualContext = makeContext({ eventName: 'workflow_dispatch' }); + const harness = makeHarness( + [[makeWorkflowRun({ event: 'workflow_dispatch' })]], + { context: manualContext }, + ); + + const result = await waitForDailyCI(harness.args); + + assert.deepEqual(result, { runId: 41, scheduledDate: DETROIT_DAY }); + assert.equal(harness.listCalls[0].event, 'workflow_dispatch'); + assert.deepEqual(harness.infos, [`Verified manually dispatched CI run 41 for ${DETROIT_DAY}.`]); +}); + +test('rejects manual deployment event, branch, and CI-run identity mismatches', async () => { + const manualContext = makeContext({ eventName: 'workflow_dispatch' }); + const wrongDeployEvent = makeHarness([], { + context: manualContext, + deployRun: makeWorkflowRun({ event: 'schedule' }), + }); + + await assert.rejects( + waitForDailyCI(wrongDeployEvent.args), + /Could not verify this workflow_dispatch deploy run belongs to main in the current repository/, + ); + assert.equal(wrongDeployEvent.listCalls.length, 0); + + const wrongBranch = makeHarness([], { + context: makeContext({ eventName: 'workflow_dispatch', ref: 'refs/heads/release' }), + }); + await assert.rejects( + waitForDailyCI(wrongBranch.args), + /CI wait is only valid for a scheduled or manually dispatched run on main/, + ); + assert.equal(wrongBranch.listCalls.length, 0); + + const wrongCiEvent = makeHarness( + [[makeWorkflowRun({ event: 'schedule' })]], + { context: manualContext }, + ); + await assert.rejects( + waitForDailyCI({ ...wrongCiEvent.args, timeoutMs: 0 }), + /Timed out after 0ms waiting for manually dispatched CI on 2026-10-04; refusing deployment/, + ); + assert.equal(wrongCiEvent.listCalls[0].event, 'workflow_dispatch'); +}); + +test('rejects a deployment whose GitHub run metadata does not match the current main commit', async () => { + const harness = makeHarness([], { + deployRun: makeWorkflowRun({ head_sha: 'b'.repeat(40) }), + }); + + await assert.rejects( + waitForDailyCI(harness.args), + /Could not verify this schedule deploy run belongs to main in the current repository/, + ); + + assert.equal(harness.listCalls.length, 0); + assert.deepEqual(harness.sleeps, []); +}); + test('fails closed when the matching scheduled CI run completed unsuccessfully', async () => { const harness = makeHarness([ [makeWorkflowRun({ conclusion: 'failure' })], diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c91301..60f1498 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,21 @@ jobs: run: ci-storage-efficiency start - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - name: Test deploy CI identity and transient retry gate + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { spawnSync } = require('node:child_process'); + const result = spawnSync( + process.execPath, + ['--test', '.github/scripts/wait-for-daily-ci.test.cjs'], + { stdio: 'inherit' }, + ); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`Deploy CI gate tests failed with exit code ${result.status}.`); + } + - name: Set up authenticated remote BuildKit uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 with: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index da7d893..842e55a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,6 +4,7 @@ on: schedule: - cron: '5 4 * * *' timezone: America/Detroit + workflow_dispatch: {} workflow_run: workflows: ["CI Pipeline"] types: [completed] @@ -21,7 +22,9 @@ jobs: name: Build and Push Docker Image if: >- (github.event_name == 'schedule' && github.ref == 'refs/heads/main') || - (github.event.workflow_run.conclusion == 'success' && + (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository) @@ -44,8 +47,8 @@ jobs: ref: ${{ github.event.workflow_run.head_sha || github.sha }} fetch-depth: 1 - - name: Require successful daily CI for scheduled publication - if: github.event_name == 'schedule' + - name: Require successful matching CI before publication + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | diff --git a/README.md b/README.md index bd8541b..142e99d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Vite serves the development UI on port 3000. `npm run build` runs TypeScript che ## CI and containers -[CI](.github/workflows/ci.yml) and [deployment configuration](.github/workflows/deploy.yml) contain the automated build and image-publication rules. See the [maintenance audit](docs/maintenance-audit-2026-09-28.md) for the private-runner contract, validation evidence, and activation boundaries. +[CI](.github/workflows/ci.yml) and [deployment configuration](.github/workflows/deploy.yml) contain the automated build and image-publication rules. See the [maintenance audit](docs/maintenance-audit-2026-09-28.md) for the private-runner contract, validation evidence, and activation boundaries, and the [daily deploy gate repair](docs/daily-deploy-gate-repair-2026-10-08.md) for transient API retry and manual validation behavior. The [Dockerfile](Dockerfile) tests and builds with Java 25 and Node 26, then uses `jdeps` and `jlink` to create the Java runtime. The final Alpine image contains the application and linked runtime, runs as UID/GID 1001, and excludes Maven, Node, Java compiler tools, and source resources outside the JAR. CI boots that image, exercises PDF/content/UI routes, and scans its filesystem with Trivy before image publication. Any reported OS or Java-library vulnerability fails the scan; the scan uses current advisory data rather than guaranteeing future vulnerability absence. diff --git a/docs/daily-deploy-gate-repair-2026-10-08.md b/docs/daily-deploy-gate-repair-2026-10-08.md new file mode 100644 index 0000000..c515ade --- /dev/null +++ b/docs/daily-deploy-gate-repair-2026-10-08.md @@ -0,0 +1,19 @@ +# Daily deployment gate repair — October 8, 2026 + +## Failure + +Scheduled production deployment run [37801492760](https://github.com/javaDevJT/website/actions/runs/37801492760) at `4e71d679a5e8cd62d41ddb1725f259002bf5bb76` failed in “Require successful daily CI for scheduled publication.” GitHub's Actions API request raised a transient DNS error (`getaddrinfo EAI_AGAIN api.github.com`). The deployment correctly stopped before build and publication. The same-commit scheduled CI run [37801518895](https://github.com/javaDevJT/website/actions/runs/37801518895), attempt 2, subsequently passed; it did not turn the already failed deployment into a publication. + +## Repair + +The deployment gate now retries only transient GET failures, including temporary DNS/socket errors and HTTP 408, 425, 429, and 5xx responses. It makes at most three attempts, uses bounded exponential backoff, and propagates permanent errors immediately. It still rejects failed matching CI and requires the deploy run and CI run to share the same repository, `main` branch, full commit SHA, and Detroit calendar day. + +The deployment workflow also has a manual validation route on `main`. A manual deploy must find a successful same-day `workflow_dispatch` CI run for the identical repository, branch, and SHA. The workflow keeps both current-main checks and the vulnerability scan before publishing release tags. Manual validation does not emulate a schedule event. + +The CI workflow runs the helper's Node tests using its already pinned `actions/github-script` runtime. The test cases cover transient recovery and exhaustion, immediate failure for permanent errors, same-commit identity, failed CI, Detroit-day boundaries, and manual-event identity and branch checks. + +## Validation + +- Local focused test command: `node --test .github/scripts/wait-for-daily-ci.test.cjs` — 10 tests passed. +- Workflow checks and a fresh Actions run at the repaired source: recorded after remote validation. +- Publication or remaining blocker: recorded after the repaired-source deploy validation.