From 8aaf894cfc41d6bd70a64e8b429d7e5f52bf69d3 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 13:56:43 +1100 Subject: [PATCH] fix(deploy): keep skipped runs from cancelling queued deploys Every completed Test run starts Deploy to Cloudflare, including pull request runs and cancelled main runs. Their deploy job skips, but they still joined the deploy concurrency group, and a newer pending run replaces the one waiting there. On 2026-10-07 a pull request's run cancelled the queued deploy of #526, so production stayed one commit behind main. Only a successful main push can deploy, so every other run now takes a group of its own. The same day the Artifact signer deploy failed once with Cloudflare API code 10013 and passed on a rerun. Both wrangler deploy paths now retry a transient Cloudflare API fault twice, after 15 and 45 seconds. Any other failure stops the job at once. --- .github/workflows/deploy-cloudflare.yml | 15 +++++- package.json | 1 + scripts/deploy-artifact-signer.ts | 21 ++++++++ scripts/deploy-production.ts | 35 ++----------- scripts/lib/cloudflare-retry.ts | 45 +++++++++++++++++ scripts/lib/spawn-command.ts | 37 ++++++++++++++ test/unit/cloudflare-retry.test.ts | 65 +++++++++++++++++++++++++ 7 files changed, 185 insertions(+), 34 deletions(-) create mode 100644 scripts/deploy-artifact-signer.ts create mode 100644 scripts/lib/cloudflare-retry.ts create mode 100644 scripts/lib/spawn-command.ts create mode 100644 test/unit/cloudflare-retry.test.ts diff --git a/.github/workflows/deploy-cloudflare.yml b/.github/workflows/deploy-cloudflare.yml index fc2334c3..8358a6ff 100644 --- a/.github/workflows/deploy-cloudflare.yml +++ b/.github/workflows/deploy-cloudflare.yml @@ -17,7 +17,16 @@ permissions: actions: read concurrency: - group: deploy-cloudflare-${{ inputs.target || 'site' }} + # Every Test run that completes starts this workflow, including pull request + # runs and cancelled main runs. Their deploy job skips, but a run in the + # deploy group still replaces the run waiting there. On 2026-10-07 a pull + # request's run cancelled the queued deploy of #526, so main stayed + # undeployed. Only a successful main push can deploy, so every other run + # takes a group of its own. + group: >- + ${{ (github.event_name == 'workflow_run' && (github.event.workflow_run.event != 'push' || github.event.workflow_run.conclusion != 'success')) + && format('deploy-cloudflare-noop-{0}', github.run_id) + || format('deploy-cloudflare-{0}', inputs.target || 'site') }} # A deploy in flight must finish. This workflow applies D1 migrations and then # deploys the signer and public Worker, then runs smoke and rollback. Cancelling # between those steps leaves migrated data against the previous Worker. @@ -85,8 +94,10 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # Retries a transient Cloudflare API fault, such as code 10013. Any other + # failure stops the job at once. - name: Deploy Artifact signer - run: pnpm --filter @skilld/artifact-signer-worker deploy:production + run: pnpm production:deploy-signer env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/package.json b/package.json index 18132105..50de9b2c 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "seo:curate-supported": "npx tsx scripts/curate-supported-source.ts", "embeddings:parity": "tsx scripts/check-embedding-parity.ts", "production:deploy": "tsx scripts/deploy-production.ts", + "production:deploy-signer": "tsx scripts/deploy-artifact-signer.ts", "production:smoke": "tsx scripts/check-production-smoke.ts", "production:edge-cache": "tsx scripts/check-edge-cache.ts", "registry:convergence": "tsx scripts/check-registry-convergence.ts", diff --git a/scripts/deploy-artifact-signer.ts b/scripts/deploy-artifact-signer.ts new file mode 100644 index 00000000..8300b341 --- /dev/null +++ b/scripts/deploy-artifact-signer.ts @@ -0,0 +1,21 @@ +// The Artifact signer has no smoke of its own: the site deploy and the +// Artifact build smoke that follow it in the workflow exercise it. +import { resolve } from 'node:path' +import { withTransientRetry } from './lib/cloudflare-retry' +import { createCommand, wait } from './lib/spawn-command' + +const workerDirectory = resolve(process.cwd(), 'workers/artifact-signer') +const deploy = withTransientRetry( + createCommand(resolve(workerDirectory, 'node_modules/.bin/wrangler'), workerDirectory), + { wait, log: console.log }, +) + +deploy(['deploy', '--config', 'wrangler.jsonc']) + .then((result) => { + if (result._tag === 'failed') + process.exitCode = result.exitCode + }) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : String(error)) + process.exitCode = 1 + }) diff --git a/scripts/deploy-production.ts b/scripts/deploy-production.ts index 41fa9dc5..6ceb1c20 100644 --- a/scripts/deploy-production.ts +++ b/scripts/deploy-production.ts @@ -1,41 +1,12 @@ -import type { ProductionCommand, ProductionCommandResult } from './lib/production-deploy' -import { spawn } from 'node:child_process' import { resolve } from 'node:path' +import { withTransientRetry } from './lib/cloudflare-retry' import { runProductionDeployment } from './lib/production-deploy' import { runProductionSmoke } from './lib/production-smoke' - -function createCommand(executable: string): ProductionCommand { - return args => new Promise((resolvePromise, rejectPromise) => { - console.log(`$ wrangler ${args.join(' ')}`) - const child = spawn(executable, args, { - cwd: process.cwd(), - env: process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }) - let stdout = '' - let stderr = '' - child.stdout.on('data', (chunk: Buffer) => { - const value = chunk.toString() - stdout += value - process.stdout.write(value) - }) - child.stderr.on('data', (chunk: Buffer) => { - const value = chunk.toString() - stderr += value - process.stderr.write(value) - }) - child.on('error', rejectPromise) - child.on('close', (exitCode) => { - resolvePromise(exitCode === 0 - ? { _tag: 'passed', stdout, stderr } - : { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 }) - }) - }) -} +import { createCommand, wait } from './lib/spawn-command' async function main(): Promise { const result = await runProductionDeployment({ - command: createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')), + command: withTransientRetry(createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')), { wait, log: console.log }), smoke: () => runProductionSmoke({ baseUrl: process.env.PRODUCTION_SMOKE_BASE_URL ?? 'https://skilld.dev', }), diff --git a/scripts/lib/cloudflare-retry.ts b/scripts/lib/cloudflare-retry.ts new file mode 100644 index 00000000..8d0044ef --- /dev/null +++ b/scripts/lib/cloudflare-retry.ts @@ -0,0 +1,45 @@ +import type { ProductionCommand, ProductionCommandResult } from './production-deploy' + +/** + * Cloudflare API faults that a later identical request can pass. On + * 2026-10-07 the Artifact signer deploy failed once with code 10013 ("An + * unknown error has occurred") and passed on a rerun. Authentication, + * permission, and configuration errors never match, so they fail at once. + */ +const API_FAILURE = /A request to the Cloudflare API \([^)]*\) failed/ +const API_FAULT = /\[code: 10013\]|An unknown error has occurred|\b50[0234]\b|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout/ +const NETWORK_FAULT = /\bfetch failed\b|ECONNRESET|ETIMEDOUT|socket hang up/ + +export function isTransientCloudflareFailure(output: string): boolean { + return (API_FAILURE.test(output) && API_FAULT.test(output)) || NETWORK_FAULT.test(output) +} + +export interface TransientRetryOptions { + wait: (milliseconds: number) => Promise + log: (message: string) => void + /** One delay per retry. Two retries cover a fault that clears within a minute. */ + delaysMs?: readonly number[] +} + +const DEFAULT_DELAYS_MS = [15_000, 45_000] as const + +/** + * Runs a Cloudflare command again after a transient API fault. Every other + * result returns unchanged, including the last failure when retries run out. + * A repeated `wrangler deploy` uploads the same script again, so a retry after + * a partial deploy leaves the same code active. + */ +export function withTransientRetry(command: ProductionCommand, options: TransientRetryOptions): ProductionCommand { + const delays = options.delaysMs ?? DEFAULT_DELAYS_MS + return async (args) => { + let result: ProductionCommandResult = await command(args) + for (const [index, delay] of delays.entries()) { + if (result._tag === 'passed' || !isTransientCloudflareFailure(`${result.stdout}\n${result.stderr}`)) + return result + options.log(`Cloudflare API fault on attempt ${index + 1} of ${delays.length + 1}. Retrying in ${delay / 1000}s.`) + await options.wait(delay) + result = await command(args) + } + return result + } +} diff --git a/scripts/lib/spawn-command.ts b/scripts/lib/spawn-command.ts new file mode 100644 index 00000000..07b9f2f6 --- /dev/null +++ b/scripts/lib/spawn-command.ts @@ -0,0 +1,37 @@ +import type { ProductionCommand, ProductionCommandResult } from './production-deploy' +import { spawn } from 'node:child_process' +import { basename } from 'node:path' + +/** Runs one executable, echoes its output, and keeps the output for the caller. */ +export function createCommand(executable: string, cwd: string = process.cwd()): ProductionCommand { + return args => new Promise((resolvePromise, rejectPromise) => { + console.log(`$ ${basename(executable)} ${args.join(' ')}`) + const child = spawn(executable, args, { + cwd, + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }) + let stdout = '' + let stderr = '' + child.stdout.on('data', (chunk: Buffer) => { + const value = chunk.toString() + stdout += value + process.stdout.write(value) + }) + child.stderr.on('data', (chunk: Buffer) => { + const value = chunk.toString() + stderr += value + process.stderr.write(value) + }) + child.on('error', rejectPromise) + child.on('close', (exitCode) => { + resolvePromise(exitCode === 0 + ? { _tag: 'passed', stdout, stderr } + : { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 }) + }) + }) +} + +export function wait(milliseconds: number): Promise { + return new Promise(resolvePromise => setTimeout(resolvePromise, milliseconds)) +} diff --git a/test/unit/cloudflare-retry.test.ts b/test/unit/cloudflare-retry.test.ts new file mode 100644 index 00000000..85fe3688 --- /dev/null +++ b/test/unit/cloudflare-retry.test.ts @@ -0,0 +1,65 @@ +import type { ProductionCommandResult } from '../../scripts/lib/production-deploy' +import { describe, expect, it, vi } from 'vitest' +import { isTransientCloudflareFailure, withTransientRetry } from '../../scripts/lib/cloudflare-retry' + +// The Deploy Artifact signer step on 2026-10-07, run 37562888913 attempt 1. +const UNKNOWN_ERROR = [ + '✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/skilld-artifact-signer/deployments) failed.', + ' An unknown error has occurred. If this error persists, please file a report in workers-sdk (https://github.com/cloudflare/workers-sdk/issues) or reach out to your account team. [code: 10013]', +].join('\n') +const AUTH_ERROR = '✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n Authentication error [code: 10000]' + +function failed(stderr: string): ProductionCommandResult { + return { _tag: 'failed', stdout: '', stderr, exitCode: 1 } +} + +const passed: ProductionCommandResult = { _tag: 'passed', stdout: 'Deployed', stderr: '' } + +describe('isTransientCloudflareFailure', () => { + it('treats the Cloudflare API unknown error as transient', () => { + expect(isTransientCloudflareFailure(UNKNOWN_ERROR)).toBe(true) + }) + + it('treats a gateway error from the Cloudflare API as transient', () => { + expect(isTransientCloudflareFailure('A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n 503 Service Unavailable')).toBe(true) + }) + + it('never retries an authentication or configuration error', () => { + expect(isTransientCloudflareFailure(AUTH_ERROR)).toBe(false) + expect(isTransientCloudflareFailure('✘ [ERROR] Missing entry-point to Worker script or to assets directory')).toBe(false) + }) +}) + +describe('withTransientRetry', () => { + it('retries a transient failure and returns the passing result', async () => { + const command = vi.fn() + .mockResolvedValueOnce(failed(UNKNOWN_ERROR)) + .mockResolvedValueOnce(passed) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {} }) + + await expect(retrying(['deploy'])).resolves.toEqual(passed) + expect(command).toHaveBeenCalledTimes(2) + expect(wait).toHaveBeenCalledOnce() + }) + + it('returns a non-transient failure without a retry', async () => { + const command = vi.fn().mockResolvedValue(failed(AUTH_ERROR)) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {} }) + + await expect(retrying(['deploy'])).resolves.toEqual(failed(AUTH_ERROR)) + expect(command).toHaveBeenCalledOnce() + expect(wait).not.toHaveBeenCalled() + }) + + it('stops after the last delay and returns the last failure', async () => { + const command = vi.fn().mockResolvedValue(failed(UNKNOWN_ERROR)) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {}, delaysMs: [10, 20] }) + + await expect(retrying(['deploy'])).resolves.toEqual(failed(UNKNOWN_ERROR)) + expect(command).toHaveBeenCalledTimes(3) + expect(wait.mock.calls).toEqual([[10], [20]]) + }) +})