diff --git a/.github/workflows/deploy-cloudflare.yml b/.github/workflows/deploy-cloudflare.yml index fc2334c3..8358a6ff 100644 --- a/.github/workflows/deploy-cloudflare.yml +++ b/.github/workflows/deploy-cloudflare.yml @@ -17,7 +17,16 @@ permissions: actions: read concurrency: - group: deploy-cloudflare-${{ inputs.target || 'site' }} + # Every Test run that completes starts this workflow, including pull request + # runs and cancelled main runs. Their deploy job skips, but a run in the + # deploy group still replaces the run waiting there. On 2026-10-07 a pull + # request's run cancelled the queued deploy of #526, so main stayed + # undeployed. Only a successful main push can deploy, so every other run + # takes a group of its own. + group: >- + ${{ (github.event_name == 'workflow_run' && (github.event.workflow_run.event != 'push' || github.event.workflow_run.conclusion != 'success')) + && format('deploy-cloudflare-noop-{0}', github.run_id) + || format('deploy-cloudflare-{0}', inputs.target || 'site') }} # A deploy in flight must finish. This workflow applies D1 migrations and then # deploys the signer and public Worker, then runs smoke and rollback. Cancelling # between those steps leaves migrated data against the previous Worker. @@ -85,8 +94,10 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # Retries a transient Cloudflare API fault, such as code 10013. Any other + # failure stops the job at once. - name: Deploy Artifact signer - run: pnpm --filter @skilld/artifact-signer-worker deploy:production + run: pnpm production:deploy-signer env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/package.json b/package.json index 18132105..50de9b2c 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "seo:curate-supported": "npx tsx scripts/curate-supported-source.ts", "embeddings:parity": "tsx scripts/check-embedding-parity.ts", "production:deploy": "tsx scripts/deploy-production.ts", + "production:deploy-signer": "tsx scripts/deploy-artifact-signer.ts", "production:smoke": "tsx scripts/check-production-smoke.ts", "production:edge-cache": "tsx scripts/check-edge-cache.ts", "registry:convergence": "tsx scripts/check-registry-convergence.ts", diff --git a/scripts/deploy-artifact-signer.ts b/scripts/deploy-artifact-signer.ts new file mode 100644 index 00000000..8300b341 --- /dev/null +++ b/scripts/deploy-artifact-signer.ts @@ -0,0 +1,21 @@ +// The Artifact signer has no smoke of its own: the site deploy and the +// Artifact build smoke that follow it in the workflow exercise it. +import { resolve } from 'node:path' +import { withTransientRetry } from './lib/cloudflare-retry' +import { createCommand, wait } from './lib/spawn-command' + +const workerDirectory = resolve(process.cwd(), 'workers/artifact-signer') +const deploy = withTransientRetry( + createCommand(resolve(workerDirectory, 'node_modules/.bin/wrangler'), workerDirectory), + { wait, log: console.log }, +) + +deploy(['deploy', '--config', 'wrangler.jsonc']) + .then((result) => { + if (result._tag === 'failed') + process.exitCode = result.exitCode + }) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.stack : String(error)) + process.exitCode = 1 + }) diff --git a/scripts/deploy-production.ts b/scripts/deploy-production.ts index 41fa9dc5..6ceb1c20 100644 --- a/scripts/deploy-production.ts +++ b/scripts/deploy-production.ts @@ -1,41 +1,12 @@ -import type { ProductionCommand, ProductionCommandResult } from './lib/production-deploy' -import { spawn } from 'node:child_process' import { resolve } from 'node:path' +import { withTransientRetry } from './lib/cloudflare-retry' import { runProductionDeployment } from './lib/production-deploy' import { runProductionSmoke } from './lib/production-smoke' - -function createCommand(executable: string): ProductionCommand { - return args => new Promise((resolvePromise, rejectPromise) => { - console.log(`$ wrangler ${args.join(' ')}`) - const child = spawn(executable, args, { - cwd: process.cwd(), - env: process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }) - let stdout = '' - let stderr = '' - child.stdout.on('data', (chunk: Buffer) => { - const value = chunk.toString() - stdout += value - process.stdout.write(value) - }) - child.stderr.on('data', (chunk: Buffer) => { - const value = chunk.toString() - stderr += value - process.stderr.write(value) - }) - child.on('error', rejectPromise) - child.on('close', (exitCode) => { - resolvePromise(exitCode === 0 - ? { _tag: 'passed', stdout, stderr } - : { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 }) - }) - }) -} +import { createCommand, wait } from './lib/spawn-command' async function main(): Promise { const result = await runProductionDeployment({ - command: createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')), + command: withTransientRetry(createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')), { wait, log: console.log }), smoke: () => runProductionSmoke({ baseUrl: process.env.PRODUCTION_SMOKE_BASE_URL ?? 'https://skilld.dev', }), diff --git a/scripts/lib/cloudflare-retry.ts b/scripts/lib/cloudflare-retry.ts new file mode 100644 index 00000000..8d0044ef --- /dev/null +++ b/scripts/lib/cloudflare-retry.ts @@ -0,0 +1,45 @@ +import type { ProductionCommand, ProductionCommandResult } from './production-deploy' + +/** + * Cloudflare API faults that a later identical request can pass. On + * 2026-10-07 the Artifact signer deploy failed once with code 10013 ("An + * unknown error has occurred") and passed on a rerun. Authentication, + * permission, and configuration errors never match, so they fail at once. + */ +const API_FAILURE = /A request to the Cloudflare API \([^)]*\) failed/ +const API_FAULT = /\[code: 10013\]|An unknown error has occurred|\b50[0234]\b|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout/ +const NETWORK_FAULT = /\bfetch failed\b|ECONNRESET|ETIMEDOUT|socket hang up/ + +export function isTransientCloudflareFailure(output: string): boolean { + return (API_FAILURE.test(output) && API_FAULT.test(output)) || NETWORK_FAULT.test(output) +} + +export interface TransientRetryOptions { + wait: (milliseconds: number) => Promise + log: (message: string) => void + /** One delay per retry. Two retries cover a fault that clears within a minute. */ + delaysMs?: readonly number[] +} + +const DEFAULT_DELAYS_MS = [15_000, 45_000] as const + +/** + * Runs a Cloudflare command again after a transient API fault. Every other + * result returns unchanged, including the last failure when retries run out. + * A repeated `wrangler deploy` uploads the same script again, so a retry after + * a partial deploy leaves the same code active. + */ +export function withTransientRetry(command: ProductionCommand, options: TransientRetryOptions): ProductionCommand { + const delays = options.delaysMs ?? DEFAULT_DELAYS_MS + return async (args) => { + let result: ProductionCommandResult = await command(args) + for (const [index, delay] of delays.entries()) { + if (result._tag === 'passed' || !isTransientCloudflareFailure(`${result.stdout}\n${result.stderr}`)) + return result + options.log(`Cloudflare API fault on attempt ${index + 1} of ${delays.length + 1}. Retrying in ${delay / 1000}s.`) + await options.wait(delay) + result = await command(args) + } + return result + } +} diff --git a/scripts/lib/spawn-command.ts b/scripts/lib/spawn-command.ts new file mode 100644 index 00000000..07b9f2f6 --- /dev/null +++ b/scripts/lib/spawn-command.ts @@ -0,0 +1,37 @@ +import type { ProductionCommand, ProductionCommandResult } from './production-deploy' +import { spawn } from 'node:child_process' +import { basename } from 'node:path' + +/** Runs one executable, echoes its output, and keeps the output for the caller. */ +export function createCommand(executable: string, cwd: string = process.cwd()): ProductionCommand { + return args => new Promise((resolvePromise, rejectPromise) => { + console.log(`$ ${basename(executable)} ${args.join(' ')}`) + const child = spawn(executable, args, { + cwd, + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }) + let stdout = '' + let stderr = '' + child.stdout.on('data', (chunk: Buffer) => { + const value = chunk.toString() + stdout += value + process.stdout.write(value) + }) + child.stderr.on('data', (chunk: Buffer) => { + const value = chunk.toString() + stderr += value + process.stderr.write(value) + }) + child.on('error', rejectPromise) + child.on('close', (exitCode) => { + resolvePromise(exitCode === 0 + ? { _tag: 'passed', stdout, stderr } + : { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 }) + }) + }) +} + +export function wait(milliseconds: number): Promise { + return new Promise(resolvePromise => setTimeout(resolvePromise, milliseconds)) +} diff --git a/test/unit/cloudflare-retry.test.ts b/test/unit/cloudflare-retry.test.ts new file mode 100644 index 00000000..85fe3688 --- /dev/null +++ b/test/unit/cloudflare-retry.test.ts @@ -0,0 +1,65 @@ +import type { ProductionCommandResult } from '../../scripts/lib/production-deploy' +import { describe, expect, it, vi } from 'vitest' +import { isTransientCloudflareFailure, withTransientRetry } from '../../scripts/lib/cloudflare-retry' + +// The Deploy Artifact signer step on 2026-10-07, run 37562888913 attempt 1. +const UNKNOWN_ERROR = [ + '✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/skilld-artifact-signer/deployments) failed.', + ' An unknown error has occurred. If this error persists, please file a report in workers-sdk (https://github.com/cloudflare/workers-sdk/issues) or reach out to your account team. [code: 10013]', +].join('\n') +const AUTH_ERROR = '✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n Authentication error [code: 10000]' + +function failed(stderr: string): ProductionCommandResult { + return { _tag: 'failed', stdout: '', stderr, exitCode: 1 } +} + +const passed: ProductionCommandResult = { _tag: 'passed', stdout: 'Deployed', stderr: '' } + +describe('isTransientCloudflareFailure', () => { + it('treats the Cloudflare API unknown error as transient', () => { + expect(isTransientCloudflareFailure(UNKNOWN_ERROR)).toBe(true) + }) + + it('treats a gateway error from the Cloudflare API as transient', () => { + expect(isTransientCloudflareFailure('A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n 503 Service Unavailable')).toBe(true) + }) + + it('never retries an authentication or configuration error', () => { + expect(isTransientCloudflareFailure(AUTH_ERROR)).toBe(false) + expect(isTransientCloudflareFailure('✘ [ERROR] Missing entry-point to Worker script or to assets directory')).toBe(false) + }) +}) + +describe('withTransientRetry', () => { + it('retries a transient failure and returns the passing result', async () => { + const command = vi.fn() + .mockResolvedValueOnce(failed(UNKNOWN_ERROR)) + .mockResolvedValueOnce(passed) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {} }) + + await expect(retrying(['deploy'])).resolves.toEqual(passed) + expect(command).toHaveBeenCalledTimes(2) + expect(wait).toHaveBeenCalledOnce() + }) + + it('returns a non-transient failure without a retry', async () => { + const command = vi.fn().mockResolvedValue(failed(AUTH_ERROR)) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {} }) + + await expect(retrying(['deploy'])).resolves.toEqual(failed(AUTH_ERROR)) + expect(command).toHaveBeenCalledOnce() + expect(wait).not.toHaveBeenCalled() + }) + + it('stops after the last delay and returns the last failure', async () => { + const command = vi.fn().mockResolvedValue(failed(UNKNOWN_ERROR)) + const wait = vi.fn().mockResolvedValue(undefined) + const retrying = withTransientRetry(command, { wait, log: () => {}, delaysMs: [10, 20] }) + + await expect(retrying(['deploy'])).resolves.toEqual(failed(UNKNOWN_ERROR)) + expect(command).toHaveBeenCalledTimes(3) + expect(wait.mock.calls).toEqual([[10], [20]]) + }) +})