Skip to content

Commit 8aaf894

Browse files
committed
fix(deploy): keep skipped runs from cancelling queued deploys
Every completed Test run starts Deploy to Cloudflare, including pull request runs and cancelled main runs. Their deploy job skips, but they still joined the deploy concurrency group, and a newer pending run replaces the one waiting there. On 2026-10-07 a pull request's run cancelled the queued deploy of #526, so production stayed one commit behind main. Only a successful main push can deploy, so every other run now takes a group of its own. The same day the Artifact signer deploy failed once with Cloudflare API code 10013 and passed on a rerun. Both wrangler deploy paths now retry a transient Cloudflare API fault twice, after 15 and 45 seconds. Any other failure stops the job at once.
1 parent 66f22c7 commit 8aaf894

7 files changed

Lines changed: 185 additions & 34 deletions

File tree

‎.github/workflows/deploy-cloudflare.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,16 @@ permissions:
1717
actions: read
1818

1919
concurrency:
20-
group: deploy-cloudflare-${{ inputs.target || 'site' }}
20+
# Every Test run that completes starts this workflow, including pull request
21+
# runs and cancelled main runs. Their deploy job skips, but a run in the
22+
# deploy group still replaces the run waiting there. On 2026-10-07 a pull
23+
# request's run cancelled the queued deploy of #526, so main stayed
24+
# undeployed. Only a successful main push can deploy, so every other run
25+
# takes a group of its own.
26+
group: >-
27+
${{ (github.event_name == 'workflow_run' && (github.event.workflow_run.event != 'push' || github.event.workflow_run.conclusion != 'success'))
28+
&& format('deploy-cloudflare-noop-{0}', github.run_id)
29+
|| format('deploy-cloudflare-{0}', inputs.target || 'site') }}
2130
# A deploy in flight must finish. This workflow applies D1 migrations and then
2231
# deploys the signer and public Worker, then runs smoke and rollback. Cancelling
2332
# between those steps leaves migrated data against the previous Worker.
@@ -85,8 +94,10 @@ jobs:
8594
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
8695
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
8796

97+
# Retries a transient Cloudflare API fault, such as code 10013. Any other
98+
# failure stops the job at once.
8899
- name: Deploy Artifact signer
89-
run: pnpm --filter @skilld/artifact-signer-worker deploy:production
100+
run: pnpm production:deploy-signer
90101
env:
91102
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
92103
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
"seo:curate-supported": "npx tsx scripts/curate-supported-source.ts",
3333
"embeddings:parity": "tsx scripts/check-embedding-parity.ts",
3434
"production:deploy": "tsx scripts/deploy-production.ts",
35+
"production:deploy-signer": "tsx scripts/deploy-artifact-signer.ts",
3536
"production:smoke": "tsx scripts/check-production-smoke.ts",
3637
"production:edge-cache": "tsx scripts/check-edge-cache.ts",
3738
"registry:convergence": "tsx scripts/check-registry-convergence.ts",

‎scripts/deploy-artifact-signer.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// The Artifact signer has no smoke of its own: the site deploy and the
2+
// Artifact build smoke that follow it in the workflow exercise it.
3+
import { resolve } from 'node:path'
4+
import { withTransientRetry } from './lib/cloudflare-retry'
5+
import { createCommand, wait } from './lib/spawn-command'
6+
7+
const workerDirectory = resolve(process.cwd(), 'workers/artifact-signer')
8+
const deploy = withTransientRetry(
9+
createCommand(resolve(workerDirectory, 'node_modules/.bin/wrangler'), workerDirectory),
10+
{ wait, log: console.log },
11+
)
12+
13+
deploy(['deploy', '--config', 'wrangler.jsonc'])
14+
.then((result) => {
15+
if (result._tag === 'failed')
16+
process.exitCode = result.exitCode
17+
})
18+
.catch((error: unknown) => {
19+
console.error(error instanceof Error ? error.stack : String(error))
20+
process.exitCode = 1
21+
})

‎scripts/deploy-production.ts‎

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,12 @@
1-
import type { ProductionCommand, ProductionCommandResult } from './lib/production-deploy'
2-
import { spawn } from 'node:child_process'
31
import { resolve } from 'node:path'
2+
import { withTransientRetry } from './lib/cloudflare-retry'
43
import { runProductionDeployment } from './lib/production-deploy'
54
import { runProductionSmoke } from './lib/production-smoke'
6-
7-
function createCommand(executable: string): ProductionCommand {
8-
return args => new Promise<ProductionCommandResult>((resolvePromise, rejectPromise) => {
9-
console.log(`$ wrangler ${args.join(' ')}`)
10-
const child = spawn(executable, args, {
11-
cwd: process.cwd(),
12-
env: process.env,
13-
stdio: ['ignore', 'pipe', 'pipe'],
14-
})
15-
let stdout = ''
16-
let stderr = ''
17-
child.stdout.on('data', (chunk: Buffer) => {
18-
const value = chunk.toString()
19-
stdout += value
20-
process.stdout.write(value)
21-
})
22-
child.stderr.on('data', (chunk: Buffer) => {
23-
const value = chunk.toString()
24-
stderr += value
25-
process.stderr.write(value)
26-
})
27-
child.on('error', rejectPromise)
28-
child.on('close', (exitCode) => {
29-
resolvePromise(exitCode === 0
30-
? { _tag: 'passed', stdout, stderr }
31-
: { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 })
32-
})
33-
})
34-
}
5+
import { createCommand, wait } from './lib/spawn-command'
356

367
async function main(): Promise<void> {
378
const result = await runProductionDeployment({
38-
command: createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')),
9+
command: withTransientRetry(createCommand(resolve(process.cwd(), 'node_modules/.bin/wrangler')), { wait, log: console.log }),
3910
smoke: () => runProductionSmoke({
4011
baseUrl: process.env.PRODUCTION_SMOKE_BASE_URL ?? 'https://skilld.dev',
4112
}),

‎scripts/lib/cloudflare-retry.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import type { ProductionCommand, ProductionCommandResult } from './production-deploy'
2+
3+
/**
4+
* Cloudflare API faults that a later identical request can pass. On
5+
* 2026-10-07 the Artifact signer deploy failed once with code 10013 ("An
6+
* unknown error has occurred") and passed on a rerun. Authentication,
7+
* permission, and configuration errors never match, so they fail at once.
8+
*/
9+
const API_FAILURE = /A request to the Cloudflare API \([^)]*\) failed/
10+
const API_FAULT = /\[code: 10013\]|An unknown error has occurred|\b50[0234]\b|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout/
11+
const NETWORK_FAULT = /\bfetch failed\b|ECONNRESET|ETIMEDOUT|socket hang up/
12+
13+
export function isTransientCloudflareFailure(output: string): boolean {
14+
return (API_FAILURE.test(output) && API_FAULT.test(output)) || NETWORK_FAULT.test(output)
15+
}
16+
17+
export interface TransientRetryOptions {
18+
wait: (milliseconds: number) => Promise<void>
19+
log: (message: string) => void
20+
/** One delay per retry. Two retries cover a fault that clears within a minute. */
21+
delaysMs?: readonly number[]
22+
}
23+
24+
const DEFAULT_DELAYS_MS = [15_000, 45_000] as const
25+
26+
/**
27+
* Runs a Cloudflare command again after a transient API fault. Every other
28+
* result returns unchanged, including the last failure when retries run out.
29+
* A repeated `wrangler deploy` uploads the same script again, so a retry after
30+
* a partial deploy leaves the same code active.
31+
*/
32+
export function withTransientRetry(command: ProductionCommand, options: TransientRetryOptions): ProductionCommand {
33+
const delays = options.delaysMs ?? DEFAULT_DELAYS_MS
34+
return async (args) => {
35+
let result: ProductionCommandResult = await command(args)
36+
for (const [index, delay] of delays.entries()) {
37+
if (result._tag === 'passed' || !isTransientCloudflareFailure(`${result.stdout}\n${result.stderr}`))
38+
return result
39+
options.log(`Cloudflare API fault on attempt ${index + 1} of ${delays.length + 1}. Retrying in ${delay / 1000}s.`)
40+
await options.wait(delay)
41+
result = await command(args)
42+
}
43+
return result
44+
}
45+
}

‎scripts/lib/spawn-command.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import type { ProductionCommand, ProductionCommandResult } from './production-deploy'
2+
import { spawn } from 'node:child_process'
3+
import { basename } from 'node:path'
4+
5+
/** Runs one executable, echoes its output, and keeps the output for the caller. */
6+
export function createCommand(executable: string, cwd: string = process.cwd()): ProductionCommand {
7+
return args => new Promise<ProductionCommandResult>((resolvePromise, rejectPromise) => {
8+
console.log(`$ ${basename(executable)} ${args.join(' ')}`)
9+
const child = spawn(executable, args, {
10+
cwd,
11+
env: process.env,
12+
stdio: ['ignore', 'pipe', 'pipe'],
13+
})
14+
let stdout = ''
15+
let stderr = ''
16+
child.stdout.on('data', (chunk: Buffer) => {
17+
const value = chunk.toString()
18+
stdout += value
19+
process.stdout.write(value)
20+
})
21+
child.stderr.on('data', (chunk: Buffer) => {
22+
const value = chunk.toString()
23+
stderr += value
24+
process.stderr.write(value)
25+
})
26+
child.on('error', rejectPromise)
27+
child.on('close', (exitCode) => {
28+
resolvePromise(exitCode === 0
29+
? { _tag: 'passed', stdout, stderr }
30+
: { _tag: 'failed', stdout, stderr, exitCode: exitCode ?? 1 })
31+
})
32+
})
33+
}
34+
35+
export function wait(milliseconds: number): Promise<void> {
36+
return new Promise(resolvePromise => setTimeout(resolvePromise, milliseconds))
37+
}

‎test/unit/cloudflare-retry.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import type { ProductionCommandResult } from '../../scripts/lib/production-deploy'
2+
import { describe, expect, it, vi } from 'vitest'
3+
import { isTransientCloudflareFailure, withTransientRetry } from '../../scripts/lib/cloudflare-retry'
4+
5+
// The Deploy Artifact signer step on 2026-10-07, run 37562888913 attempt 1.
6+
const UNKNOWN_ERROR = [
7+
'✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/skilld-artifact-signer/deployments) failed.',
8+
' An unknown error has occurred. If this error persists, please file a report in workers-sdk (https://github.com/cloudflare/workers-sdk/issues) or reach out to your account team. [code: 10013]',
9+
].join('\n')
10+
const AUTH_ERROR = '✘ [ERROR] A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n Authentication error [code: 10000]'
11+
12+
function failed(stderr: string): ProductionCommandResult {
13+
return { _tag: 'failed', stdout: '', stderr, exitCode: 1 }
14+
}
15+
16+
const passed: ProductionCommandResult = { _tag: 'passed', stdout: 'Deployed', stderr: '' }
17+
18+
describe('isTransientCloudflareFailure', () => {
19+
it('treats the Cloudflare API unknown error as transient', () => {
20+
expect(isTransientCloudflareFailure(UNKNOWN_ERROR)).toBe(true)
21+
})
22+
23+
it('treats a gateway error from the Cloudflare API as transient', () => {
24+
expect(isTransientCloudflareFailure('A request to the Cloudflare API (/accounts/abc/workers/scripts/x) failed.\n 503 Service Unavailable')).toBe(true)
25+
})
26+
27+
it('never retries an authentication or configuration error', () => {
28+
expect(isTransientCloudflareFailure(AUTH_ERROR)).toBe(false)
29+
expect(isTransientCloudflareFailure('✘ [ERROR] Missing entry-point to Worker script or to assets directory')).toBe(false)
30+
})
31+
})
32+
33+
describe('withTransientRetry', () => {
34+
it('retries a transient failure and returns the passing result', async () => {
35+
const command = vi.fn()
36+
.mockResolvedValueOnce(failed(UNKNOWN_ERROR))
37+
.mockResolvedValueOnce(passed)
38+
const wait = vi.fn().mockResolvedValue(undefined)
39+
const retrying = withTransientRetry(command, { wait, log: () => {} })
40+
41+
await expect(retrying(['deploy'])).resolves.toEqual(passed)
42+
expect(command).toHaveBeenCalledTimes(2)
43+
expect(wait).toHaveBeenCalledOnce()
44+
})
45+
46+
it('returns a non-transient failure without a retry', async () => {
47+
const command = vi.fn().mockResolvedValue(failed(AUTH_ERROR))
48+
const wait = vi.fn().mockResolvedValue(undefined)
49+
const retrying = withTransientRetry(command, { wait, log: () => {} })
50+
51+
await expect(retrying(['deploy'])).resolves.toEqual(failed(AUTH_ERROR))
52+
expect(command).toHaveBeenCalledOnce()
53+
expect(wait).not.toHaveBeenCalled()
54+
})
55+
56+
it('stops after the last delay and returns the last failure', async () => {
57+
const command = vi.fn().mockResolvedValue(failed(UNKNOWN_ERROR))
58+
const wait = vi.fn().mockResolvedValue(undefined)
59+
const retrying = withTransientRetry(command, { wait, log: () => {}, delaysMs: [10, 20] })
60+
61+
await expect(retrying(['deploy'])).resolves.toEqual(failed(UNKNOWN_ERROR))
62+
expect(command).toHaveBeenCalledTimes(3)
63+
expect(wait.mock.calls).toEqual([[10], [20]])
64+
})
65+
})

0 commit comments

Comments
 (0)