Skip to content

Commit bf558ec

Browse files
javaDevJTjavaDevJT
committed
ci: add manual deployment validation path
2 parents 75e732d + 96c571c commit bf558ec

6 files changed

Lines changed: 242 additions & 227 deletions

File tree

‎.github/scripts/wait-for-daily-ci.cjs‎

Lines changed: 57 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,18 @@ const DETROIT_DATE_FORMATTER = new Intl.DateTimeFormat('en-US', {
1111

1212
const DEFAULT_TIMEOUT_MS = 20 * 60 * 1000;
1313
const DEFAULT_POLL_INTERVAL_MS = 20 * 1000;
14-
const MAX_REQUEST_ATTEMPTS = 3;
15-
const RETRYABLE_NETWORK_CODES = new Set([
16-
'EAI_AGAIN',
17-
'ECONNREFUSED',
18-
'ECONNRESET',
19-
'EHOSTUNREACH',
20-
'ENETUNREACH',
21-
'EPIPE',
22-
'ESOCKETTIMEDOUT',
23-
'ETIMEDOUT',
24-
'UND_ERR_CONNECT_TIMEOUT',
25-
'UND_ERR_SOCKET',
26-
]);
27-
const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429]);
28-
const RETRY_BASE_DELAY_MS = 1000;
29-
const MAX_RETRY_DELAY_MS = 10000;
14+
const TEMPORARY_NETWORK_ERRORS = new Set(['EAI_AGAIN', 'ECONNRESET', 'ETIMEDOUT']);
15+
16+
function retryableReadFailure(error) {
17+
if ([500, 502, 503, 504].includes(error?.status)) return `HTTP ${error.status}`;
18+
19+
const visited = new Set();
20+
for (let cause = error; cause && !visited.has(cause); cause = cause.cause) {
21+
visited.add(cause);
22+
if (TEMPORARY_NETWORK_ERRORS.has(cause.code)) return cause.code;
23+
}
24+
return null;
25+
}
3026

3127
function detroitCalendarDate(timestamp) {
3228
if (typeof timestamp !== 'string') return null;
@@ -53,96 +49,6 @@ function repositoryMatches(run, expectedFullName) {
5349
&& repository.toLowerCase() === expectedFullName.toLowerCase();
5450
}
5551

56-
function errorStatus(error) {
57-
const status = error?.status ?? error?.response?.status;
58-
return Number.isInteger(status) ? status : null;
59-
}
60-
61-
function errorCode(error) {
62-
let current = error;
63-
for (let depth = 0; current && depth < 4; depth += 1) {
64-
if (typeof current.code === 'string') return current.code;
65-
current = current.cause;
66-
}
67-
68-
// Octokit may preserve a native network error only in the wrapped message.
69-
const message = String(error?.message ?? '');
70-
return [...RETRYABLE_NETWORK_CODES].find((candidate) => (
71-
new RegExp(`\\b${candidate}\\b`).test(message)
72-
)) ?? null;
73-
}
74-
75-
function responseHeader(error, name) {
76-
const headers = error?.response?.headers ?? error?.headers;
77-
if (!headers) return undefined;
78-
return headers[name.toLowerCase()] ?? headers.get?.(name.toLowerCase());
79-
}
80-
81-
function isTransientRequestError(error) {
82-
const status = errorStatus(error);
83-
if (
84-
RETRYABLE_HTTP_STATUSES.has(status)
85-
|| (status !== null && status >= 500)
86-
) {
87-
return true;
88-
}
89-
90-
if (
91-
status === 403
92-
&& (responseHeader(error, 'retry-after') !== undefined
93-
|| responseHeader(error, 'x-ratelimit-remaining') === '0')
94-
) {
95-
return true;
96-
}
97-
98-
const code = errorCode(error);
99-
if (code && RETRYABLE_NETWORK_CODES.has(code)) return true;
100-
101-
return false;
102-
}
103-
104-
function retryDelayMs(error, attempt) {
105-
const retryAfter = responseHeader(error, 'retry-after');
106-
if (retryAfter !== undefined) {
107-
const seconds = Number(retryAfter);
108-
const requestedDelay = Number.isFinite(seconds)
109-
? seconds * 1000
110-
: Date.parse(retryAfter) - Date.now();
111-
if (Number.isFinite(requestedDelay)) {
112-
return Math.min(MAX_RETRY_DELAY_MS, Math.max(0, requestedDelay));
113-
}
114-
}
115-
116-
const resetAt = responseHeader(error, 'x-ratelimit-reset');
117-
if (resetAt !== undefined && responseHeader(error, 'x-ratelimit-remaining') === '0') {
118-
const requestedDelay = Number(resetAt) * 1000 - Date.now();
119-
if (Number.isFinite(requestedDelay)) {
120-
return Math.min(MAX_RETRY_DELAY_MS, Math.max(0, requestedDelay));
121-
}
122-
}
123-
124-
return Math.min(MAX_RETRY_DELAY_MS, RETRY_BASE_DELAY_MS * (2 ** (attempt - 1)));
125-
}
126-
127-
async function requestWithRetry(method, request, { sleep, core }) {
128-
for (let attempt = 1; ; attempt += 1) {
129-
try {
130-
return await request();
131-
} catch (error) {
132-
if (attempt >= MAX_REQUEST_ATTEMPTS || !isTransientRequestError(error)) {
133-
throw error;
134-
}
135-
136-
const delayMs = retryDelayMs(error, attempt);
137-
const reason = errorStatus(error) ?? errorCode(error) ?? 'transient network error';
138-
core.info(
139-
`GitHub Actions ${method} failed transiently (${reason}); retrying attempt ${attempt + 1}/${MAX_REQUEST_ATTEMPTS} in ${delayMs}ms.`,
140-
);
141-
await sleep(delayMs);
142-
}
143-
}
144-
}
145-
14652
function requireValidTiming(timeoutMs, pollIntervalMs) {
14753
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
14854
throw new TypeError('timeoutMs must be a finite non-negative number');
@@ -174,28 +80,50 @@ async function waitForDailyCI({
17480
const expectedFullName = `${owner}/${repo}`;
17581
const expectedSha = context?.sha;
17682
const deploymentRunId = context?.runId;
83+
const deploymentEvent = context?.eventName;
17784

17885
if (!owner || !repo || !expectedSha || !deploymentRunId) {
17986
throw new TypeError('GitHub repository, SHA, and workflow run context are required');
18087
}
181-
const deploymentEvent = context.eventName;
182-
if (
183-
!['schedule', 'workflow_dispatch'].includes(deploymentEvent)
184-
|| context.ref !== 'refs/heads/main'
185-
) {
88+
if (!['schedule', 'workflow_dispatch'].includes(deploymentEvent) || context.ref !== 'refs/heads/main') {
18689
throw new Error('CI wait is only valid for a scheduled or manually dispatched run on main');
18790
}
18891

18992
const startedAt = now();
190-
const deploymentResponse = await requestWithRetry(
191-
'getWorkflowRun',
192-
() => github.rest.actions.getWorkflowRun({
193-
owner,
194-
repo,
195-
run_id: deploymentRunId,
196-
}),
197-
{ sleep, core },
198-
);
93+
// Retry read-only transport failures within the same deadline as CI polling.
94+
// Every recovered response still has to pass the repository/SHA/date gates.
95+
async function readGitHub(label, request) {
96+
let retrying = false;
97+
const timedOut = (cause) => new Error(
98+
`Timed out after ${timeoutMs}ms reading GitHub ${label}; refusing deployment`,
99+
{ cause },
100+
);
101+
while (true) {
102+
const elapsedMs = now() - startedAt;
103+
// A zero timeout still permits the original immediate lookup, but no retry.
104+
if (elapsedMs > timeoutMs || (retrying && elapsedMs >= timeoutMs)) throw timedOut();
105+
try {
106+
const response = await request();
107+
if (now() - startedAt > timeoutMs) throw timedOut();
108+
return response;
109+
} catch (error) {
110+
const reason = retryableReadFailure(error);
111+
if (!reason) throw error;
112+
113+
const remainingMs = timeoutMs - (now() - startedAt);
114+
if (remainingMs <= 0) throw timedOut(error);
115+
core.info(`Temporary GitHub ${label} read failure (${reason}); retrying within the CI wait deadline.`);
116+
await sleep(Math.min(pollIntervalMs, remainingMs));
117+
retrying = true;
118+
}
119+
}
120+
}
121+
122+
const deploymentResponse = await readGitHub('deployment run', () => github.rest.actions.getWorkflowRun({
123+
owner,
124+
repo,
125+
run_id: deploymentRunId,
126+
}));
199127
const deploymentRun = deploymentResponse?.data;
200128

201129
if (
@@ -215,18 +143,14 @@ async function waitForDailyCI({
215143
const ciRunKind = deploymentEvent === 'schedule' ? 'scheduled' : 'manually dispatched';
216144

217145
while (true) {
218-
const response = await requestWithRetry(
219-
'listWorkflowRuns',
220-
() => github.rest.actions.listWorkflowRuns({
221-
owner,
222-
repo,
223-
workflow_id: 'ci.yml',
224-
event: deploymentEvent,
225-
branch: 'main',
226-
per_page: 100,
227-
}),
228-
{ sleep, core },
229-
);
146+
const response = await readGitHub(`${ciRunKind} CI runs`, () => github.rest.actions.listWorkflowRuns({
147+
owner,
148+
repo,
149+
workflow_id: 'ci.yml',
150+
event: deploymentEvent,
151+
branch: 'main',
152+
per_page: 100,
153+
}));
230154
const workflowRuns = response?.data?.workflow_runs;
231155
if (!Array.isArray(workflowRuns)) {
232156
throw new Error('GitHub returned an invalid CI workflow-runs response');
@@ -245,7 +169,7 @@ async function waitForDailyCI({
245169
));
246170
if (unsuccessfulRun) {
247171
throw new Error(
248-
`Scheduled CI run ${unsuccessfulRun.id} completed with conclusion ${unsuccessfulRun.conclusion ?? 'unknown'}; refusing deployment`,
172+
`${ciRunKind[0].toUpperCase()}${ciRunKind.slice(1)} CI run ${unsuccessfulRun.id} completed with conclusion ${unsuccessfulRun.conclusion ?? 'unknown'}; refusing deployment`,
249173
);
250174
}
251175

0 commit comments

Comments
 (0)