From b5ede20fedc93bfeb06e86eedb1ef255b6e89de4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 23:11:34 +0000 Subject: [PATCH 01/10] feat: retry idempotent requests on 429/502/503/504 in the shared fetch layer Add one retry policy (Retry-After as seconds or HTTP-date, capped; bounded attempts; exponential backoff with jitter) and apply it to fetch (fetchWithUnifiedRemint), the billing SDK transport, and the workspace axios client. Only GET/HEAD/OPTIONS are retried unless a caller opts in. Retries run beneath the 401 re-mint, so each keeps its own budget. Keep the HTTP status and code on the AuthStoreError thrown by fetchBalance. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/platform/auth/unified/remintRetry.test.ts | 82 +++++++++ src/platform/auth/unified/remintRetry.ts | 5 +- .../workspace/api/workspaceApi.test.ts | 2 + .../workspace/api/workspaceApiAuth.ts | 6 +- .../workspace/billing/sdk/billingSdkStore.ts | 2 + src/stores/authStore.test.ts | 31 ++++ src/stores/authStore.ts | 6 +- src/utils/axiosRetryableStatus.test.ts | 112 +++++++++++ src/utils/axiosRetryableStatus.ts | 55 ++++++ src/utils/fetchWithRetry.test.ts | 172 +++++++++++++++++ src/utils/fetchWithRetry.ts | 51 +++++ src/utils/retryPolicy.test.ts | 174 ++++++++++++++++++ src/utils/retryPolicy.ts | 102 ++++++++++ 13 files changed, 795 insertions(+), 5 deletions(-) create mode 100644 src/utils/axiosRetryableStatus.test.ts create mode 100644 src/utils/axiosRetryableStatus.ts create mode 100644 src/utils/fetchWithRetry.test.ts create mode 100644 src/utils/fetchWithRetry.ts create mode 100644 src/utils/retryPolicy.test.ts create mode 100644 src/utils/retryPolicy.ts diff --git a/src/platform/auth/unified/remintRetry.test.ts b/src/platform/auth/unified/remintRetry.test.ts index 8a0b0c44763..21e6a4a53a2 100644 --- a/src/platform/auth/unified/remintRetry.test.ts +++ b/src/platform/auth/unified/remintRetry.test.ts @@ -14,6 +14,7 @@ import { attachUnifiedRemintInterceptor, fetchWithUnifiedRemint } from '@/platform/auth/unified/remintRetry' +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' vi.mock(import('@/platform/telemetry')) @@ -119,6 +120,69 @@ describe('fetchWithUnifiedRemint', () => { expect(useTelemetry()?.trackUnifiedAuthRetry).not.toHaveBeenCalled() }) + describe('with retryable server failures', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + const unavailable = () => + new Response(null, { status: 503, headers: { 'Retry-After': '1' } }) + + it('retries a 503 without re-minting', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(ok) + + const pending = fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true + ) + await vi.advanceTimersByTimeAsync(1_000) + + expect(await pending).toBe(ok) + expect(fetch).toHaveBeenCalledTimes(2) + expect(useWorkspaceAuthStore().remintUnifiedOnce).not.toHaveBeenCalled() + }) + + it('gives the re-minted request its own retry budget', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unauthorized) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(ok) + vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( + 'tokenB' + ) + + const pending = fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true + ) + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toBe(ok) + expect(fetch).toHaveBeenCalledTimes(6) + expect(useWorkspaceAuthStore().remintUnifiedOnce).toHaveBeenCalledTimes(1) + }) + + it('does not retry a POST that got a 503', async () => { + vi.mocked(fetch).mockResolvedValueOnce(unavailable()) + + const result = await fetchWithUnifiedRemint( + 'https://cloud/x', + { method: 'POST', body: '{}' }, + true + ) + + expect(result.status).toBe(503) + expect(fetch).toHaveBeenCalledTimes(1) + }) + }) + it('does not retry a non-401 response', async () => { const serverError = { status: 500 } as Response vi.mocked(fetch).mockResolvedValueOnce(serverError) @@ -512,6 +576,24 @@ describe('attachUnifiedRemintInterceptor', () => { }) }) + it('keeps the 503 retries and the 401 re-mint on separate budgets', async () => { + const adapter = makeAdapter([503, 503, 401, 503, 200]) + const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + attachRetryableStatusRetry(client, { sleep: async () => {} }) + attachUnifiedRemintInterceptor(client) + vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( + 'tokenB' + ) + + const res = await client.get('https://cloud/x', { + headers: { Authorization: 'Bearer tokenA' as const } + }) + + expect(res.status).toBe(200) + expect(adapter).toHaveBeenCalledTimes(5) + expect(useWorkspaceAuthStore().remintUnifiedOnce).toHaveBeenCalledTimes(1) + }) + it('does not re-mint when the flag is OFF (AC3)', async () => { vi.mocked(useFeatureFlags().flags).unifiedCloudAuthEnabled = false const { client, adapter } = makeClient([401]) diff --git a/src/platform/auth/unified/remintRetry.ts b/src/platform/auth/unified/remintRetry.ts index d3075fe7327..dae1e526ecc 100644 --- a/src/platform/auth/unified/remintRetry.ts +++ b/src/platform/auth/unified/remintRetry.ts @@ -18,6 +18,7 @@ import type { UnifiedAuthRetryFailureReason, UnifiedAuthRetryMetadata } from '@/platform/telemetry/types' +import { fetchWithRetry } from '@/utils/fetchWithRetry' let cachedUnifiedFlags: | { readonly unifiedCloudAuthEnabled: boolean } @@ -152,7 +153,7 @@ export async function fetchWithUnifiedRemint( shouldRetryOn401 && input instanceof Request && input.body !== null ? input.clone() : input - const response = await fetch(input, init) + const response = await fetchWithRetry(input, init) if (response.status === 403 && isCloud) { await presentSsoRequired(() => presentForResponse(response)) } @@ -188,7 +189,7 @@ export async function fetchWithUnifiedRemint( ? { ...init, headers, signal: retrySignalLifecycle.createSignal() } : { ...init, headers } try { - const retryResponse = await fetch(retryInput, retryInit) + const retryResponse = await fetchWithRetry(retryInput, retryInit) trackRetryResponse('fetch', retryResponse.status) return retryResponse } catch (error) { diff --git a/src/platform/workspace/api/workspaceApi.test.ts b/src/platform/workspace/api/workspaceApi.test.ts index 0f8e5b3292d..d1eb0d55842 100644 --- a/src/platform/workspace/api/workspaceApi.test.ts +++ b/src/platform/workspace/api/workspaceApi.test.ts @@ -15,6 +15,7 @@ const { mockAxiosInstance } = vi.hoisted(() => ({ put: vi.fn(), patch: vi.fn(), delete: vi.fn(), + defaults: {}, interceptors: { response: { use: vi.fn() } } } })) @@ -22,6 +23,7 @@ const { mockAxiosInstance } = vi.hoisted(() => ({ vi.mock(import('axios'), () => ({ default: { create: vi.fn(() => mockAxiosInstance), + getAdapter: vi.fn(() => vi.fn()), isAxiosError: vi.fn((err: unknown) => { return ( err !== null && diff --git a/src/platform/workspace/api/workspaceApiAuth.ts b/src/platform/workspace/api/workspaceApiAuth.ts index 47b32134883..73a19f7ab22 100644 --- a/src/platform/workspace/api/workspaceApiAuth.ts +++ b/src/platform/workspace/api/workspaceApiAuth.ts @@ -1,12 +1,16 @@ import { attachUnifiedRemintInterceptor } from '@/platform/auth/unified/remintRetry' import { setWorkspaceApiAuth } from '@/platform/workspace/api/workspaceApi' import { useAuthStore } from '@/stores/authStore' +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' export function installWorkspaceApiAuth(): void { setWorkspaceApiAuth({ getWorkspaceAuthHeader: () => useAuthStore().getWorkspaceAuthHeaderOrThrow(), getFirebaseAuthHeader: () => useAuthStore().getFirebaseAuthHeaderOrThrow(), - attachRetryInterceptor: attachUnifiedRemintInterceptor + attachRetryInterceptor: (client) => { + attachRetryableStatusRetry(client) + attachUnifiedRemintInterceptor(client) + } }) } diff --git a/src/platform/workspace/billing/sdk/billingSdkStore.ts b/src/platform/workspace/billing/sdk/billingSdkStore.ts index c08ab43ab40..b83f7a513e9 100644 --- a/src/platform/workspace/billing/sdk/billingSdkStore.ts +++ b/src/platform/workspace/billing/sdk/billingSdkStore.ts @@ -61,6 +61,7 @@ import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspace import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' import { getCheckoutJourneyPaymentIntentSource } from '@/platform/workspace/utils/checkoutJourney' import { useDialogStore } from '@/stores/dialogStore' +import { fetchWithRetry } from '@/utils/fetchWithRetry' import { projectBillingCapabilities } from './billingCapabilitiesView' import { projectBillingPlans } from './billingPlansView' @@ -144,6 +145,7 @@ export const useBillingSdkStore = defineStore('billingSdk', () => { const sdk = createBillingSdk({ ...sessionPorts(), resolveUrl: workspaceApiUrl, + fetchImpl: fetchWithRetry, pointerStorage: sessionStorage, hostedDestination: () => flags.hostedBillingDestination, onTelemetry: reportTelemetry diff --git a/src/stores/authStore.test.ts b/src/stores/authStore.test.ts index d34d7e03387..7eb2d8c5d39 100644 --- a/src/stores/authStore.test.ts +++ b/src/stores/authStore.test.ts @@ -339,6 +339,37 @@ describe('useAuthStore', () => { }) }) }) + + it('keeps the status and code on the thrown error', async () => { + respondToFetch(BALANCE_URL, () => + Response.json( + { error: 'SERVICE_UNAVAILABLE', message: 'try again' }, + { status: 503, headers: { 'Retry-After': '3600' } } + ) + ) + + await expect(store.fetchBalance()).rejects.toMatchObject({ + name: 'AuthStoreError', + status: 503, + code: 'SERVICE_UNAVAILABLE' + }) + }) + + it('returns the balance when a 503 clears on retry', async () => { + vi.useFakeTimers() + let calls = 0 + respondToFetch(BALANCE_URL, () => + ++calls === 1 + ? errorResponse(503, 'Balance temporarily unavailable') + : balanceResponse() + ) + + const pending = store.fetchBalance() + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual({ balance: 0 }) + expect(calls).toBe(2) + }) }) describe('password update', () => { diff --git a/src/stores/authStore.ts b/src/stores/authStore.ts index 9c14c3d3c3c..8a2e17de394 100644 --- a/src/stores/authStore.ts +++ b/src/stores/authStore.ts @@ -705,14 +705,16 @@ export const useAuthStore = defineStore('auth', () => { // Customer not found is expected for new users return null } - const { message } = await parseErrorResponse(response) + const { message, code } = await parseErrorResponse(response) if (!requestIsCurrent()) { return null } throw new AuthStoreError( t('toastMessages.failedToFetchBalance', { error: message - }) + }), + response.status, + code ) } diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts new file mode 100644 index 00000000000..7651dcf04c3 --- /dev/null +++ b/src/utils/axiosRetryableStatus.test.ts @@ -0,0 +1,112 @@ +import type { AxiosAdapter, AxiosResponse } from 'axios' +import axios, { AxiosError } from 'axios' +import { describe, expect, it, vi } from 'vitest' + +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' + +interface ScriptedAnswer { + readonly status: number + readonly headers?: Record +} + +function makeClient(answers: ScriptedAnswer[]) { + let call = 0 + const adapter = vi.fn(async (config) => { + const { status, headers = {} } = + answers[Math.min(call++, answers.length - 1)] + const response: AxiosResponse = { + data: {}, + status, + statusText: String(status), + headers, + config + } + if (status < 400) return response + throw new AxiosError( + `Request failed with status code ${status}`, + AxiosError.ERR_BAD_RESPONSE, + config, + null, + response + ) + }) + const sleep = vi.fn<(ms: number) => Promise>(async () => {}) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { sleep, random: () => 1 }) + return { client, adapter, sleep } +} + +describe('attachRetryableStatusRetry', () => { + it('returns a 200 after one 503', async () => { + const { client, adapter } = makeClient([{ status: 503 }, { status: 200 }]) + + const response = await client.get('https://api/x') + + expect(response.status).toBe(200) + expect(adapter).toHaveBeenCalledTimes(2) + }) + + it('rejects with the final 503 once the attempts are spent', async () => { + const { client, adapter } = makeClient([{ status: 503 }]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it.for([401, 404, 500])('does not retry a %i', async (status) => { + const { client, adapter } = makeClient([{ status }]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status } + }) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it.for(['post', 'put', 'patch', 'delete'] as const)( + 'does not retry a %s that got a 503', + async (method) => { + const { client, adapter } = makeClient([{ status: 503 }]) + + await expect( + client.request({ url: 'https://api/x', method }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + } + ) + + it('waits the Retry-After the server sent', async () => { + const { client, sleep } = makeClient([ + { status: 429, headers: { 'retry-after': '2' } }, + { status: 200 } + ]) + + await client.get('https://api/x') + + expect(sleep).toHaveBeenCalledExactlyOnceWith(2_000, undefined) + }) + + it('rejects without waiting when Retry-After is past the cap', async () => { + const { client, adapter, sleep } = makeClient([ + { status: 503, headers: { 'retry-after': '3600' } } + ]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + + it('rethrows an error that has no response', async () => { + const networkError = new AxiosError('Network Error', AxiosError.ERR_NETWORK) + const adapter = vi.fn().mockRejectedValue(networkError) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { sleep: async () => {} }) + + await expect(client.get('https://api/x')).rejects.toBe(networkError) + expect(adapter).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts new file mode 100644 index 00000000000..20da698eb9d --- /dev/null +++ b/src/utils/axiosRetryableStatus.ts @@ -0,0 +1,55 @@ +import type { AxiosInstance, AxiosResponse } from 'axios' +import axios from 'axios' + +import type { RetryPolicy } from '@/utils/retryPolicy' +import { retryDelayMs, sleep as sleepFor } from '@/utils/retryPolicy' + +export interface AxiosRetryableStatusOptions { + readonly policy?: RetryPolicy + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise + readonly random?: () => number +} + +function retryAfterHeader(response: AxiosResponse): string | undefined { + const value = response.headers['retry-after'] + return typeof value === 'string' || typeof value === 'number' + ? String(value) + : undefined +} + +/** + * Wraps the client's adapter so a retryable status is retried below the + * interceptor chain. Each request that reaches the adapter, including the one + * an interceptor re-issues after a 401 re-mint, gets its own attempt budget. + */ +export function attachRetryableStatusRetry( + client: AxiosInstance, + options: AxiosRetryableStatusOptions = {} +): void { + const { policy, sleep = sleepFor, random } = options + const adapter = axios.getAdapter(client.defaults.adapter) + + client.defaults.adapter = async (config) => { + const signal = + config.signal instanceof AbortSignal ? config.signal : undefined + for (let attempt = 1; ; attempt++) { + try { + return await adapter(config) + } catch (error) { + const response = axios.isAxiosError(error) ? error.response : undefined + const delay = response + ? retryDelayMs({ + method: config.method ?? 'GET', + status: response.status, + readRetryAfter: () => retryAfterHeader(response), + attempt, + policy, + random + }) + : undefined + if (delay === undefined) throw error + await sleep(delay, signal) + } + } + } +} diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts new file mode 100644 index 00000000000..d4395290046 --- /dev/null +++ b/src/utils/fetchWithRetry.test.ts @@ -0,0 +1,172 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { fetchWithRetry } from '@/utils/fetchWithRetry' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' + +function respond(status: number, headers: Record = {}) { + return new Response(status === 200 ? '{}' : null, { status, headers }) +} + +function scriptedFetch(...responses: Response[]) { + const queue = [...responses] + return vi.fn(async () => { + const next = queue.shift() + if (!next) throw new Error('fetch called more times than scripted') + return next + }) +} + +describe('fetchWithRetry', () => { + const sleep = vi.fn<(ms: number) => Promise>(async () => {}) + const random = () => 1 + + it('returns a 200 after one 503', async () => { + const final = respond(200) + const fetchImpl = scriptedFetch(respond(503), final) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result).toBe(final) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('returns the final 503 once the attempts are spent', async () => { + const last = respond(503) + const fetchImpl = scriptedFetch(respond(503), respond(503), last) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result).toBe(last) + expect(fetchImpl).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it.for([400, 401, 404, 500])('does not retry a %i', async (status) => { + const fetchImpl = scriptedFetch(respond(status)) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(status) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it.for(['POST', 'PUT', 'PATCH', 'DELETE'])( + 'does not retry a %s that got a 503', + async (method) => { + const fetchImpl = scriptedFetch(respond(503)) + + const result = await fetchWithRetry( + 'https://api/x', + { method }, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + } + ) + + it('does not retry a Request whose method is not idempotent', async () => { + const fetchImpl = scriptedFetch(respond(503)) + + await fetchWithRetry( + new Request('https://api/x', { method: 'POST', body: '{}' }), + {}, + { fetchImpl, sleep, random } + ) + + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it('retries a POST only when the policy opts in', async () => { + const fetchImpl = scriptedFetch(respond(503), respond(200)) + + const result = await fetchWithRetry( + 'https://api/x', + { method: 'POST' }, + { + fetchImpl, + sleep, + random, + policy: { ...DEFAULT_RETRY_POLICY, retryNonIdempotent: true } + } + ) + + expect(result.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('waits the Retry-After the server sent', async () => { + const fetchImpl = scriptedFetch( + respond(503, { 'Retry-After': '4' }), + respond(200) + ) + + await fetchWithRetry('https://api/x', {}, { fetchImpl, sleep, random }) + + expect(sleep).toHaveBeenCalledExactlyOnceWith(4_000, undefined) + }) + + it('returns the 503 without waiting when Retry-After is past the cap', async () => { + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '3600' })) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + + describe('with the real clock source', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + it('holds the retry back until Retry-After has elapsed', async () => { + const fetchImpl = scriptedFetch( + respond(503, { 'Retry-After': '3' }), + respond(200) + ) + + const pending = fetchWithRetry('https://api/x', {}, { fetchImpl }) + + await vi.advanceTimersByTimeAsync(2_999) + expect(fetchImpl).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect((await pending).status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('stops waiting when the caller aborts', async () => { + const controller = new AbortController() + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '5' })) + + const pending = fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + const outcome = pending.catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort('cancelled') + + expect(await outcome).toBe('cancelled') + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts new file mode 100644 index 00000000000..49179644040 --- /dev/null +++ b/src/utils/fetchWithRetry.ts @@ -0,0 +1,51 @@ +import type { RetryPolicy } from '@/utils/retryPolicy' +import { retryDelayMs, sleep as sleepFor } from '@/utils/retryPolicy' + +export interface RetryingFetchOptions { + readonly policy?: RetryPolicy + readonly fetchImpl?: typeof fetch + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise + readonly random?: () => number +} + +function requestMethod(input: RequestInfo | URL, init: RequestInit): string { + return init.method ?? (input instanceof Request ? input.method : 'GET') +} + +function requestSignal( + input: RequestInfo | URL, + init: RequestInit +): AbortSignal | undefined { + return init.signal ?? (input instanceof Request ? input.signal : undefined) +} + +/** + * `fetch` that re-issues idempotent requests the server answered with a + * retryable status, honoring `Retry-After`. Returns the last response once + * attempts are spent; never retries a non-idempotent method unless the policy + * opts in. + */ +export async function fetchWithRetry( + input: RequestInfo | URL, + init: RequestInit = {}, + options: RetryingFetchOptions = {} +): Promise { + const { policy, fetchImpl = fetch, sleep = sleepFor, random } = options + const method = requestMethod(input, init) + const signal = requestSignal(input, init) + + for (let attempt = 1; ; attempt++) { + const response = await fetchImpl(input, init) + const delay = retryDelayMs({ + method, + status: response.status, + readRetryAfter: () => response.headers.get('Retry-After'), + attempt, + policy, + random + }) + if (delay === undefined) return response + await response.body?.cancel() + await sleep(delay, signal) + } +} diff --git a/src/utils/retryPolicy.test.ts b/src/utils/retryPolicy.test.ts new file mode 100644 index 00000000000..a5f01641e0b --- /dev/null +++ b/src/utils/retryPolicy.test.ts @@ -0,0 +1,174 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { RetryPolicy } from '@/utils/retryPolicy' +import { + DEFAULT_RETRY_POLICY, + parseRetryAfterMs, + retryDelayMs, + sleep +} from '@/utils/retryPolicy' + +const NOW = Date.parse('2026-10-18T12:00:00Z') + +describe('parseRetryAfterMs', () => { + it.for<[string, string, number]>([ + ['delta-seconds', '5', 5_000], + ['zero seconds', '0', 0], + ['padded delta-seconds', ' 7 ', 7_000], + ['an HTTP-date in the future', 'Sun, 18 Oct 2026 12:00:30 GMT', 30_000], + ['an HTTP-date in the past', 'Sun, 18 Oct 2026 11:59:00 GMT', 0] + ])('reads %s', ([_name, header, expected]) => { + expect(parseRetryAfterMs(header, NOW)).toBe(expected) + }) + + it.for<[string, string | null | undefined]>([ + ['absent', undefined], + ['null', null], + ['empty', ''], + ['garbage', 'soon'], + ['negative', '-5'], + ['fractional', '1.5'] + ])('ignores a header that is %s', ([_name, header]) => { + expect(parseRetryAfterMs(header, NOW)).toBeUndefined() + }) +}) + +describe('retryDelayMs', () => { + const decide = ( + overrides: Partial[0]> = {} + ) => + retryDelayMs({ + method: 'GET', + status: 503, + readRetryAfter: () => null, + attempt: 1, + random: () => 1, + now: () => NOW, + ...overrides + }) + + it.for([429, 502, 503, 504])('retries status %i', (status) => { + expect(decide({ status })).toBeDefined() + }) + + it.for([200, 400, 401, 403, 404, 409, 500, 501])( + 'does not retry status %i', + (status) => { + expect(decide({ status })).toBeUndefined() + } + ) + + it.for(['GET', 'get', 'HEAD', 'OPTIONS'])('retries %s', (method) => { + expect(decide({ method })).toBeDefined() + }) + + it.for(['POST', 'PUT', 'PATCH', 'DELETE'])( + 'does not retry %s by default', + (method) => { + expect(decide({ method })).toBeUndefined() + } + ) + + it('retries a non-idempotent method when the policy opts in', () => { + const policy: RetryPolicy = { + ...DEFAULT_RETRY_POLICY, + retryNonIdempotent: true + } + expect(decide({ method: 'POST', policy })).toBeDefined() + }) + + it('stops once the attempts are spent', () => { + const { maxAttempts } = DEFAULT_RETRY_POLICY + expect(decide({ attempt: maxAttempts - 1 })).toBeDefined() + expect(decide({ attempt: maxAttempts })).toBeUndefined() + }) + + it('backs off exponentially up to the cap', () => { + const policy: RetryPolicy = { + maxAttempts: 10, + baseDelayMs: 100, + maxDelayMs: 500, + retryNonIdempotent: false + } + const delays = [1, 2, 3, 4, 5].map((attempt) => + decide({ attempt, policy, random: () => 1 }) + ) + expect(delays).toEqual([100, 200, 400, 500, 500]) + }) + + it.for([0, 0.25, 0.5, 0.99, 1])( + 'keeps jitter within half to full backoff (random=%f)', + (random) => { + const delay = decide({ attempt: 2, random: () => random }) + const backoff = DEFAULT_RETRY_POLICY.baseDelayMs * 2 + expect(delay).toBeGreaterThanOrEqual(backoff / 2) + expect(delay).toBeLessThanOrEqual(backoff) + } + ) + + it('waits at least as long as Retry-After asks', () => { + expect(decide({ readRetryAfter: () => '4' })).toBe(4_000) + }) + + it('reads an HTTP-date Retry-After against the clock', () => { + expect( + decide({ readRetryAfter: () => 'Sun, 18 Oct 2026 12:00:03 GMT' }) + ).toBe(3_000) + }) + + it('gives up when Retry-After exceeds the cap', () => { + const tooLong = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000 + 1) + expect(decide({ readRetryAfter: () => tooLong })).toBeUndefined() + }) + + it('accepts Retry-After exactly at the cap', () => { + const atCap = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000) + expect(decide({ readRetryAfter: () => atCap })).toBe( + DEFAULT_RETRY_POLICY.maxDelayMs + ) + }) + + it('does not read the header for a response it will not retry', () => { + const readRetryAfter = vi.fn(() => '1') + decide({ status: 500, readRetryAfter }) + decide({ method: 'POST', readRetryAfter }) + expect(readRetryAfter).not.toHaveBeenCalled() + }) +}) + +describe('sleep', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + it('resolves after the delay', async () => { + let settled = false + const pending = sleep(1_000).then(() => { + settled = true + }) + + await vi.advanceTimersByTimeAsync(999) + expect(settled).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await pending + expect(settled).toBe(true) + }) + + it('rejects with the abort reason when aborted mid-wait', async () => { + const controller = new AbortController() + const pending = sleep(1_000, controller.signal) + const outcome = pending.catch((reason: unknown) => reason) + + controller.abort('stop') + + expect(await outcome).toBe('stop') + expect(vi.getTimerCount()).toBe(0) + }) + + it('rejects immediately when already aborted', async () => { + const controller = new AbortController() + controller.abort('stop') + + await expect(sleep(1_000, controller.signal)).rejects.toBe('stop') + }) +}) diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts new file mode 100644 index 00000000000..fa6a1fcd526 --- /dev/null +++ b/src/utils/retryPolicy.ts @@ -0,0 +1,102 @@ +export interface RetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number + readonly retryNonIdempotent: boolean +} + +export const DEFAULT_RETRY_POLICY: RetryPolicy = { + maxAttempts: 3, + baseDelayMs: 500, + maxDelayMs: 10_000, + retryNonIdempotent: false +} + +const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]) +const IDEMPOTENT_METHODS: ReadonlySet = new Set([ + 'GET', + 'HEAD', + 'OPTIONS' +]) + +function isRetryableStatus(status: number): boolean { + return RETRYABLE_STATUSES.has(status) +} + +function isIdempotentMethod(method: string): boolean { + return IDEMPOTENT_METHODS.has(method.toUpperCase()) +} + +/** Milliseconds a `Retry-After` header asks for (delta-seconds or HTTP-date). */ +export function parseRetryAfterMs( + value: string | null | undefined, + now: number = Date.now() +): number | undefined { + const trimmed = value?.trim() + if (!trimmed) return undefined + if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000 + if (!/[a-z]/i.test(trimmed)) return undefined + const at = Date.parse(trimmed) + return Number.isNaN(at) ? undefined : Math.max(0, at - now) +} + +interface RetryDecisionInput { + readonly method: string + readonly status: number + readonly readRetryAfter: () => string | null | undefined + readonly attempt: number + readonly policy?: RetryPolicy + readonly random?: () => number + readonly now?: () => number +} + +/** + * How long to wait before the next attempt, or `undefined` to stop retrying: + * the request is not retryable, attempts are spent, or the server asked to + * wait longer than the policy allows. + */ +export function retryDelayMs({ + method, + status, + readRetryAfter, + attempt, + policy = DEFAULT_RETRY_POLICY, + random = Math.random, + now = Date.now +}: RetryDecisionInput): number | undefined { + if (attempt >= policy.maxAttempts) return undefined + if (!isRetryableStatus(status)) return undefined + if (!policy.retryNonIdempotent && !isIdempotentMethod(method)) { + return undefined + } + + const retryAfterMs = parseRetryAfterMs(readRetryAfter(), now()) + if (retryAfterMs !== undefined && retryAfterMs > policy.maxDelayMs) { + return undefined + } + + const ceiling = Math.min( + policy.maxDelayMs, + policy.baseDelayMs * 2 ** (attempt - 1) + ) + const jittered = ceiling / 2 + (random() * ceiling) / 2 + return Math.max(jittered, retryAfterMs ?? 0) +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason) + return + } + const onAbort = () => { + clearTimeout(timer) + reject(signal?.reason) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} From d767c7a1a775fb84275da5e2e3b707a8b1c58f32 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 23:47:58 +0000 Subject: [PATCH 02/10] test: cover an unparseable balance failure body in fetchBalance Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/stores/authStore.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/stores/authStore.test.ts b/src/stores/authStore.test.ts index f2f44bc1271..799e8aa67cf 100644 --- a/src/stores/authStore.test.ts +++ b/src/stores/authStore.test.ts @@ -354,6 +354,22 @@ describe('useAuthStore', () => { }) }) + it('reports an unknown code when the failure body is not JSON', async () => { + respondToFetch( + BALANCE_URL, + () => + new Response('upstream connect error', { + status: 503, + headers: { 'Retry-After': '3600' } + }) + ) + + await expect(store.fetchBalance()).rejects.toMatchObject({ + status: 503, + code: 'UNKNOWN_ERROR' + }) + }) + it('returns the balance when a 503 clears on retry', async () => { vi.useFakeTimers() let calls = 0 From 72dd15ce588e13ccfee805f53701859c2ef88355 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 20:41:22 +0000 Subject: [PATCH 03/10] fix: retry per-request axios adapters and keep retries inside the request timeout The axios retry wrapped only the client's default adapter, so a request that selects its own adapter (the web-session workspace path) surfaced the first 503. Wrap whichever adapter each request selects, once. The request timeout restarted on every attempt and did not bound the backoff wait. Treat timeout as one budget: each attempt gets the time remaining, and a wait that would not fit stops retrying and surfaces the last response. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- .../workspace/api/workspaceApi.test.ts | 4 +- .../api/workspaceApi.webSessionRetry.test.ts | 70 +++++++++++++++++++ src/utils/axiosRetryableStatus.test.ts | 60 +++++++++++++++- src/utils/axiosRetryableStatus.ts | 56 +++++++++++---- 4 files changed, 170 insertions(+), 20 deletions(-) create mode 100644 src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts diff --git a/src/platform/workspace/api/workspaceApi.test.ts b/src/platform/workspace/api/workspaceApi.test.ts index 5e73a40795d..e3b8ee8e827 100644 --- a/src/platform/workspace/api/workspaceApi.test.ts +++ b/src/platform/workspace/api/workspaceApi.test.ts @@ -15,15 +15,13 @@ const { mockAxiosInstance } = vi.hoisted(() => ({ put: vi.fn(), patch: vi.fn(), delete: vi.fn(), - defaults: {}, - interceptors: { response: { use: vi.fn() } } + interceptors: { request: { use: vi.fn() }, response: { use: vi.fn() } } } })) vi.mock(import('axios'), () => ({ default: { create: vi.fn(() => mockAxiosInstance), - getAdapter: vi.fn(() => vi.fn()), isAxiosError: vi.fn((err: unknown) => { return ( err !== null && diff --git a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts new file mode 100644 index 00000000000..eab6c649588 --- /dev/null +++ b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts @@ -0,0 +1,70 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' + +import type { + WebSessionRequests, + WebSessionRequestScope +} from '@/platform/auth/session/webSessionFetch' +import { provideWebSessionRequests } from '@/platform/auth/session/webSessionFetch' +import { stubFirebaseAuthHarness } from '@/utils/__tests__/stubAccountIdentityPort' + +import { workspaceApi } from './workspaceApi' +import { installWorkspaceApiAuth } from './workspaceApiAuth' + +vi.mock(import('@/i18n')) + +vi.mock(import('./workspaceApiUrl'), () => ({ + workspaceApiUrl: (path: string) => `/api${path}` +})) + +vi.mock(import('firebase/auth'), { spy: true }) + +beforeEach(() => { + stubFirebaseAuthHarness() +}) + +installWorkspaceApiAuth() + +function provideSessionSending(...answers: Response[]) { + const send = vi.fn(async () => { + const answer = answers.shift() + if (!answer) throw new Error('send called more times than scripted') + return answer + }) + const release = provideWebSessionRequests( + fromPartial({ + scope: async () => fromPartial({}), + send + }) + ) + onTestFinished(release) + return send +} + +const unavailable = (headers?: Record) => + new Response('{"message":"unavailable"}', { status: 503, headers }) + +describe('workspaceApi on the web session', () => { + it('recovers from a 503 on the session adapter', async () => { + vi.useFakeTimers() + const send = provideSessionSending( + unavailable(), + new Response('{"workspaces":[]}', { status: 200 }) + ) + + const pending = workspaceApi.list() + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual({ workspaces: [] }) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('does not retry a write that got a 503', async () => { + const send = provideSessionSending(unavailable()) + + await expect(workspaceApi.create({ name: 'Team' })).rejects.toMatchObject({ + status: 503 + }) + expect(send).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts index 7651dcf04c3..05558aaf30d 100644 --- a/src/utils/axiosRetryableStatus.test.ts +++ b/src/utils/axiosRetryableStatus.test.ts @@ -10,9 +10,9 @@ interface ScriptedAnswer { readonly headers?: Record } -function makeClient(answers: ScriptedAnswer[]) { +function scriptedAdapter(answers: ScriptedAnswer[]) { let call = 0 - const adapter = vi.fn(async (config) => { + return vi.fn(async (config) => { const { status, headers = {} } = answers[Math.min(call++, answers.length - 1)] const response: AxiosResponse = { @@ -31,6 +31,10 @@ function makeClient(answers: ScriptedAnswer[]) { response ) }) +} + +function makeClient(answers: ScriptedAnswer[]) { + const adapter = scriptedAdapter(answers) const sleep = vi.fn<(ms: number) => Promise>(async () => {}) const client = axios.create({ adapter }) attachRetryableStatusRetry(client, { sleep, random: () => 1 }) @@ -109,4 +113,56 @@ describe('attachRetryableStatusRetry', () => { await expect(client.get('https://api/x')).rejects.toBe(networkError) expect(adapter).toHaveBeenCalledTimes(1) }) + it('retries through an adapter the request selects for itself', async () => { + const { client, adapter: defaultAdapter } = makeClient([{ status: 200 }]) + const requestAdapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + + const response = await client.get('https://api/x', { + adapter: requestAdapter + }) + + expect(response.status).toBe(200) + expect(requestAdapter).toHaveBeenCalledTimes(2) + expect(defaultAdapter).not.toHaveBeenCalled() + }) + + describe('with a request timeout', () => { + function makeTimedClient(answers: ScriptedAnswer[]) { + const adapter = scriptedAdapter(answers) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { random: () => 1 }) + return { client, adapter } + } + + it('stops instead of waiting past the timeout', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '1' } }, + { status: 200 } + ]) + + await expect( + client.get('https://api/x', { timeout: 100 }) + ).rejects.toMatchObject({ response: { status: 503 } }) + + expect(adapter).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }) + + it('retries when the wait fits inside the timeout and shrinks the next attempt', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '1' } }, + { status: 200 } + ]) + + const pending = client.get('https://api/x', { timeout: 5_000 }) + await vi.advanceTimersByTimeAsync(1_000) + + expect((await pending).status).toBe(200) + expect(adapter.mock.calls.map(([config]) => config.timeout)).toEqual([ + 5_000, 4_000 + ]) + }) + }) }) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index 20da698eb9d..e310e7d1fba 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -1,4 +1,4 @@ -import type { AxiosInstance, AxiosResponse } from 'axios' +import type { AxiosAdapter, AxiosInstance, AxiosResponse } from 'axios' import axios from 'axios' import type { RetryPolicy } from '@/utils/retryPolicy' @@ -17,24 +17,24 @@ function retryAfterHeader(response: AxiosResponse): string | undefined { : undefined } -/** - * Wraps the client's adapter so a retryable status is retried below the - * interceptor chain. Each request that reaches the adapter, including the one - * an interceptor re-issues after a 401 re-mint, gets its own attempt budget. - */ -export function attachRetryableStatusRetry( - client: AxiosInstance, - options: AxiosRetryableStatusOptions = {} -): void { - const { policy, sleep = sleepFor, random } = options - const adapter = axios.getAdapter(client.defaults.adapter) - - client.defaults.adapter = async (config) => { +function retryingAdapter( + adapter: AxiosAdapter, + { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions +): AxiosAdapter { + return async (config) => { const signal = config.signal instanceof AbortSignal ? config.signal : undefined + const budgetMs = config.timeout || undefined + const startedAt = Date.now() + const remainingMs = () => + budgetMs === undefined ? undefined : budgetMs - (Date.now() - startedAt) + for (let attempt = 1; ; attempt++) { + const remaining = remainingMs() try { - return await adapter(config) + return await adapter( + remaining === undefined ? config : { ...config, timeout: remaining } + ) } catch (error) { const response = axios.isAxiosError(error) ? error.response : undefined const delay = response @@ -47,9 +47,35 @@ export function attachRetryableStatusRetry( random }) : undefined + const budgetLeft = remainingMs() if (delay === undefined) throw error + if (budgetLeft !== undefined && delay >= budgetLeft) throw error await sleep(delay, signal) } } } } + +/** + * Retries a retryable status around whichever adapter each request selects, + * so a per-request adapter is covered too. The retry sits below the + * interceptor chain: each request that reaches the adapter, including the one + * an interceptor re-issues after a 401 re-mint, gets its own attempt budget, + * and `timeout` bounds the attempts and the waits between them together. + */ +export function attachRetryableStatusRetry( + client: AxiosInstance, + options: AxiosRetryableStatusOptions = {} +): void { + const retrying = new WeakSet() + + client.interceptors.request.use((config) => { + const selected = config.adapter ?? client.defaults.adapter + if (typeof selected === 'function' && retrying.has(selected)) return config + + const adapter = retryingAdapter(axios.getAdapter(selected), options) + retrying.add(adapter) + config.adapter = adapter + return config + }) +} From bbad486f472afb1afdb7fddfb71209a6d85e8412 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 20:45:42 +0000 Subject: [PATCH 04/10] refactor: split the axios retrying adapter into smaller functions Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/utils/axiosRetryableStatus.ts | 53 ++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index e310e7d1fba..1e7d243efea 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -1,4 +1,9 @@ -import type { AxiosAdapter, AxiosInstance, AxiosResponse } from 'axios' +import type { + AxiosAdapter, + AxiosInstance, + AxiosResponse, + InternalAxiosRequestConfig +} from 'axios' import axios from 'axios' import type { RetryPolicy } from '@/utils/retryPolicy' @@ -17,17 +22,36 @@ function retryAfterHeader(response: AxiosResponse): string | undefined { : undefined } +function timeBudget(timeoutMs: number | undefined) { + const startedAt = Date.now() + return () => (timeoutMs ? timeoutMs - (Date.now() - startedAt) : undefined) +} + function retryingAdapter( adapter: AxiosAdapter, { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions ): AxiosAdapter { + function waitBeforeRetry( + error: unknown, + config: InternalAxiosRequestConfig, + attempt: number + ): number | undefined { + const response = axios.isAxiosError(error) ? error.response : undefined + if (!response) return undefined + return retryDelayMs({ + method: config.method ?? 'GET', + status: response.status, + readRetryAfter: () => retryAfterHeader(response), + attempt, + policy, + random + }) + } + return async (config) => { const signal = config.signal instanceof AbortSignal ? config.signal : undefined - const budgetMs = config.timeout || undefined - const startedAt = Date.now() - const remainingMs = () => - budgetMs === undefined ? undefined : budgetMs - (Date.now() - startedAt) + const remainingMs = timeBudget(config.timeout) for (let attempt = 1; ; attempt++) { const remaining = remainingMs() @@ -36,20 +60,11 @@ function retryingAdapter( remaining === undefined ? config : { ...config, timeout: remaining } ) } catch (error) { - const response = axios.isAxiosError(error) ? error.response : undefined - const delay = response - ? retryDelayMs({ - method: config.method ?? 'GET', - status: response.status, - readRetryAfter: () => retryAfterHeader(response), - attempt, - policy, - random - }) - : undefined - const budgetLeft = remainingMs() - if (delay === undefined) throw error - if (budgetLeft !== undefined && delay >= budgetLeft) throw error + const delay = waitBeforeRetry(error, config, attempt) + const left = remainingMs() + if (delay === undefined || (left !== undefined && delay >= left)) { + throw error + } await sleep(delay, signal) } } From b2c44aa01b4cda40b9823c3db439a7bc7b23ed6a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 21:53:56 +0000 Subject: [PATCH 05/10] fix: retry web-session sends, replay request bodies, and cover the billing wiring Apply the retry policy at the web-session send used by the Cloud API and global-settings reads; the workspace axios web-session adapter keeps its own wrapper so no request is retried twice. fetchWithRetry now clones a body-bearing Request per attempt and does not retry a stream body it cannot replay. Add a test through the billing store wiring, drop an avoidable double assertion, and shorten the retry JSDoc. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/platform/auth/unified/remintRetry.test.ts | 8 +- .../billing/sdk/billingSdkStore.retry.test.ts | 71 ++++++++++++++++++ .../settings/globalSettingsApi.test.ts | 25 +++++++ src/platform/settings/globalSettingsApi.ts | 5 +- .../api/workspaceApi.webSessionRetry.test.ts | 16 ++++ src/scripts/api.fetchApi.test.ts | 73 ++++++++++++++++++- src/scripts/api.ts | 3 +- src/utils/axiosRetryableStatus.ts | 8 +- src/utils/fetchWithRetry.test.ts | 71 ++++++++++++++++++ src/utils/fetchWithRetry.ts | 45 +++++++++--- src/utils/retryPolicy.ts | 17 +++-- 11 files changed, 311 insertions(+), 31 deletions(-) create mode 100644 src/platform/billing/sdk/billingSdkStore.retry.test.ts diff --git a/src/platform/auth/unified/remintRetry.test.ts b/src/platform/auth/unified/remintRetry.test.ts index 21e6a4a53a2..992e102251f 100644 --- a/src/platform/auth/unified/remintRetry.test.ts +++ b/src/platform/auth/unified/remintRetry.test.ts @@ -492,7 +492,7 @@ describe('attachUnifiedRemintInterceptor', () => { // A custom axios adapter is responsible for its own status handling (axios // only applies validateStatus inside its built-in adapters), so reject // non-2xx with a real AxiosError to mirror a live response. - function makeAdapter(statuses: number[]): ReturnType { + function makeAdapter(statuses: number[]) { let call = 0 return vi.fn(async (config) => { const status = statuses[Math.min(call, statuses.length - 1)] @@ -519,7 +519,7 @@ describe('attachUnifiedRemintInterceptor', () => { function makeClient(statuses: number[]) { const adapter = makeAdapter(statuses) - const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + const client = axios.create({ adapter }) attachUnifiedRemintInterceptor(client) return { client, adapter } } @@ -578,7 +578,7 @@ describe('attachUnifiedRemintInterceptor', () => { it('keeps the 503 retries and the 401 re-mint on separate budgets', async () => { const adapter = makeAdapter([503, 503, 401, 503, 200]) - const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + const client = axios.create({ adapter }) attachRetryableStatusRetry(client, { sleep: async () => {} }) attachUnifiedRemintInterceptor(client) vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( @@ -695,7 +695,7 @@ describe('attachUnifiedRemintInterceptor', () => { response ) }) - const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + const client = axios.create({ adapter }) attachUnifiedRemintInterceptor(client) vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( 'tokenB' diff --git a/src/platform/billing/sdk/billingSdkStore.retry.test.ts b/src/platform/billing/sdk/billingSdkStore.retry.test.ts new file mode 100644 index 00000000000..c48a58a5b06 --- /dev/null +++ b/src/platform/billing/sdk/billingSdkStore.retry.test.ts @@ -0,0 +1,71 @@ +import { CREDITS_ROUTE } from '@comfyorg/account-core/billing' +import type { + AccountCredential, + SessionSnapshot +} from '@comfyorg/account-core/session' +import { fromPartial } from '@total-typescript/shoehorn' +import type { User } from 'firebase/auth' +import { beforeEach, expect, it, vi } from 'vitest' + +import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' +import { + stubAccountIdentityPort, + stubFirebaseAuthHarness +} from '@/utils/__tests__/stubAccountIdentityPort' + +import { useBillingSdkStore } from './billingSdkStore' + +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@/platform/billing/composables/useBillingCapabilities')) +vi.mock(import('@/platform/settings/composables/useSettingsDialog')) +vi.mock(import('firebase/auth')) + +const CREDENTIAL: AccountCredential = { + token: 'workspace-jwt', + expiresAt: Date.now() + 60 * 60 * 1000, + uid: 'uid-1', + workspace: { id: 'ws-1', name: 'Personal', type: 'personal' }, + role: 'owner', + permissions: ['workspace:read'] +} + +const SNAPSHOT: SessionSnapshot = { + phase: 'authenticated', + user: fromPartial({ uid: 'uid-1' }), + session: CREDENTIAL +} + +beforeEach(() => { + stubAccountIdentityPort() + stubFirebaseAuthHarness() + const session = useWorkspaceAuthStore().getUnifiedSessionClient() + vi.spyOn(session, 'getSnapshot').mockReturnValue(SNAPSHOT) + vi.spyOn(session, 'ensureFresh').mockResolvedValue({ + status: 'ok', + session: CREDENTIAL + }) +}) + +it('reads the balance again after a 503 through the store wiring', async () => { + vi.useFakeTimers() + const answers = [ + new Response(null, { status: 503 }), + Response.json({ amount_micros: 1_000_000, currency: 'USD' }) + ] + vi.mocked(fetch).mockImplementation(async (input) => { + expect(String(input)).toContain(CREDITS_ROUTE) + const answer = answers.shift() + if (!answer) throw new Error('fetch called more times than scripted') + return answer + }) + + const pending = useBillingSdkStore().readBalance() + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toMatchObject({ + status: 'ok', + value: { amount_micros: 1_000_000 } + }) + expect(fetch).toHaveBeenCalledTimes(2) +}) diff --git a/src/platform/settings/globalSettingsApi.test.ts b/src/platform/settings/globalSettingsApi.test.ts index 2f515080f3d..3c3a1016440 100644 --- a/src/platform/settings/globalSettingsApi.test.ts +++ b/src/platform/settings/globalSettingsApi.test.ts @@ -3,6 +3,7 @@ vi.mock(import('@/services/dialogService')) import type { GlobalSetting } from '@comfyorg/ingest-types' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { WebSessionSend } from '@/platform/auth/session/webSessionFetch' import { api } from '@/scripts/api' import { useAuthStore } from '@/stores/authStore' @@ -159,4 +160,28 @@ describe('Global Settings transport', () => { }) } ) + describe('on the web session', () => { + const unavailable = () => new Response(null, { status: 503 }) + + it('reads again after a 503 with exactly two sends', async () => { + vi.useFakeTimers() + const answers = [unavailable(), Response.json(stored)] + const send = vi.fn(async () => answers.shift()!) + + const pending = getGlobalSetting(key, send) + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual(stored) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('does not send a write twice after a 503', async () => { + const send = vi.fn(async () => unavailable()) + + await expect( + setGlobalSetting({ key, value: true }, send) + ).rejects.toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledOnce() + }) + }) }) diff --git a/src/platform/settings/globalSettingsApi.ts b/src/platform/settings/globalSettingsApi.ts index 905c5d458d5..be7038d04cd 100644 --- a/src/platform/settings/globalSettingsApi.ts +++ b/src/platform/settings/globalSettingsApi.ts @@ -15,6 +15,7 @@ import { api } from '@/scripts/api' import type { WebSessionSend } from '@/platform/auth/session/webSessionFetch' import { useAuthStore } from '@/stores/authStore' import type { AuthHeader } from '@/types/authTypes' +import { retryingSend } from '@/utils/fetchWithRetry' export class GlobalSettingsApiError extends Error { constructor( @@ -57,7 +58,9 @@ function sendGlobalSetting( auth: GlobalSettingsAuth, headers: Record = {} ): Promise { - if (typeof auth === 'function') return auth(url, { ...init, headers }) + if (typeof auth === 'function') { + return retryingSend(auth)(url, { ...init, headers }) + } return shouldRemintCloudRequest().then((remint) => fetchWithUnifiedRemint( url, diff --git a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts index eab6c649588..fc430a158bb 100644 --- a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts +++ b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts @@ -7,6 +7,7 @@ import type { } from '@/platform/auth/session/webSessionFetch' import { provideWebSessionRequests } from '@/platform/auth/session/webSessionFetch' import { stubFirebaseAuthHarness } from '@/utils/__tests__/stubAccountIdentityPort' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' import { workspaceApi } from './workspaceApi' import { installWorkspaceApiAuth } from './workspaceApiAuth' @@ -59,6 +60,21 @@ describe('workspaceApi on the web session', () => { expect(send).toHaveBeenCalledTimes(2) }) + it('sends no more than the policy attempt count on a persistent 503', async () => { + vi.useFakeTimers() + const send = provideSessionSending( + unavailable(), + unavailable(), + unavailable() + ) + + const pending = workspaceApi.list().catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + it('does not retry a write that got a 503', async () => { const send = provideSessionSending(unavailable()) diff --git a/src/scripts/api.fetchApi.test.ts b/src/scripts/api.fetchApi.test.ts index 055d7ccbb1b..688c61d99c1 100644 --- a/src/scripts/api.fetchApi.test.ts +++ b/src/scripts/api.fetchApi.test.ts @@ -1,5 +1,13 @@ import { fromPartial } from '@total-typescript/shoehorn' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + afterEach, + beforeEach, + describe, + expect, + it, + onTestFinished, + vi +} from 'vitest' import type { useFeatureFlags } from '@/composables/useFeatureFlags' import type { ApiAuthProvider } from '@/platform/auth/apiAuthProvider' @@ -11,6 +19,7 @@ import type { WebSessionRequests } from '@/platform/auth/session/webSessionFetch import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' import { useTelemetry } from '@/platform/telemetry' import { useAuthStore } from '@/stores/authStore' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' const { addBreadcrumb } = vi.hoisted(() => ({ addBreadcrumb: vi.fn() @@ -227,6 +236,68 @@ describe('api.fetchApi', () => { expect(send).toHaveBeenCalledOnce() }) + describe('on the session with a retryable failure', () => { + function provideSessionSending(...answers: Response[]) { + const send = vi.fn(async () => { + const answer = answers.shift() + if (!answer) throw new Error('send called more times than scripted') + return answer + }) + onTestFinished( + provideWebSessionRequests( + fromPartial({ + scope: async () => ({ session: fromPartial({}), epoch: 1 }), + send + }) + ) + ) + return send + } + + const unavailable = () => new Response(null, { status: 503 }) + + beforeEach(() => { + signInOnCloud() + vi.useFakeTimers() + }) + + it('recovers from a 503 with exactly two sends', async () => { + const send = provideSessionSending( + unavailable(), + new Response('ok', { status: 200 }) + ) + + const pending = api.fetchApi('/test') + await vi.advanceTimersByTimeAsync(10_000) + + expect((await pending).status).toBe(200) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('stops at the policy attempt count on a persistent 503', async () => { + const send = provideSessionSending( + unavailable(), + unavailable(), + unavailable() + ) + + const pending = api.fetchApi('/test') + await vi.advanceTimersByTimeAsync(10_000) + + expect((await pending).status).toBe(503) + expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it('does not retry a POST', async () => { + const send = provideSessionSending(unavailable()) + + const response = await api.fetchApi('/test', { method: 'POST' }) + + expect(response.status).toBe(503) + expect(send).toHaveBeenCalledOnce() + }) + }) + it('reports cloud-auth-header when a header was attached', async () => { signInOnCloud() vi.mocked(useAuthStore().getAuthHeader).mockResolvedValue({ diff --git a/src/scripts/api.ts b/src/scripts/api.ts index a6a42125478..78db2c5d405 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -84,6 +84,7 @@ import type { import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' import type { AuthCredential, AuthScheme } from '@/types/authTypes' import type { NodeExecutionId } from '@/types/nodeIdentification' +import { retryingSend } from '@/utils/fetchWithRetry' import { fetchHistory, fetchJobAssets, @@ -617,7 +618,7 @@ export class ComfyApi extends EventTarget { const requests = webSessionRequests() if (!requests) return undefined const scope = await requests.scope() - return scope && ((url, init) => requests.send(url, init, scope)) + return scope && retryingSend((url, init) => requests.send(url, init, scope)) } /** Sends a same-origin request on the web session; undefined when this tab is not on it. */ diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index 1e7d243efea..52f78883741 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -71,13 +71,7 @@ function retryingAdapter( } } -/** - * Retries a retryable status around whichever adapter each request selects, - * so a per-request adapter is covered too. The retry sits below the - * interceptor chain: each request that reaches the adapter, including the one - * an interceptor re-issues after a 401 re-mint, gets its own attempt budget, - * and `timeout` bounds the attempts and the waits between them together. - */ +/** Retries retryable statuses around each request's adapter, within its `timeout`. */ export function attachRetryableStatusRetry( client: AxiosInstance, options: AxiosRetryableStatusOptions = {} diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts index d4395290046..b93a7026495 100644 --- a/src/utils/fetchWithRetry.test.ts +++ b/src/utils/fetchWithRetry.test.ts @@ -89,6 +89,77 @@ describe('fetchWithRetry', () => { expect(fetchImpl).toHaveBeenCalledTimes(1) }) + describe('replaying a body', () => { + const optedIn = { ...DEFAULT_RETRY_POLICY, retryNonIdempotent: true } + + function consumingFetch(...responses: Response[]) { + const bodies: string[] = [] + const queue = [...responses] + const fetchImpl = vi.fn(async (input, init) => { + bodies.push(await new Request(input, init).text()) + const next = queue.shift() + if (!next) throw new Error('fetch called more times than scripted') + return next + }) + return { fetchImpl, bodies } + } + + it('sends the body of a Request again on the second attempt', async () => { + const { fetchImpl, bodies } = consumingFetch(respond(503), respond(200)) + + const result = await fetchWithRetry( + new Request('https://api/x', { method: 'POST', body: 'payload' }), + {}, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(result.status).toBe(200) + expect(bodies).toEqual(['payload', 'payload']) + }) + + it('sends an init body again on the second attempt', async () => { + const { fetchImpl, bodies } = consumingFetch(respond(503), respond(200)) + + await fetchWithRetry( + 'https://api/x', + { method: 'POST', body: 'payload' }, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(bodies).toEqual(['payload', 'payload']) + }) + + it('leaves the caller request unconsumed', async () => { + const { fetchImpl } = consumingFetch(respond(503), respond(200)) + const request = new Request('https://api/x', { + method: 'POST', + body: 'payload' + }) + + await fetchWithRetry( + request, + {}, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(request.bodyUsed).toBe(false) + }) + + it('does not retry a stream body that cannot be replayed', async () => { + const fetchImpl = scriptedFetch(respond(503)) + const body = new ReadableStream() + + const result = await fetchWithRetry( + 'https://api/x', + { method: 'POST', body, duplex: 'half' } as RequestInit, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + }) + it('retries a POST only when the policy opts in', async () => { const fetchImpl = scriptedFetch(respond(503), respond(200)) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts index 49179644040..0bb3a89bbdb 100644 --- a/src/utils/fetchWithRetry.ts +++ b/src/utils/fetchWithRetry.ts @@ -1,9 +1,18 @@ import type { RetryPolicy } from '@/utils/retryPolicy' -import { retryDelayMs, sleep as sleepFor } from '@/utils/retryPolicy' +import { + canRetryMethod, + retryDelayMs, + sleep as sleepFor +} from '@/utils/retryPolicy' + +type FetchLike = ( + input: RequestInfo | URL, + init?: RequestInit +) => Promise export interface RetryingFetchOptions { readonly policy?: RetryPolicy - readonly fetchImpl?: typeof fetch + readonly fetchImpl?: FetchLike readonly sleep?: (ms: number, signal?: AbortSignal) => Promise readonly random?: () => number } @@ -19,12 +28,14 @@ function requestSignal( return init.signal ?? (input instanceof Request ? input.signal : undefined) } -/** - * `fetch` that re-issues idempotent requests the server answered with a - * retryable status, honoring `Retry-After`. Returns the last response once - * attempts are spent; never retries a non-idempotent method unless the policy - * opts in. - */ +function isReplayable(input: RequestInfo | URL, init: RequestInit): boolean { + return ( + !(init.body instanceof ReadableStream) && + !(input instanceof Request && input.bodyUsed) + ) +} + +/** `fetch` that retries retryable statuses on idempotent methods, honoring `Retry-After`. */ export async function fetchWithRetry( input: RequestInfo | URL, init: RequestInit = {}, @@ -32,10 +43,16 @@ export async function fetchWithRetry( ): Promise { const { policy, fetchImpl = fetch, sleep = sleepFor, random } = options const method = requestMethod(input, init) + if (!canRetryMethod(method, policy) || !isReplayable(input, init)) { + return fetchImpl(input, init) + } + const signal = requestSignal(input, init) + const template = + input instanceof Request && input.body !== null ? input.clone() : undefined for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(input, init) + const response = await fetchImpl(template?.clone() ?? input, init) const delay = retryDelayMs({ method, status: response.status, @@ -49,3 +66,13 @@ export async function fetchWithRetry( await sleep(delay, signal) } } + +/** Wraps a url-based send so it retries like {@link fetchWithRetry}. */ +export function retryingSend( + send: (url: string, init: RequestInit) => Promise +): (url: string, init: RequestInit) => Promise { + return (url, init) => + fetchWithRetry(url, init, { + fetchImpl: (input, attemptInit) => send(String(input), attemptInit ?? {}) + }) +} diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts index fa6a1fcd526..029a4ae0bc8 100644 --- a/src/utils/retryPolicy.ts +++ b/src/utils/retryPolicy.ts @@ -27,6 +27,13 @@ function isIdempotentMethod(method: string): boolean { return IDEMPOTENT_METHODS.has(method.toUpperCase()) } +export function canRetryMethod( + method: string, + policy: RetryPolicy = DEFAULT_RETRY_POLICY +): boolean { + return policy.retryNonIdempotent || isIdempotentMethod(method) +} + /** Milliseconds a `Retry-After` header asks for (delta-seconds or HTTP-date). */ export function parseRetryAfterMs( value: string | null | undefined, @@ -50,11 +57,7 @@ interface RetryDecisionInput { readonly now?: () => number } -/** - * How long to wait before the next attempt, or `undefined` to stop retrying: - * the request is not retryable, attempts are spent, or the server asked to - * wait longer than the policy allows. - */ +/** Delay before the next attempt, or `undefined` to stop retrying. */ export function retryDelayMs({ method, status, @@ -66,9 +69,7 @@ export function retryDelayMs({ }: RetryDecisionInput): number | undefined { if (attempt >= policy.maxAttempts) return undefined if (!isRetryableStatus(status)) return undefined - if (!policy.retryNonIdempotent && !isIdempotentMethod(method)) { - return undefined - } + if (!canRetryMethod(method, policy)) return undefined const retryAfterMs = parseRetryAfterMs(readRetryAfter(), now()) if (retryAfterMs !== undefined && retryAfterMs > policy.maxDelayMs) { From d7fb9892b0379b68d1612d4b4d65dedc780fc616 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 02:39:25 +0000 Subject: [PATCH 06/10] fix: keep retry deadlines, cancellation and config intact across retries Stop retrying once the request timeout is spent after a late wait, hand the original config back on errors so a 401 re-mint replays the full timeout, report an abort or cancel token fired during the wait as a cancellation, and make attaching the retry twice idempotent. Add jitter on top of Retry-After without going below it or past the cap. fetchWithRetry returns the last response when the timeout signal fires during the wait. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/utils/axiosRetryableStatus.test.ts | 86 +++++++++++++++++++++++++- src/utils/axiosRetryableStatus.ts | 40 ++++++++++-- src/utils/fetchWithRetry.test.ts | 23 ++++++- src/utils/fetchWithRetry.ts | 15 ++++- src/utils/retryPolicy.test.ts | 19 ++++-- src/utils/retryPolicy.ts | 4 +- 6 files changed, 169 insertions(+), 18 deletions(-) diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts index 05558aaf30d..041f3533736 100644 --- a/src/utils/axiosRetryableStatus.test.ts +++ b/src/utils/axiosRetryableStatus.test.ts @@ -89,7 +89,7 @@ describe('attachRetryableStatusRetry', () => { await client.get('https://api/x') - expect(sleep).toHaveBeenCalledExactlyOnceWith(2_000, undefined) + expect(sleep).toHaveBeenCalledExactlyOnceWith(2_250, undefined) }) it('rejects without waiting when Retry-After is past the cap', async () => { @@ -126,6 +126,19 @@ describe('attachRetryableStatusRetry', () => { expect(defaultAdapter).not.toHaveBeenCalled() }) + it('does not stack a second retry loop when attached twice', async () => { + const adapter = scriptedAdapter([{ status: 503 }]) + const client = axios.create({ adapter }) + const sleep = async () => {} + attachRetryableStatusRetry(client, { sleep }) + attachRetryableStatusRetry(client, { sleep }) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + describe('with a request timeout', () => { function makeTimedClient(answers: ScriptedAnswer[]) { const adapter = scriptedAdapter(answers) @@ -149,6 +162,73 @@ describe('attachRetryableStatusRetry', () => { expect(vi.getTimerCount()).toBe(0) }) + it('surfaces the last failure when the wait overruns the timeout', async () => { + vi.useFakeTimers() + const adapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + vi.setSystemTime(Date.now() + 5_000) + } + }) + + await expect( + client.get('https://api/x', { timeout: 1_000 }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it('hands the original timeout back on the error it throws', async () => { + vi.useFakeTimers() + const adapter = scriptedAdapter([{ status: 503 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + vi.setSystemTime(Date.now() + 100) + } + }) + + const error = await client + .get('https://api/x', { timeout: 5_000 }) + .catch((reason: unknown) => reason) + + expect(error).toMatchObject({ config: { timeout: 5_000 } }) + }) + + it('reports an abort during the wait as a cancellation', async () => { + vi.useFakeTimers() + const { client } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const controller = new AbortController() + + const outcome = client + .get('https://api/x', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort() + + expect(axios.isCancel(await outcome)).toBe(true) + }) + + it('reports a cancel token fired during the wait as a cancellation', async () => { + vi.useFakeTimers() + const { client } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const source = axios.CancelToken.source() + + const outcome = client + .get('https://api/x', { cancelToken: source.token }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + source.cancel() + + expect(axios.isCancel(await outcome)).toBe(true) + }) + it('retries when the wait fits inside the timeout and shrinks the next attempt', async () => { vi.useFakeTimers() const { client, adapter } = makeTimedClient([ @@ -157,11 +237,11 @@ describe('attachRetryableStatusRetry', () => { ]) const pending = client.get('https://api/x', { timeout: 5_000 }) - await vi.advanceTimersByTimeAsync(1_000) + await vi.advanceTimersByTimeAsync(1_250) expect((await pending).status).toBe(200) expect(adapter.mock.calls.map(([config]) => config.timeout)).toEqual([ - 5_000, 4_000 + 5_000, 3_750 ]) }) }) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index 52f78883741..c5748bc793a 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -27,6 +27,20 @@ function timeBudget(timeoutMs: number | undefined) { return () => (timeoutMs ? timeoutMs - (Date.now() - startedAt) : undefined) } +function cancellationSignal( + config: InternalAxiosRequestConfig +): AbortSignal | undefined { + const signal = + config.signal instanceof AbortSignal ? config.signal : undefined + if (!config.cancelToken) return signal + + const controller = new AbortController() + void config.cancelToken.promise.then(() => controller.abort()) + return signal + ? AbortSignal.any([signal, controller.signal]) + : controller.signal +} + function retryingAdapter( adapter: AxiosAdapter, { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions @@ -48,36 +62,50 @@ function retryingAdapter( }) } + async function pause(delay: number, config: InternalAxiosRequestConfig) { + const signal = cancellationSignal(config) + try { + await sleep(delay, signal) + } catch (error) { + if (!signal?.aborted) throw error + throw new axios.CanceledError('canceled', config) + } + } + return async (config) => { - const signal = - config.signal instanceof AbortSignal ? config.signal : undefined const remainingMs = timeBudget(config.timeout) + let lastError: unknown for (let attempt = 1; ; attempt++) { const remaining = remainingMs() + if (remaining !== undefined && remaining <= 0) throw lastError try { - return await adapter( + const response = await adapter( remaining === undefined ? config : { ...config, timeout: remaining } ) + response.config = config + return response } catch (error) { + if (axios.isAxiosError(error) && error.config) error.config = config + lastError = error const delay = waitBeforeRetry(error, config, attempt) const left = remainingMs() if (delay === undefined || (left !== undefined && delay >= left)) { throw error } - await sleep(delay, signal) + await pause(delay, config) } } } } +const retrying = new WeakSet() + /** Retries retryable statuses around each request's adapter, within its `timeout`. */ export function attachRetryableStatusRetry( client: AxiosInstance, options: AxiosRetryableStatusOptions = {} ): void { - const retrying = new WeakSet() - client.interceptors.request.use((config) => { const selected = config.adapter ?? client.defaults.adapter if (typeof selected === 'function' && retrying.has(selected)) return config diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts index b93a7026495..99fbf7fe639 100644 --- a/src/utils/fetchWithRetry.test.ts +++ b/src/utils/fetchWithRetry.test.ts @@ -186,7 +186,7 @@ describe('fetchWithRetry', () => { await fetchWithRetry('https://api/x', {}, { fetchImpl, sleep, random }) - expect(sleep).toHaveBeenCalledExactlyOnceWith(4_000, undefined) + expect(sleep).toHaveBeenCalledExactlyOnceWith(4_250, undefined) }) it('returns the 503 without waiting when Retry-After is past the cap', async () => { @@ -214,15 +214,32 @@ describe('fetchWithRetry', () => { respond(200) ) - const pending = fetchWithRetry('https://api/x', {}, { fetchImpl }) + const pending = fetchWithRetry('https://api/x', {}, { fetchImpl, random }) - await vi.advanceTimersByTimeAsync(2_999) + await vi.advanceTimersByTimeAsync(3_249) expect(fetchImpl).toHaveBeenCalledTimes(1) await vi.advanceTimersByTimeAsync(1) expect((await pending).status).toBe(200) expect(fetchImpl).toHaveBeenCalledTimes(2) }) + it('returns the last response when the timeout signal fires mid-wait', async () => { + const controller = new AbortController() + const last = respond(503, { 'Retry-After': '5' }) + const fetchImpl = scriptedFetch(last) + + const pending = fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort(new DOMException('Fetch timeout', 'TimeoutError')) + + expect(await pending).toBe(last) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + it('stops waiting when the caller aborts', async () => { const controller = new AbortController() const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '5' })) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts index 0bb3a89bbdb..a3773279abf 100644 --- a/src/utils/fetchWithRetry.ts +++ b/src/utils/fetchWithRetry.ts @@ -28,6 +28,14 @@ function requestSignal( return init.signal ?? (input instanceof Request ? input.signal : undefined) } +function timedOut(signal: AbortSignal | undefined): boolean { + return ( + signal?.aborted === true && + signal.reason instanceof DOMException && + signal.reason.name === 'TimeoutError' + ) +} + function isReplayable(input: RequestInfo | URL, init: RequestInit): boolean { return ( !(init.body instanceof ReadableStream) && @@ -62,8 +70,13 @@ export async function fetchWithRetry( random }) if (delay === undefined) return response + try { + await sleep(delay, signal) + } catch (error) { + if (timedOut(signal)) return response + throw error + } await response.body?.cancel() - await sleep(delay, signal) } } diff --git a/src/utils/retryPolicy.test.ts b/src/utils/retryPolicy.test.ts index a5f01641e0b..ddacf467f99 100644 --- a/src/utils/retryPolicy.test.ts +++ b/src/utils/retryPolicy.test.ts @@ -106,13 +106,24 @@ describe('retryDelayMs', () => { } ) - it('waits at least as long as Retry-After asks', () => { - expect(decide({ readRetryAfter: () => '4' })).toBe(4_000) - }) + it.for([ + [0, 4_000], + [1, 4_250] + ])( + 'adds jitter on top of Retry-After, never below it (random=%f)', + ([random, expected]) => { + expect(decide({ readRetryAfter: () => '4', random: () => random })).toBe( + expected + ) + } + ) it('reads an HTTP-date Retry-After against the clock', () => { expect( - decide({ readRetryAfter: () => 'Sun, 18 Oct 2026 12:00:03 GMT' }) + decide({ + readRetryAfter: () => 'Sun, 18 Oct 2026 12:00:03 GMT', + random: () => 0 + }) ).toBe(3_000) }) diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts index 029a4ae0bc8..f8bd66be4e4 100644 --- a/src/utils/retryPolicy.ts +++ b/src/utils/retryPolicy.ts @@ -81,7 +81,9 @@ export function retryDelayMs({ policy.baseDelayMs * 2 ** (attempt - 1) ) const jittered = ceiling / 2 + (random() * ceiling) / 2 - return Math.max(jittered, retryAfterMs ?? 0) + if (retryAfterMs === undefined) return jittered + const spread = (random() * ceiling) / 2 + return Math.min(policy.maxDelayMs, Math.max(jittered, retryAfterMs + spread)) } export function sleep(ms: number, signal?: AbortSignal): Promise { From f74bb148ce94d290f16b61caa408673c536f0987 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 02:41:47 +0000 Subject: [PATCH 07/10] refactor: split the axios retrying adapter attempt and budget checks Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/utils/axiosRetryableStatus.ts | 36 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index c5748bc793a..2748deac51d 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -41,6 +41,30 @@ function cancellationSignal( : controller.signal } +function fitsBudget( + delay: number | undefined, + remaining: number | undefined +): delay is number { + return delay !== undefined && (remaining === undefined || delay < remaining) +} + +async function runAttempt( + adapter: AxiosAdapter, + config: InternalAxiosRequestConfig, + remaining: number | undefined +): Promise { + try { + const response = await adapter( + remaining === undefined ? config : { ...config, timeout: remaining } + ) + response.config = config + return response + } catch (error) { + if (axios.isAxiosError(error) && error.config) error.config = config + throw error + } +} + function retryingAdapter( adapter: AxiosAdapter, { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions @@ -80,19 +104,11 @@ function retryingAdapter( const remaining = remainingMs() if (remaining !== undefined && remaining <= 0) throw lastError try { - const response = await adapter( - remaining === undefined ? config : { ...config, timeout: remaining } - ) - response.config = config - return response + return await runAttempt(adapter, config, remaining) } catch (error) { - if (axios.isAxiosError(error) && error.config) error.config = config lastError = error const delay = waitBeforeRetry(error, config, attempt) - const left = remainingMs() - if (delay === undefined || (left !== undefined && delay >= left)) { - throw error - } + if (!fitsBudget(delay, remainingMs())) throw error await pause(delay, config) } } From 5f876dba7983cd82f5375bbea0ee2714e11a204a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 03:00:44 +0000 Subject: [PATCH 08/10] test: consume the Request the retry hands to fetch The body-replay cases now read the body of the Request they receive, so sending the same Request twice fails them. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- src/utils/fetchWithRetry.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts index 99fbf7fe639..b2b8eb3c399 100644 --- a/src/utils/fetchWithRetry.test.ts +++ b/src/utils/fetchWithRetry.test.ts @@ -96,7 +96,8 @@ describe('fetchWithRetry', () => { const bodies: string[] = [] const queue = [...responses] const fetchImpl = vi.fn(async (input, init) => { - bodies.push(await new Request(input, init).text()) + const sent = input instanceof Request ? input : new Request(input, init) + bodies.push(await sent.text()) const next = queue.shift() if (!next) throw new Error('fetch called more times than scripted') return next From 7e7cd76dd82ce0eae821b4b7fef39cf4cee8fd78 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 05:59:47 +0000 Subject: [PATCH 09/10] test: mock the ingest URL builder in the web-session retry test The workspace URL builder moved to the ingest module on main. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Q6WHcAWundnpSwdSQWCMph --- .../workspace/api/workspaceApi.webSessionRetry.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts index fc430a158bb..871066597d4 100644 --- a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts +++ b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts @@ -14,8 +14,8 @@ import { installWorkspaceApiAuth } from './workspaceApiAuth' vi.mock(import('@/i18n')) -vi.mock(import('./workspaceApiUrl'), () => ({ - workspaceApiUrl: (path: string) => `/api${path}` +vi.mock(import('@/platform/remote/ingest/ingestApiUrl'), () => ({ + ingestApiUrl: (path: string) => `/api${path}` })) vi.mock(import('firebase/auth'), { spy: true }) From d142bd22050d701fbf9890ac87852397d2490a17 Mon Sep 17 00:00:00 2001 From: dante01yoon Date: Tue, 27 Oct 2026 09:15:37 -0400 Subject: [PATCH 10/10] fix: reject on abort mid-backoff and skip the shared retry where callers already retry fetchWithRetry now rejects with the abort reason when a timeout or caller cancel fires during the backoff wait, matching the axios path, instead of returning the last 503 to a caller that already gave up. Adds a skipStatusRetry option (fetchApi, workspaceApi reads) and an axios __skipStatusRetry flag. The cloud boot settings load, workspace boot reads and deployment listing already retry on their own backoff, so they send once per attempt: a persistent 503 on settings is 4 requests instead of up to 12. --- src/platform/auth/apiAuthProvider.ts | 4 +- src/platform/auth/unified/remintRetry.test.ts | 16 +++++ src/platform/auth/unified/remintRetry.ts | 10 ++- src/platform/settings/settingStore.test.ts | 8 +++ src/platform/settings/settingStore.ts | 11 ++-- .../workspace/api/workspaceApi.test.ts | 4 +- src/platform/workspace/api/workspaceApi.ts | 30 +++++++-- .../api/workspaceApi.webSessionRetry.test.ts | 18 ++++++ .../stores/teamWorkspaceStore.test.ts | 4 +- .../workspace/stores/teamWorkspaceStore.ts | 17 +++-- src/scripts/api.fetchApi.test.ts | 63 ++++++++++++++++++- src/scripts/api.ts | 26 ++++++-- src/types/axios-augmentation.d.ts | 2 + src/utils/axiosRetryableStatus.test.ts | 28 +++++++++ src/utils/axiosRetryableStatus.ts | 1 + src/utils/fetchWithRetry.test.ts | 28 +++++++-- src/utils/fetchWithRetry.ts | 19 ++---- src/utils/retryPolicy.ts | 6 ++ 18 files changed, 248 insertions(+), 47 deletions(-) diff --git a/src/platform/auth/apiAuthProvider.ts b/src/platform/auth/apiAuthProvider.ts index 10294727fdc..31ed8614f6b 100644 --- a/src/platform/auth/apiAuthProvider.ts +++ b/src/platform/auth/apiAuthProvider.ts @@ -1,4 +1,5 @@ import type { AuthHeader } from '@/types/authTypes' +import type { RetryPolicy } from '@/utils/retryPolicy' export interface RetrySignalLifecycle { clearInitialTimeout: () => void @@ -19,7 +20,8 @@ export interface ApiAuthProvider { input: RequestInfo | URL, init: RequestInit, retryOn401: boolean, - retrySignalLifecycle?: RetrySignalLifecycle + retrySignalLifecycle?: RetrySignalLifecycle, + retryPolicy?: RetryPolicy ): Promise } diff --git a/src/platform/auth/unified/remintRetry.test.ts b/src/platform/auth/unified/remintRetry.test.ts index 992e102251f..03aa7a430fb 100644 --- a/src/platform/auth/unified/remintRetry.test.ts +++ b/src/platform/auth/unified/remintRetry.test.ts @@ -15,6 +15,7 @@ import { fetchWithUnifiedRemint } from '@/platform/auth/unified/remintRetry' import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' +import { NO_RETRY_POLICY } from '@/utils/retryPolicy' vi.mock(import('@/platform/telemetry')) @@ -169,6 +170,21 @@ describe('fetchWithUnifiedRemint', () => { expect(useWorkspaceAuthStore().remintUnifiedOnce).toHaveBeenCalledTimes(1) }) + it('sends once on a 503 under the no-retry policy', async () => { + vi.mocked(fetch).mockResolvedValueOnce(unavailable()) + + const result = await fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true, + undefined, + NO_RETRY_POLICY + ) + + expect(result.status).toBe(503) + expect(fetch).toHaveBeenCalledTimes(1) + }) + it('does not retry a POST that got a 503', async () => { vi.mocked(fetch).mockResolvedValueOnce(unavailable()) diff --git a/src/platform/auth/unified/remintRetry.ts b/src/platform/auth/unified/remintRetry.ts index dae1e526ecc..5636e45a238 100644 --- a/src/platform/auth/unified/remintRetry.ts +++ b/src/platform/auth/unified/remintRetry.ts @@ -19,6 +19,7 @@ import type { UnifiedAuthRetryMetadata } from '@/platform/telemetry/types' import { fetchWithRetry } from '@/utils/fetchWithRetry' +import type { RetryPolicy } from '@/utils/retryPolicy' let cachedUnifiedFlags: | { readonly unifiedCloudAuthEnabled: boolean } @@ -147,13 +148,14 @@ export async function fetchWithUnifiedRemint( input: RequestInfo | URL, init: RequestInit, shouldRetryOn401: boolean, - retrySignalLifecycle?: RetrySignalLifecycle + retrySignalLifecycle?: RetrySignalLifecycle, + retryPolicy?: RetryPolicy ): Promise { const retryInput = shouldRetryOn401 && input instanceof Request && input.body !== null ? input.clone() : input - const response = await fetchWithRetry(input, init) + const response = await fetchWithRetry(input, init, { policy: retryPolicy }) if (response.status === 403 && isCloud) { await presentSsoRequired(() => presentForResponse(response)) } @@ -189,7 +191,9 @@ export async function fetchWithUnifiedRemint( ? { ...init, headers, signal: retrySignalLifecycle.createSignal() } : { ...init, headers } try { - const retryResponse = await fetchWithRetry(retryInput, retryInit) + const retryResponse = await fetchWithRetry(retryInput, retryInit, { + policy: retryPolicy + }) trackRetryResponse('fetch', retryResponse.status) return retryResponse } catch (error) { diff --git a/src/platform/settings/settingStore.test.ts b/src/platform/settings/settingStore.test.ts index a50cbbb652c..c5a1bf248c6 100644 --- a/src/platform/settings/settingStore.test.ts +++ b/src/platform/settings/settingStore.test.ts @@ -50,6 +50,14 @@ describe('useSettingStore', () => { expect(api.getSettings).toHaveBeenCalled() }) + it('loads settings without the shared status retry stacking on its own backoff', async () => { + vi.mocked(api.getSettings).mockResolvedValue({} as Settings) + + await store.load() + + expect(api.getSettings).toHaveBeenCalledWith({ skipStatusRetry: true }) + }) + describe('Canvas Navigation override migration', () => { const NAV = 'Comfy.Canvas.NavigationMode' const LEFT = 'Comfy.Canvas.LeftMouseClickBehavior' diff --git a/src/platform/settings/settingStore.ts b/src/platform/settings/settingStore.ts index 87521a2ec6c..a097ffd56ca 100644 --- a/src/platform/settings/settingStore.ts +++ b/src/platform/settings/settingStore.ts @@ -139,10 +139,13 @@ export const useSettingStore = defineStore('setting', () => { 'Setting values must be loaded before any setting is registered.' ) } - settingValues.value = await retry(() => api.getSettings(), { - retries: 3, - delay: (attempt) => Math.min(1000 * Math.pow(2, attempt), 8000) - }) + settingValues.value = await retry( + () => api.getSettings({ skipStatusRetry: true }), + { + retries: 3, + delay: (attempt) => Math.min(1000 * Math.pow(2, attempt), 8000) + } + ) await migrateZoomThresholdToFontSize() await migrateCanvasNavigationOverrides() }, diff --git a/src/platform/workspace/api/workspaceApi.test.ts b/src/platform/workspace/api/workspaceApi.test.ts index e9c9deedb4f..c41824ad973 100644 --- a/src/platform/workspace/api/workspaceApi.test.ts +++ b/src/platform/workspace/api/workspaceApi.test.ts @@ -355,7 +355,7 @@ describe('workspaceApi', () => { }) describe('deployment picking', () => { - it('listDeployments() sends GET /workspaces/:id/deployments with a 30 second timeout', async () => { + it('listDeployments() sends GET /workspaces/:id/deployments with a 30 second timeout and no status retry', async () => { const data = { items: [], builds_visible: true } mockAxiosInstance.get.mockResolvedValue({ data }) @@ -363,7 +363,7 @@ describe('workspaceApi', () => { expect(mockAxiosInstance.get).toHaveBeenCalledWith( '/api/workspaces/ws-1/deployments', - { headers: AUTH_HEADER, timeout: 30_000 } + { headers: AUTH_HEADER, timeout: 30_000, __skipStatusRetry: true } ) expect(result).toEqual(data) }) diff --git a/src/platform/workspace/api/workspaceApi.ts b/src/platform/workspace/api/workspaceApi.ts index 948d5d12cd9..b2460c16b98 100644 --- a/src/platform/workspace/api/workspaceApi.ts +++ b/src/platform/workspace/api/workspaceApi.ts @@ -1,3 +1,5 @@ +import type { AxiosRequestConfig } from 'axios' + import type { AcceptInviteResponse, CreateInviteRequest, @@ -64,25 +66,39 @@ export type { WorkspaceDeploymentCpuServer } const CPU_SERVER_READ_TIMEOUT_MS = 15_000 +/** For callers that already retry on their own schedule. */ +interface ReadOptions { + skipStatusRetry?: boolean +} + +function withReadOptions( + auth: AxiosRequestConfig, + { skipStatusRetry }: ReadOptions +): AxiosRequestConfig { + return skipStatusRetry ? { ...auth, __skipStatusRetry: true } : auth +} + export const workspaceApi = { - async list(): Promise { + async list(options: ReadOptions = {}): Promise { const auth = await requestAuth() try { const response = await workspaceApiClient.get( ingestApiUrl('/workspaces'), - auth + withReadOptions(auth, options) ) return response.data } catch (err) { handleAxiosError(err, 'list') } }, - async getCurrentWorkspace(): Promise { + async getCurrentWorkspace( + options: ReadOptions = {} + ): Promise { const auth = await requestAuth() try { const response = await workspaceApiClient.get( ingestApiUrl('/workspaces/current'), - auth + withReadOptions(auth, options) ) return response.data } catch (err) { @@ -125,7 +141,11 @@ export const workspaceApi = { try { const response = await workspaceApiClient.get( ingestApiUrl(`/workspaces/${workspaceId}/deployments`), - { ...auth, timeout: DEPLOYMENT_LISTING_TIMEOUT_MS } + { + ...auth, + timeout: DEPLOYMENT_LISTING_TIMEOUT_MS, + __skipStatusRetry: true + } ) return response.data } catch (err) { diff --git a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts index 871066597d4..60fe0a371c3 100644 --- a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts +++ b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts @@ -75,6 +75,24 @@ describe('workspaceApi on the web session', () => { expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) }) + it('sends a read once when the caller opts out of the status retry', async () => { + const send = provideSessionSending(unavailable()) + + await expect( + workspaceApi.list({ skipStatusRetry: true }) + ).rejects.toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledTimes(1) + }) + + it('sends the deployment listing once, leaving retries to its boot loop', async () => { + const send = provideSessionSending(unavailable()) + + await expect(workspaceApi.listDeployments('ws-1')).rejects.toMatchObject({ + status: 503 + }) + expect(send).toHaveBeenCalledTimes(1) + }) + it('does not retry a write that got a 503', async () => { const send = provideSessionSending(unavailable()) diff --git a/src/platform/workspace/stores/teamWorkspaceStore.test.ts b/src/platform/workspace/stores/teamWorkspaceStore.test.ts index b0c681e2e48..0ab9896c074 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.test.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.test.ts @@ -239,7 +239,9 @@ describe('useTeamWorkspaceStore', () => { await store.initialize() - expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(1) + expect(mockWorkspaceApi.list).toHaveBeenCalledExactlyOnceWith({ + skipStatusRetry: true + }) expect(store.initState).toBe('ready') expect(store.workspaces).toHaveLength(2) expect(store.activeWorkspaceId).toBe(mockPersonalWorkspace.id) diff --git a/src/platform/workspace/stores/teamWorkspaceStore.ts b/src/platform/workspace/stores/teamWorkspaceStore.ts index 74270cadb5b..772bdcb291f 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.ts @@ -197,6 +197,7 @@ function reportSsoLanding( const MAX_OWNED_WORKSPACES = 10 const MAX_INIT_RETRIES = 3 const BASE_RETRY_DELAY_MS = 1000 +const INIT_READ_WITHOUT_STATUS_RETRY = { skipStatusRetry: true } const MAX_SESSION_REVOKED_RETRIES = 1 @@ -357,7 +358,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { for (let attempt = 0; attempt <= MAX_INIT_RETRIES; attempt++) { try { if (isDesktopHostSignedIn()) { - const response = await workspaceApi.list() + const response = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) const hostWorkspaceId = desktopHostUser.value?.workspaceId @@ -371,7 +374,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { } if (isApiKeySession) { - const current = await workspaceApi.getCurrentWorkspace() + const current = await workspaceApi.getCurrentWorkspace( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return workspaces.value = [createWorkspaceStateFromCredential(current)] mutableActiveWorkspaceId.value = current.id @@ -386,7 +391,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { const hasValidSession = workspaceAuthStore.initializeFromSession() if (hasValidSession && workspaceAuthStore.currentWorkspace) { - const response = await workspaceApi.list() + const response = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) @@ -425,7 +432,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { return } - const response: ListedWorkspaces = await workspaceApi.list() + const response: ListedWorkspaces = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) diff --git a/src/scripts/api.fetchApi.test.ts b/src/scripts/api.fetchApi.test.ts index 688c61d99c1..0d144498953 100644 --- a/src/scripts/api.fetchApi.test.ts +++ b/src/scripts/api.fetchApi.test.ts @@ -19,7 +19,7 @@ import type { WebSessionRequests } from '@/platform/auth/session/webSessionFetch import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' import { useTelemetry } from '@/platform/telemetry' import { useAuthStore } from '@/stores/authStore' -import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' +import { DEFAULT_RETRY_POLICY, NO_RETRY_POLICY } from '@/utils/retryPolicy' const { addBreadcrumb } = vi.hoisted(() => ({ addBreadcrumb: vi.fn() @@ -288,6 +288,45 @@ describe('api.fetchApi', () => { expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) }) + it('rejects with the timeout, not the last 503, when it fires mid-backoff', async () => { + const send = provideSessionSending( + new Response(null, { status: 503, headers: { 'Retry-After': '5' } }) + ) + + const settled = api + .fetchApi('/test', { timeoutMs: 1_000 }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + + expect(await settled).toMatchObject(fetchTimeoutRejection.reason) + expect(send).toHaveBeenCalledOnce() + }) + + it('rejects with the caller reason when it cancels mid-backoff', async () => { + const controller = new AbortController() + const send = provideSessionSending( + new Response(null, { status: 503, headers: { 'Retry-After': '5' } }) + ) + + const settled = api + .fetchApi('/test', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(500) + controller.abort('dialog closed') + + expect(await settled).toBe('dialog closed') + expect(send).toHaveBeenCalledOnce() + }) + + it('sends once on a 503 when skipStatusRetry is set', async () => { + const send = provideSessionSending(unavailable()) + + const response = await api.fetchApi('/test', { skipStatusRetry: true }) + + expect(response.status).toBe(503) + expect(send).toHaveBeenCalledOnce() + }) + it('does not retry a POST', async () => { const send = provideSessionSending(unavailable()) @@ -458,6 +497,28 @@ describe('api.fetchApi', () => { ) expect(global.fetch).not.toHaveBeenCalled() }) + + it.for([ + { skipStatusRetry: true, expected: NO_RETRY_POLICY }, + { skipStatusRetry: undefined, expected: undefined } + ])( + 'passes retry policy $expected when skipStatusRetry is $skipStatusRetry', + async ({ skipStatusRetry, expected }) => { + mockDistribution.isCloud = true + const provider: ApiAuthProvider = { + ...anonymousApiAuthProvider, + fetch: vi.fn().mockResolvedValue(new Response()) + } + api.setAuthProvider(provider) + + await api.fetchApi('/test', { skipStatusRetry }) + + const [, init, , , retryPolicy] = vi.mocked(provider.fetch).mock + .calls[0] + expect(retryPolicy).toBe(expected) + expect(init).not.toHaveProperty('skipStatusRetry') + } + ) }) describe('socketAuthParams', () => { diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 78db2c5d405..251cb5fb6bf 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -85,6 +85,8 @@ import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' import type { AuthCredential, AuthScheme } from '@/types/authTypes' import type { NodeExecutionId } from '@/types/nodeIdentification' import { retryingSend } from '@/utils/fetchWithRetry' +import { NO_RETRY_POLICY } from '@/utils/retryPolicy' +import type { RetryPolicy } from '@/utils/retryPolicy' import { fetchHistory, fetchJobAssets, @@ -155,6 +157,8 @@ interface FetchApiOptions extends RequestInit { onAuthScheme?: (scheme: AuthScheme) => void /** Reports the kind of credential sent, to tell a missing token from a wrong one. */ onAuthCredential?: (credential: AuthCredential) => void + /** Send once, for callers that already retry on their own backoff. */ + skipStatusRetry?: boolean } const FETCH_ROUTE_GROUPS = new Set([ @@ -614,11 +618,16 @@ export class ComfyApi extends EventTarget { this.authProvider = provider } - private async getWebSessionSend(): Promise { + private async getWebSessionSend( + retryPolicy?: RetryPolicy + ): Promise { const requests = webSessionRequests() if (!requests) return undefined const scope = await requests.scope() - return scope && retryingSend((url, init) => requests.send(url, init, scope)) + return ( + scope && + retryingSend((url, init) => requests.send(url, init, scope), retryPolicy) + ) } /** Sends a same-origin request on the web session; undefined when this tab is not on it. */ @@ -670,15 +679,17 @@ export class ComfyApi extends EventTarget { onAuthHeader, onAuthScheme, onAuthCredential, + skipStatusRetry, ...requestOptions } = options ?? {} + const retryPolicy = skipStatusRetry ? NO_RETRY_POLICY : undefined const headers: HeadersInit = requestOptions.headers ?? {} let unifiedRetryOn401 = false let sendOnWebSession: WebSessionSend | undefined if (isCloud) { await this.authProvider.waitForInitialization() - sendOnWebSession = await this.getWebSessionSend() + sendOnWebSession = await this.getWebSessionSend(retryPolicy) if (sendOnWebSession) { onAuthHeader?.(true) onAuthScheme?.('web-session') @@ -778,7 +789,8 @@ export class ComfyApi extends EventTarget { this.apiURL(route), init, unifiedRetryOn401, - retrySignalLifecycle + retrySignalLifecycle, + retryPolicy ) return response.finally(() => { if (timeoutId !== undefined) clearTimeout(timeoutId) @@ -1682,8 +1694,10 @@ export class ComfyApi extends EventTarget { * Gets all setting values for the current user * @returns { Promise } A dictionary of id -> value */ - async getSettings(): Promise { - const resp = await this.fetchApi('/settings') + async getSettings( + options?: Pick + ): Promise { + const resp = await this.fetchApi('/settings', options) if (resp.status == 401) { // `statusText` is ALWAYS empty over HTTP/2 — the protocol carries no diff --git a/src/types/axios-augmentation.d.ts b/src/types/axios-augmentation.d.ts index 101054788cd..b67f043f5d3 100644 --- a/src/types/axios-augmentation.d.ts +++ b/src/types/axios-augmentation.d.ts @@ -8,5 +8,7 @@ declare module 'axios' { // Exempts a deliberately Firebase-authed request (acceptInvite) from the // unified re-mint. __skipUnifiedRemint?: boolean + /** Sends once, skipping the shared 429/502/503/504 retry. */ + __skipStatusRetry?: boolean } } diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts index 041f3533736..79935b32b32 100644 --- a/src/utils/axiosRetryableStatus.test.ts +++ b/src/utils/axiosRetryableStatus.test.ts @@ -113,6 +113,17 @@ describe('attachRetryableStatusRetry', () => { await expect(client.get('https://api/x')).rejects.toBe(networkError) expect(adapter).toHaveBeenCalledTimes(1) }) + + it('sends once when the request opts out of the status retry', async () => { + const { client, adapter, sleep } = makeClient([{ status: 503 }]) + + await expect( + client.get('https://api/x', { __skipStatusRetry: true }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + it('retries through an adapter the request selects for itself', async () => { const { client, adapter: defaultAdapter } = makeClient([{ status: 200 }]) const requestAdapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) @@ -213,6 +224,23 @@ describe('attachRetryableStatusRetry', () => { expect(axios.isCancel(await outcome)).toBe(true) }) + it('reports a timeout signal fired during the wait as a cancellation, not a stale 503', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const controller = new AbortController() + + const outcome = client + .get('https://api/x', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort(new DOMException('Fetch timeout', 'TimeoutError')) + + expect(axios.isCancel(await outcome)).toBe(true) + expect(adapter).toHaveBeenCalledTimes(1) + }) + it('reports a cancel token fired during the wait as a cancellation', async () => { vi.useFakeTimers() const { client } = makeTimedClient([ diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index 2748deac51d..4e644cec6cb 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -97,6 +97,7 @@ function retryingAdapter( } return async (config) => { + if (config.__skipStatusRetry) return adapter(config) const remainingMs = timeBudget(config.timeout) let lastError: unknown diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts index b2b8eb3c399..82ccbf4fc5b 100644 --- a/src/utils/fetchWithRetry.test.ts +++ b/src/utils/fetchWithRetry.test.ts @@ -224,20 +224,38 @@ describe('fetchWithRetry', () => { expect(fetchImpl).toHaveBeenCalledTimes(2) }) - it('returns the last response when the timeout signal fires mid-wait', async () => { + it('rejects with the timeout reason when the timeout signal fires mid-wait', async () => { const controller = new AbortController() - const last = respond(503, { 'Retry-After': '5' }) - const fetchImpl = scriptedFetch(last) + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '5' })) + const timeout = new DOMException('Fetch timeout', 'TimeoutError') const pending = fetchWithRetry( 'https://api/x', { signal: controller.signal }, { fetchImpl } ) + const outcome = pending.catch((reason: unknown) => reason) await vi.advanceTimersByTimeAsync(1_000) - controller.abort(new DOMException('Fetch timeout', 'TimeoutError')) + controller.abort(timeout) + + expect(await outcome).toBe(timeout) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it('rejects when the signal was already aborted before the wait began', async () => { + const controller = new AbortController() + const fetchImpl = vi.fn(async () => { + controller.abort('gone') + return respond(503) + }) - expect(await pending).toBe(last) + await expect( + fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + ).rejects.toBe('gone') expect(fetchImpl).toHaveBeenCalledTimes(1) }) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts index a3773279abf..aa2164317f8 100644 --- a/src/utils/fetchWithRetry.ts +++ b/src/utils/fetchWithRetry.ts @@ -28,14 +28,6 @@ function requestSignal( return init.signal ?? (input instanceof Request ? input.signal : undefined) } -function timedOut(signal: AbortSignal | undefined): boolean { - return ( - signal?.aborted === true && - signal.reason instanceof DOMException && - signal.reason.name === 'TimeoutError' - ) -} - function isReplayable(input: RequestInfo | URL, init: RequestInit): boolean { return ( !(init.body instanceof ReadableStream) && @@ -70,22 +62,19 @@ export async function fetchWithRetry( random }) if (delay === undefined) return response - try { - await sleep(delay, signal) - } catch (error) { - if (timedOut(signal)) return response - throw error - } + await sleep(delay, signal) await response.body?.cancel() } } /** Wraps a url-based send so it retries like {@link fetchWithRetry}. */ export function retryingSend( - send: (url: string, init: RequestInit) => Promise + send: (url: string, init: RequestInit) => Promise, + policy?: RetryPolicy ): (url: string, init: RequestInit) => Promise { return (url, init) => fetchWithRetry(url, init, { + policy, fetchImpl: (input, attemptInit) => send(String(input), attemptInit ?? {}) }) } diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts index f8bd66be4e4..56d55fdb238 100644 --- a/src/utils/retryPolicy.ts +++ b/src/utils/retryPolicy.ts @@ -12,6 +12,12 @@ export const DEFAULT_RETRY_POLICY: RetryPolicy = { retryNonIdempotent: false } +/** Sends once: for callers that already retry on their own backoff. */ +export const NO_RETRY_POLICY: RetryPolicy = { + ...DEFAULT_RETRY_POLICY, + maxAttempts: 1 +} + const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]) const IDEMPOTENT_METHODS: ReadonlySet = new Set([ 'GET',