diff --git a/src/platform/auth/apiAuthProvider.ts b/src/platform/auth/apiAuthProvider.ts index 10294727fdc..31ed8614f6b 100644 --- a/src/platform/auth/apiAuthProvider.ts +++ b/src/platform/auth/apiAuthProvider.ts @@ -1,4 +1,5 @@ import type { AuthHeader } from '@/types/authTypes' +import type { RetryPolicy } from '@/utils/retryPolicy' export interface RetrySignalLifecycle { clearInitialTimeout: () => void @@ -19,7 +20,8 @@ export interface ApiAuthProvider { input: RequestInfo | URL, init: RequestInit, retryOn401: boolean, - retrySignalLifecycle?: RetrySignalLifecycle + retrySignalLifecycle?: RetrySignalLifecycle, + retryPolicy?: RetryPolicy ): Promise } diff --git a/src/platform/auth/unified/remintRetry.test.ts b/src/platform/auth/unified/remintRetry.test.ts index 8a0b0c44763..03aa7a430fb 100644 --- a/src/platform/auth/unified/remintRetry.test.ts +++ b/src/platform/auth/unified/remintRetry.test.ts @@ -14,6 +14,8 @@ import { attachUnifiedRemintInterceptor, fetchWithUnifiedRemint } from '@/platform/auth/unified/remintRetry' +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' +import { NO_RETRY_POLICY } from '@/utils/retryPolicy' vi.mock(import('@/platform/telemetry')) @@ -119,6 +121,84 @@ describe('fetchWithUnifiedRemint', () => { expect(useTelemetry()?.trackUnifiedAuthRetry).not.toHaveBeenCalled() }) + describe('with retryable server failures', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + const unavailable = () => + new Response(null, { status: 503, headers: { 'Retry-After': '1' } }) + + it('retries a 503 without re-minting', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(ok) + + const pending = fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true + ) + await vi.advanceTimersByTimeAsync(1_000) + + expect(await pending).toBe(ok) + expect(fetch).toHaveBeenCalledTimes(2) + expect(useWorkspaceAuthStore().remintUnifiedOnce).not.toHaveBeenCalled() + }) + + it('gives the re-minted request its own retry budget', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unauthorized) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(unavailable()) + .mockResolvedValueOnce(ok) + vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( + 'tokenB' + ) + + const pending = fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true + ) + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toBe(ok) + expect(fetch).toHaveBeenCalledTimes(6) + expect(useWorkspaceAuthStore().remintUnifiedOnce).toHaveBeenCalledTimes(1) + }) + + it('sends once on a 503 under the no-retry policy', async () => { + vi.mocked(fetch).mockResolvedValueOnce(unavailable()) + + const result = await fetchWithUnifiedRemint( + 'https://cloud/x', + { headers: { Authorization: 'Bearer tokenA' as const } }, + true, + undefined, + NO_RETRY_POLICY + ) + + expect(result.status).toBe(503) + expect(fetch).toHaveBeenCalledTimes(1) + }) + + it('does not retry a POST that got a 503', async () => { + vi.mocked(fetch).mockResolvedValueOnce(unavailable()) + + const result = await fetchWithUnifiedRemint( + 'https://cloud/x', + { method: 'POST', body: '{}' }, + true + ) + + expect(result.status).toBe(503) + expect(fetch).toHaveBeenCalledTimes(1) + }) + }) + it('does not retry a non-401 response', async () => { const serverError = { status: 500 } as Response vi.mocked(fetch).mockResolvedValueOnce(serverError) @@ -428,7 +508,7 @@ describe('attachUnifiedRemintInterceptor', () => { // A custom axios adapter is responsible for its own status handling (axios // only applies validateStatus inside its built-in adapters), so reject // non-2xx with a real AxiosError to mirror a live response. - function makeAdapter(statuses: number[]): ReturnType { + function makeAdapter(statuses: number[]) { let call = 0 return vi.fn(async (config) => { const status = statuses[Math.min(call, statuses.length - 1)] @@ -455,7 +535,7 @@ describe('attachUnifiedRemintInterceptor', () => { function makeClient(statuses: number[]) { const adapter = makeAdapter(statuses) - const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + const client = axios.create({ adapter }) attachUnifiedRemintInterceptor(client) return { client, adapter } } @@ -512,6 +592,24 @@ describe('attachUnifiedRemintInterceptor', () => { }) }) + it('keeps the 503 retries and the 401 re-mint on separate budgets', async () => { + const adapter = makeAdapter([503, 503, 401, 503, 200]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { sleep: async () => {} }) + attachUnifiedRemintInterceptor(client) + vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( + 'tokenB' + ) + + const res = await client.get('https://cloud/x', { + headers: { Authorization: 'Bearer tokenA' as const } + }) + + expect(res.status).toBe(200) + expect(adapter).toHaveBeenCalledTimes(5) + expect(useWorkspaceAuthStore().remintUnifiedOnce).toHaveBeenCalledTimes(1) + }) + it('does not re-mint when the flag is OFF (AC3)', async () => { vi.mocked(useFeatureFlags().flags).unifiedCloudAuthEnabled = false const { client, adapter } = makeClient([401]) @@ -613,7 +711,7 @@ describe('attachUnifiedRemintInterceptor', () => { response ) }) - const client = axios.create({ adapter: adapter as unknown as AxiosAdapter }) + const client = axios.create({ adapter }) attachUnifiedRemintInterceptor(client) vi.mocked(useWorkspaceAuthStore().remintUnifiedOnce).mockResolvedValue( 'tokenB' diff --git a/src/platform/auth/unified/remintRetry.ts b/src/platform/auth/unified/remintRetry.ts index d3075fe7327..5636e45a238 100644 --- a/src/platform/auth/unified/remintRetry.ts +++ b/src/platform/auth/unified/remintRetry.ts @@ -18,6 +18,8 @@ import type { UnifiedAuthRetryFailureReason, UnifiedAuthRetryMetadata } from '@/platform/telemetry/types' +import { fetchWithRetry } from '@/utils/fetchWithRetry' +import type { RetryPolicy } from '@/utils/retryPolicy' let cachedUnifiedFlags: | { readonly unifiedCloudAuthEnabled: boolean } @@ -146,13 +148,14 @@ export async function fetchWithUnifiedRemint( input: RequestInfo | URL, init: RequestInit, shouldRetryOn401: boolean, - retrySignalLifecycle?: RetrySignalLifecycle + retrySignalLifecycle?: RetrySignalLifecycle, + retryPolicy?: RetryPolicy ): Promise { const retryInput = shouldRetryOn401 && input instanceof Request && input.body !== null ? input.clone() : input - const response = await fetch(input, init) + const response = await fetchWithRetry(input, init, { policy: retryPolicy }) if (response.status === 403 && isCloud) { await presentSsoRequired(() => presentForResponse(response)) } @@ -188,7 +191,9 @@ export async function fetchWithUnifiedRemint( ? { ...init, headers, signal: retrySignalLifecycle.createSignal() } : { ...init, headers } try { - const retryResponse = await fetch(retryInput, retryInit) + const retryResponse = await fetchWithRetry(retryInput, retryInit, { + policy: retryPolicy + }) trackRetryResponse('fetch', retryResponse.status) return retryResponse } catch (error) { diff --git a/src/platform/billing/sdk/billingSdkStore.retry.test.ts b/src/platform/billing/sdk/billingSdkStore.retry.test.ts new file mode 100644 index 00000000000..c48a58a5b06 --- /dev/null +++ b/src/platform/billing/sdk/billingSdkStore.retry.test.ts @@ -0,0 +1,71 @@ +import { CREDITS_ROUTE } from '@comfyorg/account-core/billing' +import type { + AccountCredential, + SessionSnapshot +} from '@comfyorg/account-core/session' +import { fromPartial } from '@total-typescript/shoehorn' +import type { User } from 'firebase/auth' +import { beforeEach, expect, it, vi } from 'vitest' + +import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' +import { + stubAccountIdentityPort, + stubFirebaseAuthHarness +} from '@/utils/__tests__/stubAccountIdentityPort' + +import { useBillingSdkStore } from './billingSdkStore' + +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@/platform/billing/composables/useBillingCapabilities')) +vi.mock(import('@/platform/settings/composables/useSettingsDialog')) +vi.mock(import('firebase/auth')) + +const CREDENTIAL: AccountCredential = { + token: 'workspace-jwt', + expiresAt: Date.now() + 60 * 60 * 1000, + uid: 'uid-1', + workspace: { id: 'ws-1', name: 'Personal', type: 'personal' }, + role: 'owner', + permissions: ['workspace:read'] +} + +const SNAPSHOT: SessionSnapshot = { + phase: 'authenticated', + user: fromPartial({ uid: 'uid-1' }), + session: CREDENTIAL +} + +beforeEach(() => { + stubAccountIdentityPort() + stubFirebaseAuthHarness() + const session = useWorkspaceAuthStore().getUnifiedSessionClient() + vi.spyOn(session, 'getSnapshot').mockReturnValue(SNAPSHOT) + vi.spyOn(session, 'ensureFresh').mockResolvedValue({ + status: 'ok', + session: CREDENTIAL + }) +}) + +it('reads the balance again after a 503 through the store wiring', async () => { + vi.useFakeTimers() + const answers = [ + new Response(null, { status: 503 }), + Response.json({ amount_micros: 1_000_000, currency: 'USD' }) + ] + vi.mocked(fetch).mockImplementation(async (input) => { + expect(String(input)).toContain(CREDITS_ROUTE) + const answer = answers.shift() + if (!answer) throw new Error('fetch called more times than scripted') + return answer + }) + + const pending = useBillingSdkStore().readBalance() + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toMatchObject({ + status: 'ok', + value: { amount_micros: 1_000_000 } + }) + expect(fetch).toHaveBeenCalledTimes(2) +}) diff --git a/src/platform/billing/sdk/billingSdkStore.ts b/src/platform/billing/sdk/billingSdkStore.ts index b5496c1ea5f..69731af31d2 100644 --- a/src/platform/billing/sdk/billingSdkStore.ts +++ b/src/platform/billing/sdk/billingSdkStore.ts @@ -61,6 +61,7 @@ import { useWorkspaceIdentity } from '@/platform/workspace/ports/workspaceIdenti import { useWorkspaceSession } from '@/platform/workspace/ports/workspaceSession' import { getCheckoutJourneyPaymentIntentSource } from '@/platform/billing/utils/checkoutJourney' import { useDialogStore } from '@/stores/dialogStore' +import { fetchWithRetry } from '@/utils/fetchWithRetry' import { projectBillingCapabilities } from './billingCapabilitiesView' import { projectBillingPlans } from './billingPlansView' @@ -144,6 +145,7 @@ export const useBillingSdkStore = defineStore('billingSdk', () => { const sdk = createBillingSdk({ ...sessionPorts(), resolveUrl: ingestApiUrl, + fetchImpl: fetchWithRetry, pointerStorage: sessionStorage, hostedDestination: () => flags.hostedBillingDestination, onTelemetry: reportTelemetry diff --git a/src/platform/settings/globalSettingsApi.test.ts b/src/platform/settings/globalSettingsApi.test.ts index 2f515080f3d..3c3a1016440 100644 --- a/src/platform/settings/globalSettingsApi.test.ts +++ b/src/platform/settings/globalSettingsApi.test.ts @@ -3,6 +3,7 @@ vi.mock(import('@/services/dialogService')) import type { GlobalSetting } from '@comfyorg/ingest-types' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { WebSessionSend } from '@/platform/auth/session/webSessionFetch' import { api } from '@/scripts/api' import { useAuthStore } from '@/stores/authStore' @@ -159,4 +160,28 @@ describe('Global Settings transport', () => { }) } ) + describe('on the web session', () => { + const unavailable = () => new Response(null, { status: 503 }) + + it('reads again after a 503 with exactly two sends', async () => { + vi.useFakeTimers() + const answers = [unavailable(), Response.json(stored)] + const send = vi.fn(async () => answers.shift()!) + + const pending = getGlobalSetting(key, send) + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual(stored) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('does not send a write twice after a 503', async () => { + const send = vi.fn(async () => unavailable()) + + await expect( + setGlobalSetting({ key, value: true }, send) + ).rejects.toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledOnce() + }) + }) }) diff --git a/src/platform/settings/globalSettingsApi.ts b/src/platform/settings/globalSettingsApi.ts index 905c5d458d5..be7038d04cd 100644 --- a/src/platform/settings/globalSettingsApi.ts +++ b/src/platform/settings/globalSettingsApi.ts @@ -15,6 +15,7 @@ import { api } from '@/scripts/api' import type { WebSessionSend } from '@/platform/auth/session/webSessionFetch' import { useAuthStore } from '@/stores/authStore' import type { AuthHeader } from '@/types/authTypes' +import { retryingSend } from '@/utils/fetchWithRetry' export class GlobalSettingsApiError extends Error { constructor( @@ -57,7 +58,9 @@ function sendGlobalSetting( auth: GlobalSettingsAuth, headers: Record = {} ): Promise { - if (typeof auth === 'function') return auth(url, { ...init, headers }) + if (typeof auth === 'function') { + return retryingSend(auth)(url, { ...init, headers }) + } return shouldRemintCloudRequest().then((remint) => fetchWithUnifiedRemint( url, diff --git a/src/platform/settings/settingStore.test.ts b/src/platform/settings/settingStore.test.ts index a50cbbb652c..c5a1bf248c6 100644 --- a/src/platform/settings/settingStore.test.ts +++ b/src/platform/settings/settingStore.test.ts @@ -50,6 +50,14 @@ describe('useSettingStore', () => { expect(api.getSettings).toHaveBeenCalled() }) + it('loads settings without the shared status retry stacking on its own backoff', async () => { + vi.mocked(api.getSettings).mockResolvedValue({} as Settings) + + await store.load() + + expect(api.getSettings).toHaveBeenCalledWith({ skipStatusRetry: true }) + }) + describe('Canvas Navigation override migration', () => { const NAV = 'Comfy.Canvas.NavigationMode' const LEFT = 'Comfy.Canvas.LeftMouseClickBehavior' diff --git a/src/platform/settings/settingStore.ts b/src/platform/settings/settingStore.ts index 87521a2ec6c..a097ffd56ca 100644 --- a/src/platform/settings/settingStore.ts +++ b/src/platform/settings/settingStore.ts @@ -139,10 +139,13 @@ export const useSettingStore = defineStore('setting', () => { 'Setting values must be loaded before any setting is registered.' ) } - settingValues.value = await retry(() => api.getSettings(), { - retries: 3, - delay: (attempt) => Math.min(1000 * Math.pow(2, attempt), 8000) - }) + settingValues.value = await retry( + () => api.getSettings({ skipStatusRetry: true }), + { + retries: 3, + delay: (attempt) => Math.min(1000 * Math.pow(2, attempt), 8000) + } + ) await migrateZoomThresholdToFontSize() await migrateCanvasNavigationOverrides() }, diff --git a/src/platform/workspace/api/workspaceApi.test.ts b/src/platform/workspace/api/workspaceApi.test.ts index 5c1072b1772..c41824ad973 100644 --- a/src/platform/workspace/api/workspaceApi.test.ts +++ b/src/platform/workspace/api/workspaceApi.test.ts @@ -14,7 +14,7 @@ const { mockAxiosInstance } = vi.hoisted(() => ({ put: vi.fn(), patch: vi.fn(), delete: vi.fn(), - interceptors: { response: { use: vi.fn() } } + interceptors: { request: { use: vi.fn() }, response: { use: vi.fn() } } } })) @@ -355,7 +355,7 @@ describe('workspaceApi', () => { }) describe('deployment picking', () => { - it('listDeployments() sends GET /workspaces/:id/deployments with a 30 second timeout', async () => { + it('listDeployments() sends GET /workspaces/:id/deployments with a 30 second timeout and no status retry', async () => { const data = { items: [], builds_visible: true } mockAxiosInstance.get.mockResolvedValue({ data }) @@ -363,7 +363,7 @@ describe('workspaceApi', () => { expect(mockAxiosInstance.get).toHaveBeenCalledWith( '/api/workspaces/ws-1/deployments', - { headers: AUTH_HEADER, timeout: 30_000 } + { headers: AUTH_HEADER, timeout: 30_000, __skipStatusRetry: true } ) expect(result).toEqual(data) }) diff --git a/src/platform/workspace/api/workspaceApi.ts b/src/platform/workspace/api/workspaceApi.ts index 948d5d12cd9..b2460c16b98 100644 --- a/src/platform/workspace/api/workspaceApi.ts +++ b/src/platform/workspace/api/workspaceApi.ts @@ -1,3 +1,5 @@ +import type { AxiosRequestConfig } from 'axios' + import type { AcceptInviteResponse, CreateInviteRequest, @@ -64,25 +66,39 @@ export type { WorkspaceDeploymentCpuServer } const CPU_SERVER_READ_TIMEOUT_MS = 15_000 +/** For callers that already retry on their own schedule. */ +interface ReadOptions { + skipStatusRetry?: boolean +} + +function withReadOptions( + auth: AxiosRequestConfig, + { skipStatusRetry }: ReadOptions +): AxiosRequestConfig { + return skipStatusRetry ? { ...auth, __skipStatusRetry: true } : auth +} + export const workspaceApi = { - async list(): Promise { + async list(options: ReadOptions = {}): Promise { const auth = await requestAuth() try { const response = await workspaceApiClient.get( ingestApiUrl('/workspaces'), - auth + withReadOptions(auth, options) ) return response.data } catch (err) { handleAxiosError(err, 'list') } }, - async getCurrentWorkspace(): Promise { + async getCurrentWorkspace( + options: ReadOptions = {} + ): Promise { const auth = await requestAuth() try { const response = await workspaceApiClient.get( ingestApiUrl('/workspaces/current'), - auth + withReadOptions(auth, options) ) return response.data } catch (err) { @@ -125,7 +141,11 @@ export const workspaceApi = { try { const response = await workspaceApiClient.get( ingestApiUrl(`/workspaces/${workspaceId}/deployments`), - { ...auth, timeout: DEPLOYMENT_LISTING_TIMEOUT_MS } + { + ...auth, + timeout: DEPLOYMENT_LISTING_TIMEOUT_MS, + __skipStatusRetry: true + } ) return response.data } catch (err) { diff --git a/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts new file mode 100644 index 00000000000..60fe0a371c3 --- /dev/null +++ b/src/platform/workspace/api/workspaceApi.webSessionRetry.test.ts @@ -0,0 +1,104 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' + +import type { + WebSessionRequests, + WebSessionRequestScope +} from '@/platform/auth/session/webSessionFetch' +import { provideWebSessionRequests } from '@/platform/auth/session/webSessionFetch' +import { stubFirebaseAuthHarness } from '@/utils/__tests__/stubAccountIdentityPort' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' + +import { workspaceApi } from './workspaceApi' +import { installWorkspaceApiAuth } from './workspaceApiAuth' + +vi.mock(import('@/i18n')) + +vi.mock(import('@/platform/remote/ingest/ingestApiUrl'), () => ({ + ingestApiUrl: (path: string) => `/api${path}` +})) + +vi.mock(import('firebase/auth'), { spy: true }) + +beforeEach(() => { + stubFirebaseAuthHarness() +}) + +installWorkspaceApiAuth() + +function provideSessionSending(...answers: Response[]) { + const send = vi.fn(async () => { + const answer = answers.shift() + if (!answer) throw new Error('send called more times than scripted') + return answer + }) + const release = provideWebSessionRequests( + fromPartial({ + scope: async () => fromPartial({}), + send + }) + ) + onTestFinished(release) + return send +} + +const unavailable = (headers?: Record) => + new Response('{"message":"unavailable"}', { status: 503, headers }) + +describe('workspaceApi on the web session', () => { + it('recovers from a 503 on the session adapter', async () => { + vi.useFakeTimers() + const send = provideSessionSending( + unavailable(), + new Response('{"workspaces":[]}', { status: 200 }) + ) + + const pending = workspaceApi.list() + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual({ workspaces: [] }) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('sends no more than the policy attempt count on a persistent 503', async () => { + vi.useFakeTimers() + const send = provideSessionSending( + unavailable(), + unavailable(), + unavailable() + ) + + const pending = workspaceApi.list().catch((error: unknown) => error) + await vi.advanceTimersByTimeAsync(10_000) + + expect(await pending).toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it('sends a read once when the caller opts out of the status retry', async () => { + const send = provideSessionSending(unavailable()) + + await expect( + workspaceApi.list({ skipStatusRetry: true }) + ).rejects.toMatchObject({ status: 503 }) + expect(send).toHaveBeenCalledTimes(1) + }) + + it('sends the deployment listing once, leaving retries to its boot loop', async () => { + const send = provideSessionSending(unavailable()) + + await expect(workspaceApi.listDeployments('ws-1')).rejects.toMatchObject({ + status: 503 + }) + expect(send).toHaveBeenCalledTimes(1) + }) + + it('does not retry a write that got a 503', async () => { + const send = provideSessionSending(unavailable()) + + await expect(workspaceApi.create({ name: 'Team' })).rejects.toMatchObject({ + status: 503 + }) + expect(send).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/platform/workspace/api/workspaceApiAuth.ts b/src/platform/workspace/api/workspaceApiAuth.ts index dd0052c56a2..434c055c4eb 100644 --- a/src/platform/workspace/api/workspaceApiAuth.ts +++ b/src/platform/workspace/api/workspaceApiAuth.ts @@ -1,12 +1,16 @@ import { attachUnifiedRemintInterceptor } from '@/platform/auth/unified/remintRetry' import { setIngestApiAuth } from '@/platform/remote/ingest/ingestApi' import { useAuthStore } from '@/stores/authStore' +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' export function installWorkspaceApiAuth(): void { setIngestApiAuth({ getWorkspaceAuthHeader: () => useAuthStore().getWorkspaceAuthHeaderOrThrow(), getFirebaseAuthHeader: () => useAuthStore().getFirebaseAuthHeaderOrThrow(), - attachRetryInterceptor: attachUnifiedRemintInterceptor + attachRetryInterceptor: (client) => { + attachRetryableStatusRetry(client) + attachUnifiedRemintInterceptor(client) + } }) } diff --git a/src/platform/workspace/stores/teamWorkspaceStore.test.ts b/src/platform/workspace/stores/teamWorkspaceStore.test.ts index b0c681e2e48..0ab9896c074 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.test.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.test.ts @@ -239,7 +239,9 @@ describe('useTeamWorkspaceStore', () => { await store.initialize() - expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(1) + expect(mockWorkspaceApi.list).toHaveBeenCalledExactlyOnceWith({ + skipStatusRetry: true + }) expect(store.initState).toBe('ready') expect(store.workspaces).toHaveLength(2) expect(store.activeWorkspaceId).toBe(mockPersonalWorkspace.id) diff --git a/src/platform/workspace/stores/teamWorkspaceStore.ts b/src/platform/workspace/stores/teamWorkspaceStore.ts index 74270cadb5b..772bdcb291f 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.ts @@ -197,6 +197,7 @@ function reportSsoLanding( const MAX_OWNED_WORKSPACES = 10 const MAX_INIT_RETRIES = 3 const BASE_RETRY_DELAY_MS = 1000 +const INIT_READ_WITHOUT_STATUS_RETRY = { skipStatusRetry: true } const MAX_SESSION_REVOKED_RETRIES = 1 @@ -357,7 +358,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { for (let attempt = 0; attempt <= MAX_INIT_RETRIES; attempt++) { try { if (isDesktopHostSignedIn()) { - const response = await workspaceApi.list() + const response = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) const hostWorkspaceId = desktopHostUser.value?.workspaceId @@ -371,7 +374,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { } if (isApiKeySession) { - const current = await workspaceApi.getCurrentWorkspace() + const current = await workspaceApi.getCurrentWorkspace( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return workspaces.value = [createWorkspaceStateFromCredential(current)] mutableActiveWorkspaceId.value = current.id @@ -386,7 +391,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { const hasValidSession = workspaceAuthStore.initializeFromSession() if (hasValidSession && workspaceAuthStore.currentWorkspace) { - const response = await workspaceApi.list() + const response = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) @@ -425,7 +432,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { return } - const response: ListedWorkspaces = await workspaceApi.list() + const response: ListedWorkspaces = await workspaceApi.list( + INIT_READ_WITHOUT_STATUS_RETRY + ) if (isStaleIdentity(generation)) return applyWorkspaceList(response) diff --git a/src/scripts/api.fetchApi.test.ts b/src/scripts/api.fetchApi.test.ts index 055d7ccbb1b..0d144498953 100644 --- a/src/scripts/api.fetchApi.test.ts +++ b/src/scripts/api.fetchApi.test.ts @@ -1,5 +1,13 @@ import { fromPartial } from '@total-typescript/shoehorn' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + afterEach, + beforeEach, + describe, + expect, + it, + onTestFinished, + vi +} from 'vitest' import type { useFeatureFlags } from '@/composables/useFeatureFlags' import type { ApiAuthProvider } from '@/platform/auth/apiAuthProvider' @@ -11,6 +19,7 @@ import type { WebSessionRequests } from '@/platform/auth/session/webSessionFetch import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' import { useTelemetry } from '@/platform/telemetry' import { useAuthStore } from '@/stores/authStore' +import { DEFAULT_RETRY_POLICY, NO_RETRY_POLICY } from '@/utils/retryPolicy' const { addBreadcrumb } = vi.hoisted(() => ({ addBreadcrumb: vi.fn() @@ -227,6 +236,107 @@ describe('api.fetchApi', () => { expect(send).toHaveBeenCalledOnce() }) + describe('on the session with a retryable failure', () => { + function provideSessionSending(...answers: Response[]) { + const send = vi.fn(async () => { + const answer = answers.shift() + if (!answer) throw new Error('send called more times than scripted') + return answer + }) + onTestFinished( + provideWebSessionRequests( + fromPartial({ + scope: async () => ({ session: fromPartial({}), epoch: 1 }), + send + }) + ) + ) + return send + } + + const unavailable = () => new Response(null, { status: 503 }) + + beforeEach(() => { + signInOnCloud() + vi.useFakeTimers() + }) + + it('recovers from a 503 with exactly two sends', async () => { + const send = provideSessionSending( + unavailable(), + new Response('ok', { status: 200 }) + ) + + const pending = api.fetchApi('/test') + await vi.advanceTimersByTimeAsync(10_000) + + expect((await pending).status).toBe(200) + expect(send).toHaveBeenCalledTimes(2) + }) + + it('stops at the policy attempt count on a persistent 503', async () => { + const send = provideSessionSending( + unavailable(), + unavailable(), + unavailable() + ) + + const pending = api.fetchApi('/test') + await vi.advanceTimersByTimeAsync(10_000) + + expect((await pending).status).toBe(503) + expect(send).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it('rejects with the timeout, not the last 503, when it fires mid-backoff', async () => { + const send = provideSessionSending( + new Response(null, { status: 503, headers: { 'Retry-After': '5' } }) + ) + + const settled = api + .fetchApi('/test', { timeoutMs: 1_000 }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + + expect(await settled).toMatchObject(fetchTimeoutRejection.reason) + expect(send).toHaveBeenCalledOnce() + }) + + it('rejects with the caller reason when it cancels mid-backoff', async () => { + const controller = new AbortController() + const send = provideSessionSending( + new Response(null, { status: 503, headers: { 'Retry-After': '5' } }) + ) + + const settled = api + .fetchApi('/test', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(500) + controller.abort('dialog closed') + + expect(await settled).toBe('dialog closed') + expect(send).toHaveBeenCalledOnce() + }) + + it('sends once on a 503 when skipStatusRetry is set', async () => { + const send = provideSessionSending(unavailable()) + + const response = await api.fetchApi('/test', { skipStatusRetry: true }) + + expect(response.status).toBe(503) + expect(send).toHaveBeenCalledOnce() + }) + + it('does not retry a POST', async () => { + const send = provideSessionSending(unavailable()) + + const response = await api.fetchApi('/test', { method: 'POST' }) + + expect(response.status).toBe(503) + expect(send).toHaveBeenCalledOnce() + }) + }) + it('reports cloud-auth-header when a header was attached', async () => { signInOnCloud() vi.mocked(useAuthStore().getAuthHeader).mockResolvedValue({ @@ -387,6 +497,28 @@ describe('api.fetchApi', () => { ) expect(global.fetch).not.toHaveBeenCalled() }) + + it.for([ + { skipStatusRetry: true, expected: NO_RETRY_POLICY }, + { skipStatusRetry: undefined, expected: undefined } + ])( + 'passes retry policy $expected when skipStatusRetry is $skipStatusRetry', + async ({ skipStatusRetry, expected }) => { + mockDistribution.isCloud = true + const provider: ApiAuthProvider = { + ...anonymousApiAuthProvider, + fetch: vi.fn().mockResolvedValue(new Response()) + } + api.setAuthProvider(provider) + + await api.fetchApi('/test', { skipStatusRetry }) + + const [, init, , , retryPolicy] = vi.mocked(provider.fetch).mock + .calls[0] + expect(retryPolicy).toBe(expected) + expect(init).not.toHaveProperty('skipStatusRetry') + } + ) }) describe('socketAuthParams', () => { diff --git a/src/scripts/api.ts b/src/scripts/api.ts index a6a42125478..251cb5fb6bf 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -84,6 +84,9 @@ import type { import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' import type { AuthCredential, AuthScheme } from '@/types/authTypes' import type { NodeExecutionId } from '@/types/nodeIdentification' +import { retryingSend } from '@/utils/fetchWithRetry' +import { NO_RETRY_POLICY } from '@/utils/retryPolicy' +import type { RetryPolicy } from '@/utils/retryPolicy' import { fetchHistory, fetchJobAssets, @@ -154,6 +157,8 @@ interface FetchApiOptions extends RequestInit { onAuthScheme?: (scheme: AuthScheme) => void /** Reports the kind of credential sent, to tell a missing token from a wrong one. */ onAuthCredential?: (credential: AuthCredential) => void + /** Send once, for callers that already retry on their own backoff. */ + skipStatusRetry?: boolean } const FETCH_ROUTE_GROUPS = new Set([ @@ -613,11 +618,16 @@ export class ComfyApi extends EventTarget { this.authProvider = provider } - private async getWebSessionSend(): Promise { + private async getWebSessionSend( + retryPolicy?: RetryPolicy + ): Promise { const requests = webSessionRequests() if (!requests) return undefined const scope = await requests.scope() - return scope && ((url, init) => requests.send(url, init, scope)) + return ( + scope && + retryingSend((url, init) => requests.send(url, init, scope), retryPolicy) + ) } /** Sends a same-origin request on the web session; undefined when this tab is not on it. */ @@ -669,15 +679,17 @@ export class ComfyApi extends EventTarget { onAuthHeader, onAuthScheme, onAuthCredential, + skipStatusRetry, ...requestOptions } = options ?? {} + const retryPolicy = skipStatusRetry ? NO_RETRY_POLICY : undefined const headers: HeadersInit = requestOptions.headers ?? {} let unifiedRetryOn401 = false let sendOnWebSession: WebSessionSend | undefined if (isCloud) { await this.authProvider.waitForInitialization() - sendOnWebSession = await this.getWebSessionSend() + sendOnWebSession = await this.getWebSessionSend(retryPolicy) if (sendOnWebSession) { onAuthHeader?.(true) onAuthScheme?.('web-session') @@ -777,7 +789,8 @@ export class ComfyApi extends EventTarget { this.apiURL(route), init, unifiedRetryOn401, - retrySignalLifecycle + retrySignalLifecycle, + retryPolicy ) return response.finally(() => { if (timeoutId !== undefined) clearTimeout(timeoutId) @@ -1681,8 +1694,10 @@ export class ComfyApi extends EventTarget { * Gets all setting values for the current user * @returns { Promise } A dictionary of id -> value */ - async getSettings(): Promise { - const resp = await this.fetchApi('/settings') + async getSettings( + options?: Pick + ): Promise { + const resp = await this.fetchApi('/settings', options) if (resp.status == 401) { // `statusText` is ALWAYS empty over HTTP/2 — the protocol carries no diff --git a/src/stores/authStore.test.ts b/src/stores/authStore.test.ts index c358022b799..799e8aa67cf 100644 --- a/src/stores/authStore.test.ts +++ b/src/stores/authStore.test.ts @@ -338,6 +338,53 @@ describe('useAuthStore', () => { }) }) }) + + it('keeps the status and code on the thrown error', async () => { + respondToFetch(BALANCE_URL, () => + Response.json( + { error: 'SERVICE_UNAVAILABLE', message: 'try again' }, + { status: 503, headers: { 'Retry-After': '3600' } } + ) + ) + + await expect(store.fetchBalance()).rejects.toMatchObject({ + name: 'AuthStoreError', + status: 503, + code: 'SERVICE_UNAVAILABLE' + }) + }) + + it('reports an unknown code when the failure body is not JSON', async () => { + respondToFetch( + BALANCE_URL, + () => + new Response('upstream connect error', { + status: 503, + headers: { 'Retry-After': '3600' } + }) + ) + + await expect(store.fetchBalance()).rejects.toMatchObject({ + status: 503, + code: 'UNKNOWN_ERROR' + }) + }) + + it('returns the balance when a 503 clears on retry', async () => { + vi.useFakeTimers() + let calls = 0 + respondToFetch(BALANCE_URL, () => + ++calls === 1 + ? errorResponse(503, 'Balance temporarily unavailable') + : balanceResponse() + ) + + const pending = store.fetchBalance() + await vi.advanceTimersByTimeAsync(10_000) + + await expect(pending).resolves.toEqual({ balance: 0 }) + expect(calls).toBe(2) + }) }) describe('password update', () => { diff --git a/src/stores/authStore.ts b/src/stores/authStore.ts index bb55ac661a4..fc90fde3646 100644 --- a/src/stores/authStore.ts +++ b/src/stores/authStore.ts @@ -713,14 +713,16 @@ export const useAuthStore = defineStore('auth', () => { // Customer not found is expected for new users return null } - const { message } = await parseErrorResponse(response) + const { message, code } = await parseErrorResponse(response) if (!requestIsCurrent()) { return null } throw new AuthStoreError( t('toastMessages.failedToFetchBalance', { error: message - }) + }), + response.status, + code ) } diff --git a/src/types/axios-augmentation.d.ts b/src/types/axios-augmentation.d.ts index 101054788cd..b67f043f5d3 100644 --- a/src/types/axios-augmentation.d.ts +++ b/src/types/axios-augmentation.d.ts @@ -8,5 +8,7 @@ declare module 'axios' { // Exempts a deliberately Firebase-authed request (acceptInvite) from the // unified re-mint. __skipUnifiedRemint?: boolean + /** Sends once, skipping the shared 429/502/503/504 retry. */ + __skipStatusRetry?: boolean } } diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts new file mode 100644 index 00000000000..79935b32b32 --- /dev/null +++ b/src/utils/axiosRetryableStatus.test.ts @@ -0,0 +1,276 @@ +import type { AxiosAdapter, AxiosResponse } from 'axios' +import axios, { AxiosError } from 'axios' +import { describe, expect, it, vi } from 'vitest' + +import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' + +interface ScriptedAnswer { + readonly status: number + readonly headers?: Record +} + +function scriptedAdapter(answers: ScriptedAnswer[]) { + let call = 0 + return vi.fn(async (config) => { + const { status, headers = {} } = + answers[Math.min(call++, answers.length - 1)] + const response: AxiosResponse = { + data: {}, + status, + statusText: String(status), + headers, + config + } + if (status < 400) return response + throw new AxiosError( + `Request failed with status code ${status}`, + AxiosError.ERR_BAD_RESPONSE, + config, + null, + response + ) + }) +} + +function makeClient(answers: ScriptedAnswer[]) { + const adapter = scriptedAdapter(answers) + const sleep = vi.fn<(ms: number) => Promise>(async () => {}) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { sleep, random: () => 1 }) + return { client, adapter, sleep } +} + +describe('attachRetryableStatusRetry', () => { + it('returns a 200 after one 503', async () => { + const { client, adapter } = makeClient([{ status: 503 }, { status: 200 }]) + + const response = await client.get('https://api/x') + + expect(response.status).toBe(200) + expect(adapter).toHaveBeenCalledTimes(2) + }) + + it('rejects with the final 503 once the attempts are spent', async () => { + const { client, adapter } = makeClient([{ status: 503 }]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it.for([401, 404, 500])('does not retry a %i', async (status) => { + const { client, adapter } = makeClient([{ status }]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status } + }) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it.for(['post', 'put', 'patch', 'delete'] as const)( + 'does not retry a %s that got a 503', + async (method) => { + const { client, adapter } = makeClient([{ status: 503 }]) + + await expect( + client.request({ url: 'https://api/x', method }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + } + ) + + it('waits the Retry-After the server sent', async () => { + const { client, sleep } = makeClient([ + { status: 429, headers: { 'retry-after': '2' } }, + { status: 200 } + ]) + + await client.get('https://api/x') + + expect(sleep).toHaveBeenCalledExactlyOnceWith(2_250, undefined) + }) + + it('rejects without waiting when Retry-After is past the cap', async () => { + const { client, adapter, sleep } = makeClient([ + { status: 503, headers: { 'retry-after': '3600' } } + ]) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + + it('rethrows an error that has no response', async () => { + const networkError = new AxiosError('Network Error', AxiosError.ERR_NETWORK) + const adapter = vi.fn().mockRejectedValue(networkError) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { sleep: async () => {} }) + + await expect(client.get('https://api/x')).rejects.toBe(networkError) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it('sends once when the request opts out of the status retry', async () => { + const { client, adapter, sleep } = makeClient([{ status: 503 }]) + + await expect( + client.get('https://api/x', { __skipStatusRetry: true }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + + it('retries through an adapter the request selects for itself', async () => { + const { client, adapter: defaultAdapter } = makeClient([{ status: 200 }]) + const requestAdapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + + const response = await client.get('https://api/x', { + adapter: requestAdapter + }) + + expect(response.status).toBe(200) + expect(requestAdapter).toHaveBeenCalledTimes(2) + expect(defaultAdapter).not.toHaveBeenCalled() + }) + + it('does not stack a second retry loop when attached twice', async () => { + const adapter = scriptedAdapter([{ status: 503 }]) + const client = axios.create({ adapter }) + const sleep = async () => {} + attachRetryableStatusRetry(client, { sleep }) + attachRetryableStatusRetry(client, { sleep }) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ + response: { status: 503 } + }) + expect(adapter).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + describe('with a request timeout', () => { + function makeTimedClient(answers: ScriptedAnswer[]) { + const adapter = scriptedAdapter(answers) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { random: () => 1 }) + return { client, adapter } + } + + it('stops instead of waiting past the timeout', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '1' } }, + { status: 200 } + ]) + + await expect( + client.get('https://api/x', { timeout: 100 }) + ).rejects.toMatchObject({ response: { status: 503 } }) + + expect(adapter).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + }) + + it('surfaces the last failure when the wait overruns the timeout', async () => { + vi.useFakeTimers() + const adapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + vi.setSystemTime(Date.now() + 5_000) + } + }) + + await expect( + client.get('https://api/x', { timeout: 1_000 }) + ).rejects.toMatchObject({ response: { status: 503 } }) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it('hands the original timeout back on the error it throws', async () => { + vi.useFakeTimers() + const adapter = scriptedAdapter([{ status: 503 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + vi.setSystemTime(Date.now() + 100) + } + }) + + const error = await client + .get('https://api/x', { timeout: 5_000 }) + .catch((reason: unknown) => reason) + + expect(error).toMatchObject({ config: { timeout: 5_000 } }) + }) + + it('reports an abort during the wait as a cancellation', async () => { + vi.useFakeTimers() + const { client } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const controller = new AbortController() + + const outcome = client + .get('https://api/x', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort() + + expect(axios.isCancel(await outcome)).toBe(true) + }) + + it('reports a timeout signal fired during the wait as a cancellation, not a stale 503', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const controller = new AbortController() + + const outcome = client + .get('https://api/x', { signal: controller.signal }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort(new DOMException('Fetch timeout', 'TimeoutError')) + + expect(axios.isCancel(await outcome)).toBe(true) + expect(adapter).toHaveBeenCalledTimes(1) + }) + + it('reports a cancel token fired during the wait as a cancellation', async () => { + vi.useFakeTimers() + const { client } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '5' } } + ]) + const source = axios.CancelToken.source() + + const outcome = client + .get('https://api/x', { cancelToken: source.token }) + .catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + source.cancel() + + expect(axios.isCancel(await outcome)).toBe(true) + }) + + it('retries when the wait fits inside the timeout and shrinks the next attempt', async () => { + vi.useFakeTimers() + const { client, adapter } = makeTimedClient([ + { status: 503, headers: { 'retry-after': '1' } }, + { status: 200 } + ]) + + const pending = client.get('https://api/x', { timeout: 5_000 }) + await vi.advanceTimersByTimeAsync(1_250) + + expect((await pending).status).toBe(200) + expect(adapter.mock.calls.map(([config]) => config.timeout)).toEqual([ + 5_000, 3_750 + ]) + }) + }) +}) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts new file mode 100644 index 00000000000..4e644cec6cb --- /dev/null +++ b/src/utils/axiosRetryableStatus.ts @@ -0,0 +1,135 @@ +import type { + AxiosAdapter, + AxiosInstance, + AxiosResponse, + InternalAxiosRequestConfig +} from 'axios' +import axios from 'axios' + +import type { RetryPolicy } from '@/utils/retryPolicy' +import { retryDelayMs, sleep as sleepFor } from '@/utils/retryPolicy' + +export interface AxiosRetryableStatusOptions { + readonly policy?: RetryPolicy + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise + readonly random?: () => number +} + +function retryAfterHeader(response: AxiosResponse): string | undefined { + const value = response.headers['retry-after'] + return typeof value === 'string' || typeof value === 'number' + ? String(value) + : undefined +} + +function timeBudget(timeoutMs: number | undefined) { + const startedAt = Date.now() + return () => (timeoutMs ? timeoutMs - (Date.now() - startedAt) : undefined) +} + +function cancellationSignal( + config: InternalAxiosRequestConfig +): AbortSignal | undefined { + const signal = + config.signal instanceof AbortSignal ? config.signal : undefined + if (!config.cancelToken) return signal + + const controller = new AbortController() + void config.cancelToken.promise.then(() => controller.abort()) + return signal + ? AbortSignal.any([signal, controller.signal]) + : controller.signal +} + +function fitsBudget( + delay: number | undefined, + remaining: number | undefined +): delay is number { + return delay !== undefined && (remaining === undefined || delay < remaining) +} + +async function runAttempt( + adapter: AxiosAdapter, + config: InternalAxiosRequestConfig, + remaining: number | undefined +): Promise { + try { + const response = await adapter( + remaining === undefined ? config : { ...config, timeout: remaining } + ) + response.config = config + return response + } catch (error) { + if (axios.isAxiosError(error) && error.config) error.config = config + throw error + } +} + +function retryingAdapter( + adapter: AxiosAdapter, + { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions +): AxiosAdapter { + function waitBeforeRetry( + error: unknown, + config: InternalAxiosRequestConfig, + attempt: number + ): number | undefined { + const response = axios.isAxiosError(error) ? error.response : undefined + if (!response) return undefined + return retryDelayMs({ + method: config.method ?? 'GET', + status: response.status, + readRetryAfter: () => retryAfterHeader(response), + attempt, + policy, + random + }) + } + + async function pause(delay: number, config: InternalAxiosRequestConfig) { + const signal = cancellationSignal(config) + try { + await sleep(delay, signal) + } catch (error) { + if (!signal?.aborted) throw error + throw new axios.CanceledError('canceled', config) + } + } + + return async (config) => { + if (config.__skipStatusRetry) return adapter(config) + const remainingMs = timeBudget(config.timeout) + let lastError: unknown + + for (let attempt = 1; ; attempt++) { + const remaining = remainingMs() + if (remaining !== undefined && remaining <= 0) throw lastError + try { + return await runAttempt(adapter, config, remaining) + } catch (error) { + lastError = error + const delay = waitBeforeRetry(error, config, attempt) + if (!fitsBudget(delay, remainingMs())) throw error + await pause(delay, config) + } + } + } +} + +const retrying = new WeakSet() + +/** Retries retryable statuses around each request's adapter, within its `timeout`. */ +export function attachRetryableStatusRetry( + client: AxiosInstance, + options: AxiosRetryableStatusOptions = {} +): void { + client.interceptors.request.use((config) => { + const selected = config.adapter ?? client.defaults.adapter + if (typeof selected === 'function' && retrying.has(selected)) return config + + const adapter = retryingAdapter(axios.getAdapter(selected), options) + retrying.add(adapter) + config.adapter = adapter + return config + }) +} diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts new file mode 100644 index 00000000000..82ccbf4fc5b --- /dev/null +++ b/src/utils/fetchWithRetry.test.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { fetchWithRetry } from '@/utils/fetchWithRetry' +import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' + +function respond(status: number, headers: Record = {}) { + return new Response(status === 200 ? '{}' : null, { status, headers }) +} + +function scriptedFetch(...responses: Response[]) { + const queue = [...responses] + return vi.fn(async () => { + const next = queue.shift() + if (!next) throw new Error('fetch called more times than scripted') + return next + }) +} + +describe('fetchWithRetry', () => { + const sleep = vi.fn<(ms: number) => Promise>(async () => {}) + const random = () => 1 + + it('returns a 200 after one 503', async () => { + const final = respond(200) + const fetchImpl = scriptedFetch(respond(503), final) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result).toBe(final) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('returns the final 503 once the attempts are spent', async () => { + const last = respond(503) + const fetchImpl = scriptedFetch(respond(503), respond(503), last) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result).toBe(last) + expect(fetchImpl).toHaveBeenCalledTimes(DEFAULT_RETRY_POLICY.maxAttempts) + }) + + it.for([400, 401, 404, 500])('does not retry a %i', async (status) => { + const fetchImpl = scriptedFetch(respond(status)) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(status) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it.for(['POST', 'PUT', 'PATCH', 'DELETE'])( + 'does not retry a %s that got a 503', + async (method) => { + const fetchImpl = scriptedFetch(respond(503)) + + const result = await fetchWithRetry( + 'https://api/x', + { method }, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + } + ) + + it('does not retry a Request whose method is not idempotent', async () => { + const fetchImpl = scriptedFetch(respond(503)) + + await fetchWithRetry( + new Request('https://api/x', { method: 'POST', body: '{}' }), + {}, + { fetchImpl, sleep, random } + ) + + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + describe('replaying a body', () => { + const optedIn = { ...DEFAULT_RETRY_POLICY, retryNonIdempotent: true } + + function consumingFetch(...responses: Response[]) { + const bodies: string[] = [] + const queue = [...responses] + const fetchImpl = vi.fn(async (input, init) => { + const sent = input instanceof Request ? input : new Request(input, init) + bodies.push(await sent.text()) + const next = queue.shift() + if (!next) throw new Error('fetch called more times than scripted') + return next + }) + return { fetchImpl, bodies } + } + + it('sends the body of a Request again on the second attempt', async () => { + const { fetchImpl, bodies } = consumingFetch(respond(503), respond(200)) + + const result = await fetchWithRetry( + new Request('https://api/x', { method: 'POST', body: 'payload' }), + {}, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(result.status).toBe(200) + expect(bodies).toEqual(['payload', 'payload']) + }) + + it('sends an init body again on the second attempt', async () => { + const { fetchImpl, bodies } = consumingFetch(respond(503), respond(200)) + + await fetchWithRetry( + 'https://api/x', + { method: 'POST', body: 'payload' }, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(bodies).toEqual(['payload', 'payload']) + }) + + it('leaves the caller request unconsumed', async () => { + const { fetchImpl } = consumingFetch(respond(503), respond(200)) + const request = new Request('https://api/x', { + method: 'POST', + body: 'payload' + }) + + await fetchWithRetry( + request, + {}, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(request.bodyUsed).toBe(false) + }) + + it('does not retry a stream body that cannot be replayed', async () => { + const fetchImpl = scriptedFetch(respond(503)) + const body = new ReadableStream() + + const result = await fetchWithRetry( + 'https://api/x', + { method: 'POST', body, duplex: 'half' } as RequestInit, + { fetchImpl, sleep, random, policy: optedIn } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + }) + + it('retries a POST only when the policy opts in', async () => { + const fetchImpl = scriptedFetch(respond(503), respond(200)) + + const result = await fetchWithRetry( + 'https://api/x', + { method: 'POST' }, + { + fetchImpl, + sleep, + random, + policy: { ...DEFAULT_RETRY_POLICY, retryNonIdempotent: true } + } + ) + + expect(result.status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('waits the Retry-After the server sent', async () => { + const fetchImpl = scriptedFetch( + respond(503, { 'Retry-After': '4' }), + respond(200) + ) + + await fetchWithRetry('https://api/x', {}, { fetchImpl, sleep, random }) + + expect(sleep).toHaveBeenCalledExactlyOnceWith(4_250, undefined) + }) + + it('returns the 503 without waiting when Retry-After is past the cap', async () => { + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '3600' })) + + const result = await fetchWithRetry( + 'https://api/x', + {}, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(503) + expect(fetchImpl).toHaveBeenCalledTimes(1) + expect(sleep).not.toHaveBeenCalled() + }) + + describe('with the real clock source', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + it('holds the retry back until Retry-After has elapsed', async () => { + const fetchImpl = scriptedFetch( + respond(503, { 'Retry-After': '3' }), + respond(200) + ) + + const pending = fetchWithRetry('https://api/x', {}, { fetchImpl, random }) + + await vi.advanceTimersByTimeAsync(3_249) + expect(fetchImpl).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + expect((await pending).status).toBe(200) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('rejects with the timeout reason when the timeout signal fires mid-wait', async () => { + const controller = new AbortController() + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '5' })) + const timeout = new DOMException('Fetch timeout', 'TimeoutError') + + const pending = fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + const outcome = pending.catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort(timeout) + + expect(await outcome).toBe(timeout) + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it('rejects when the signal was already aborted before the wait began', async () => { + const controller = new AbortController() + const fetchImpl = vi.fn(async () => { + controller.abort('gone') + return respond(503) + }) + + await expect( + fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + ).rejects.toBe('gone') + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + + it('stops waiting when the caller aborts', async () => { + const controller = new AbortController() + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '5' })) + + const pending = fetchWithRetry( + 'https://api/x', + { signal: controller.signal }, + { fetchImpl } + ) + const outcome = pending.catch((reason: unknown) => reason) + await vi.advanceTimersByTimeAsync(1_000) + controller.abort('cancelled') + + expect(await outcome).toBe('cancelled') + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts new file mode 100644 index 00000000000..aa2164317f8 --- /dev/null +++ b/src/utils/fetchWithRetry.ts @@ -0,0 +1,80 @@ +import type { RetryPolicy } from '@/utils/retryPolicy' +import { + canRetryMethod, + retryDelayMs, + sleep as sleepFor +} from '@/utils/retryPolicy' + +type FetchLike = ( + input: RequestInfo | URL, + init?: RequestInit +) => Promise + +export interface RetryingFetchOptions { + readonly policy?: RetryPolicy + readonly fetchImpl?: FetchLike + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise + readonly random?: () => number +} + +function requestMethod(input: RequestInfo | URL, init: RequestInit): string { + return init.method ?? (input instanceof Request ? input.method : 'GET') +} + +function requestSignal( + input: RequestInfo | URL, + init: RequestInit +): AbortSignal | undefined { + return init.signal ?? (input instanceof Request ? input.signal : undefined) +} + +function isReplayable(input: RequestInfo | URL, init: RequestInit): boolean { + return ( + !(init.body instanceof ReadableStream) && + !(input instanceof Request && input.bodyUsed) + ) +} + +/** `fetch` that retries retryable statuses on idempotent methods, honoring `Retry-After`. */ +export async function fetchWithRetry( + input: RequestInfo | URL, + init: RequestInit = {}, + options: RetryingFetchOptions = {} +): Promise { + const { policy, fetchImpl = fetch, sleep = sleepFor, random } = options + const method = requestMethod(input, init) + if (!canRetryMethod(method, policy) || !isReplayable(input, init)) { + return fetchImpl(input, init) + } + + const signal = requestSignal(input, init) + const template = + input instanceof Request && input.body !== null ? input.clone() : undefined + + for (let attempt = 1; ; attempt++) { + const response = await fetchImpl(template?.clone() ?? input, init) + const delay = retryDelayMs({ + method, + status: response.status, + readRetryAfter: () => response.headers.get('Retry-After'), + attempt, + policy, + random + }) + if (delay === undefined) return response + await sleep(delay, signal) + await response.body?.cancel() + } +} + +/** Wraps a url-based send so it retries like {@link fetchWithRetry}. */ +export function retryingSend( + send: (url: string, init: RequestInit) => Promise, + policy?: RetryPolicy +): (url: string, init: RequestInit) => Promise { + return (url, init) => + fetchWithRetry(url, init, { + policy, + fetchImpl: (input, attemptInit) => send(String(input), attemptInit ?? {}) + }) +} diff --git a/src/utils/retryPolicy.test.ts b/src/utils/retryPolicy.test.ts new file mode 100644 index 00000000000..ddacf467f99 --- /dev/null +++ b/src/utils/retryPolicy.test.ts @@ -0,0 +1,185 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { RetryPolicy } from '@/utils/retryPolicy' +import { + DEFAULT_RETRY_POLICY, + parseRetryAfterMs, + retryDelayMs, + sleep +} from '@/utils/retryPolicy' + +const NOW = Date.parse('2026-10-18T12:00:00Z') + +describe('parseRetryAfterMs', () => { + it.for<[string, string, number]>([ + ['delta-seconds', '5', 5_000], + ['zero seconds', '0', 0], + ['padded delta-seconds', ' 7 ', 7_000], + ['an HTTP-date in the future', 'Sun, 18 Oct 2026 12:00:30 GMT', 30_000], + ['an HTTP-date in the past', 'Sun, 18 Oct 2026 11:59:00 GMT', 0] + ])('reads %s', ([_name, header, expected]) => { + expect(parseRetryAfterMs(header, NOW)).toBe(expected) + }) + + it.for<[string, string | null | undefined]>([ + ['absent', undefined], + ['null', null], + ['empty', ''], + ['garbage', 'soon'], + ['negative', '-5'], + ['fractional', '1.5'] + ])('ignores a header that is %s', ([_name, header]) => { + expect(parseRetryAfterMs(header, NOW)).toBeUndefined() + }) +}) + +describe('retryDelayMs', () => { + const decide = ( + overrides: Partial[0]> = {} + ) => + retryDelayMs({ + method: 'GET', + status: 503, + readRetryAfter: () => null, + attempt: 1, + random: () => 1, + now: () => NOW, + ...overrides + }) + + it.for([429, 502, 503, 504])('retries status %i', (status) => { + expect(decide({ status })).toBeDefined() + }) + + it.for([200, 400, 401, 403, 404, 409, 500, 501])( + 'does not retry status %i', + (status) => { + expect(decide({ status })).toBeUndefined() + } + ) + + it.for(['GET', 'get', 'HEAD', 'OPTIONS'])('retries %s', (method) => { + expect(decide({ method })).toBeDefined() + }) + + it.for(['POST', 'PUT', 'PATCH', 'DELETE'])( + 'does not retry %s by default', + (method) => { + expect(decide({ method })).toBeUndefined() + } + ) + + it('retries a non-idempotent method when the policy opts in', () => { + const policy: RetryPolicy = { + ...DEFAULT_RETRY_POLICY, + retryNonIdempotent: true + } + expect(decide({ method: 'POST', policy })).toBeDefined() + }) + + it('stops once the attempts are spent', () => { + const { maxAttempts } = DEFAULT_RETRY_POLICY + expect(decide({ attempt: maxAttempts - 1 })).toBeDefined() + expect(decide({ attempt: maxAttempts })).toBeUndefined() + }) + + it('backs off exponentially up to the cap', () => { + const policy: RetryPolicy = { + maxAttempts: 10, + baseDelayMs: 100, + maxDelayMs: 500, + retryNonIdempotent: false + } + const delays = [1, 2, 3, 4, 5].map((attempt) => + decide({ attempt, policy, random: () => 1 }) + ) + expect(delays).toEqual([100, 200, 400, 500, 500]) + }) + + it.for([0, 0.25, 0.5, 0.99, 1])( + 'keeps jitter within half to full backoff (random=%f)', + (random) => { + const delay = decide({ attempt: 2, random: () => random }) + const backoff = DEFAULT_RETRY_POLICY.baseDelayMs * 2 + expect(delay).toBeGreaterThanOrEqual(backoff / 2) + expect(delay).toBeLessThanOrEqual(backoff) + } + ) + + it.for([ + [0, 4_000], + [1, 4_250] + ])( + 'adds jitter on top of Retry-After, never below it (random=%f)', + ([random, expected]) => { + expect(decide({ readRetryAfter: () => '4', random: () => random })).toBe( + expected + ) + } + ) + + it('reads an HTTP-date Retry-After against the clock', () => { + expect( + decide({ + readRetryAfter: () => 'Sun, 18 Oct 2026 12:00:03 GMT', + random: () => 0 + }) + ).toBe(3_000) + }) + + it('gives up when Retry-After exceeds the cap', () => { + const tooLong = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000 + 1) + expect(decide({ readRetryAfter: () => tooLong })).toBeUndefined() + }) + + it('accepts Retry-After exactly at the cap', () => { + const atCap = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000) + expect(decide({ readRetryAfter: () => atCap })).toBe( + DEFAULT_RETRY_POLICY.maxDelayMs + ) + }) + + it('does not read the header for a response it will not retry', () => { + const readRetryAfter = vi.fn(() => '1') + decide({ status: 500, readRetryAfter }) + decide({ method: 'POST', readRetryAfter }) + expect(readRetryAfter).not.toHaveBeenCalled() + }) +}) + +describe('sleep', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + it('resolves after the delay', async () => { + let settled = false + const pending = sleep(1_000).then(() => { + settled = true + }) + + await vi.advanceTimersByTimeAsync(999) + expect(settled).toBe(false) + await vi.advanceTimersByTimeAsync(1) + await pending + expect(settled).toBe(true) + }) + + it('rejects with the abort reason when aborted mid-wait', async () => { + const controller = new AbortController() + const pending = sleep(1_000, controller.signal) + const outcome = pending.catch((reason: unknown) => reason) + + controller.abort('stop') + + expect(await outcome).toBe('stop') + expect(vi.getTimerCount()).toBe(0) + }) + + it('rejects immediately when already aborted', async () => { + const controller = new AbortController() + controller.abort('stop') + + await expect(sleep(1_000, controller.signal)).rejects.toBe('stop') + }) +}) diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts new file mode 100644 index 00000000000..56d55fdb238 --- /dev/null +++ b/src/utils/retryPolicy.ts @@ -0,0 +1,111 @@ +export interface RetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number + readonly retryNonIdempotent: boolean +} + +export const DEFAULT_RETRY_POLICY: RetryPolicy = { + maxAttempts: 3, + baseDelayMs: 500, + maxDelayMs: 10_000, + retryNonIdempotent: false +} + +/** Sends once: for callers that already retry on their own backoff. */ +export const NO_RETRY_POLICY: RetryPolicy = { + ...DEFAULT_RETRY_POLICY, + maxAttempts: 1 +} + +const RETRYABLE_STATUSES: ReadonlySet = new Set([429, 502, 503, 504]) +const IDEMPOTENT_METHODS: ReadonlySet = new Set([ + 'GET', + 'HEAD', + 'OPTIONS' +]) + +function isRetryableStatus(status: number): boolean { + return RETRYABLE_STATUSES.has(status) +} + +function isIdempotentMethod(method: string): boolean { + return IDEMPOTENT_METHODS.has(method.toUpperCase()) +} + +export function canRetryMethod( + method: string, + policy: RetryPolicy = DEFAULT_RETRY_POLICY +): boolean { + return policy.retryNonIdempotent || isIdempotentMethod(method) +} + +/** Milliseconds a `Retry-After` header asks for (delta-seconds or HTTP-date). */ +export function parseRetryAfterMs( + value: string | null | undefined, + now: number = Date.now() +): number | undefined { + const trimmed = value?.trim() + if (!trimmed) return undefined + if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000 + if (!/[a-z]/i.test(trimmed)) return undefined + const at = Date.parse(trimmed) + return Number.isNaN(at) ? undefined : Math.max(0, at - now) +} + +interface RetryDecisionInput { + readonly method: string + readonly status: number + readonly readRetryAfter: () => string | null | undefined + readonly attempt: number + readonly policy?: RetryPolicy + readonly random?: () => number + readonly now?: () => number +} + +/** Delay before the next attempt, or `undefined` to stop retrying. */ +export function retryDelayMs({ + method, + status, + readRetryAfter, + attempt, + policy = DEFAULT_RETRY_POLICY, + random = Math.random, + now = Date.now +}: RetryDecisionInput): number | undefined { + if (attempt >= policy.maxAttempts) return undefined + if (!isRetryableStatus(status)) return undefined + if (!canRetryMethod(method, policy)) return undefined + + const retryAfterMs = parseRetryAfterMs(readRetryAfter(), now()) + if (retryAfterMs !== undefined && retryAfterMs > policy.maxDelayMs) { + return undefined + } + + const ceiling = Math.min( + policy.maxDelayMs, + policy.baseDelayMs * 2 ** (attempt - 1) + ) + const jittered = ceiling / 2 + (random() * ceiling) / 2 + if (retryAfterMs === undefined) return jittered + const spread = (random() * ceiling) / 2 + return Math.min(policy.maxDelayMs, Math.max(jittered, retryAfterMs + spread)) +} + +export function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason) + return + } + const onAbort = () => { + clearTimeout(timer) + reject(signal?.reason) + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +}