diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index acd6b0a5cb1..49fcf51c828 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -81,6 +81,7 @@ import type { TabCountMetadata, TelemetryDispatcher, TelemetryProvider, + FetchStatusRetryMetadata, FetchTimeoutMetadata, TemplateFilterMetadata, TemplateLibraryClosedMetadata, @@ -601,4 +602,9 @@ export class TelemetryRegistry implements TelemetryDispatcher { trackFetchTimeout(metadata: FetchTimeoutMetadata): void { this.dispatch((provider) => provider.trackFetchTimeout?.(metadata)) } + + // fallow-ignore-next-line unused-class-member + trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void { + this.dispatch((provider) => provider.trackFetchStatusRetry?.(metadata)) + } } diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts index f88fcad83da..fe55b08ad5e 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts @@ -149,6 +149,25 @@ describe('DatadogRumTelemetryProvider', () => { }) }) + it('records status retries as RUM actions', () => { + const metadata = { + transport: 'axios', + method: 'GET', + host: 'cloud.comfy.org', + route: '/api/billing/balance', + attempts: 3, + final_status: 503, + outcome: 'exhausted' + } as const + + new DatadogRumTelemetryProvider().trackFetchStatusRetry(metadata) + + expect(addAction).toHaveBeenCalledExactlyOnceWith( + TelemetryEvents.FETCH_STATUS_RETRY, + metadata + ) + }) + it('records fetch timeouts as RUM actions', () => { new DatadogRumTelemetryProvider().trackFetchTimeout({ route: '/userdata/:resource', diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts index eaddab78ff2..72868703bb6 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts @@ -20,6 +20,7 @@ import { useCurrentUser } from '@/composables/auth/useCurrentUser' import type { AuthMetadata, ExecutionOutcomeMetadata, + FetchStatusRetryMetadata, FetchTimeoutMetadata, ImageLoadFailureMetadata, TelemetryProvider, @@ -56,6 +57,10 @@ export class DatadogRumTelemetryProvider implements TelemetryProvider { datadogRum.addAction(TelemetryEvents.FETCH_TIMEOUT, metadata) } + trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void { + datadogRum.addAction(TelemetryEvents.FETCH_STATUS_RETRY, metadata) + } + trackUnifiedAuthRetry(metadata: UnifiedAuthRetryMetadata): void { datadogRum.addAction( metadata.outcome === 'succeeded' diff --git a/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.test.ts index 0383dea0d3f..671e1ecff00 100644 --- a/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.test.ts @@ -148,4 +148,32 @@ describe('SentryTelemetryProvider', () => { node_type: 'KSampler' }) }) + + it.for([ + ['recovered', 'info'], + ['exhausted', 'warning'], + ['retry_after_too_long', 'warning'] + ] as const)( + 'records a %s status retry as a %s breadcrumb', + ([outcome, level]) => { + const metadata = { + transport: 'fetch', + method: 'GET', + host: 'api.comfy.org', + route: '/customers/balance', + attempts: 2, + final_status: 503, + outcome + } as const + + new SentryTelemetryProvider().trackFetchStatusRetry(metadata) + + expect(mocks.addBreadcrumb).toHaveBeenCalledExactlyOnceWith({ + category: 'fetch.status_retry', + message: outcome, + level, + data: metadata + }) + } + ) }) diff --git a/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.ts index 6943686bd1b..a8085ce89a6 100644 --- a/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/SentryTelemetryProvider.ts @@ -6,6 +6,7 @@ import { useWorkflowStore } from '@/platform/workflow/management/stores/workflow import type { AuthMetadata, ExecutionErrorMetadata, + FetchStatusRetryMetadata, ExecutionSuccessMetadata, NodeAddedMetadata, ShellLayoutMetadata, @@ -30,6 +31,15 @@ export class SentryTelemetryProvider implements TelemetryProvider { this.updateAppContext() } + trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void { + addBreadcrumb({ + category: 'fetch.status_retry', + message: metadata.outcome, + level: metadata.outcome === 'recovered' ? 'info' : 'warning', + data: { ...metadata } + }) + } + trackNodeAdded({ node_type, source }: NodeAddedMetadata): void { addBreadcrumb({ category: 'workflow', diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 8b41a3924cd..edfb1116d4e 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -1424,6 +1424,28 @@ export interface FetchTimeoutMetadata { timeout_ms: number } +/** How a request that got a retryable status (429/502/503/504) ended. */ +export type FetchStatusRetryOutcome = + | 'recovered' + | 'exhausted' + | 'retry_after_too_long' + | 'timeout_budget' + | 'failed' + +/** One request that hit a retryable status in the shared status retry. */ +export interface FetchStatusRetryMetadata { + transport: 'fetch' | 'axios' + method: string + host: string + /** Route group only, as in `app:fetch_timeout`, e.g. `/billing/:resource`. */ + route: string + /** Requests sent, including the first. */ + attempts: number + /** The last status the request got, if any. */ + final_status?: number + outcome: FetchStatusRetryOutcome +} + /** * Telemetry provider interface for individual providers. * All methods are optional - providers only implement what they need. @@ -1610,6 +1632,7 @@ export interface TelemetryProvider { // Network error events trackFetchTimeout?(metadata: FetchTimeoutMetadata): void + trackFetchStatusRetry?(metadata: FetchStatusRetryMetadata): void } /** @@ -1790,6 +1813,7 @@ export const TelemetryEvents = { // Network FETCH_TIMEOUT: 'app:fetch_timeout', + FETCH_STATUS_RETRY: 'app:fetch_status_retry', WORKSPACE_SCOPE_MISMATCH: 'workspace_scope_mismatch' } as const @@ -1896,4 +1920,5 @@ export type TelemetryEventProperties = | AgentPaywallCtaMetadata | AgentCreditTransitionNoticeMetadata | FetchTimeoutMetadata + | FetchStatusRetryMetadata | WorkspaceScopeMismatchMetadata diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 72710a36857..93c102e85fd 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -84,6 +84,7 @@ import type { import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' import type { AuthCredential, AuthScheme } from '@/types/authTypes' import type { NodeExecutionId } from '@/types/nodeIdentification' +import { getFetchRouteTemplate } from '@/utils/fetchRouteTemplate' import { retryingSend } from '@/utils/fetchWithRetry' import { NO_RETRY_POLICY } from '@/utils/retryPolicy' import type { RetryPolicy } from '@/utils/retryPolicy' @@ -161,52 +162,6 @@ interface FetchApiOptions extends RequestInit { skipStatusRetry?: boolean } -const FETCH_ROUTE_GROUPS = new Set([ - 'assets', - 'embeddings', - 'experiment', - 'extensions', - 'features', - 'files', - 'folder_paths', - 'free', - 'global_subgraphs', - 'history', - 'hub', - 'internal', - 'interrupt', - 'jobs', - 'logs', - 'models', - 'node_replacements', - 'object_info', - 'prompt', - 'providers', - 'queue', - 'secrets', - 'settings', - 'system_stats', - 'upload', - 'user', - 'userdata', - 'users', - 'video_metadata', - 'view', - 'view_metadata', - 'workflow_templates', - 'workflows', - 'workspace' -]) - -function getFetchRouteTemplate(route: string): string { - const segments = (route.split(/[?#]/)[0] ?? '').split('/').filter(Boolean) - const routeSegments = segments[0] === 'api' ? segments.slice(1) : segments - const [routeGroup, ...resources] = routeSegments - - if (!routeGroup || !FETCH_ROUTE_GROUPS.has(routeGroup)) return '/other' - return `/${routeGroup}${resources.length ? '/:resource' : ''}` -} - /** * Options for queuePrompt method */ diff --git a/src/utils/axiosRetryableStatus.test.ts b/src/utils/axiosRetryableStatus.test.ts index 79935b32b32..c91237a49dd 100644 --- a/src/utils/axiosRetryableStatus.test.ts +++ b/src/utils/axiosRetryableStatus.test.ts @@ -1,9 +1,18 @@ -import type { AxiosAdapter, AxiosResponse } from 'axios' +import type { + AxiosAdapter, + AxiosResponse, + InternalAxiosRequestConfig +} from 'axios' import axios, { AxiosError } from 'axios' -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { useTelemetry } from '@/platform/telemetry' import { attachRetryableStatusRetry } from '@/utils/axiosRetryableStatus' import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' +import { clearStatusRetryBudget } from '@/utils/statusRetryTelemetry' + +vi.mock(import('@/platform/telemetry')) interface ScriptedAnswer { readonly status: number @@ -274,3 +283,192 @@ describe('attachRetryableStatusRetry', () => { }) }) }) + +describe('attachRetryableStatusRetry status retry telemetry', () => { + const tracked = () => vi.mocked(useTelemetry()!.trackFetchStatusRetry) + + beforeEach(() => clearStatusRetryBudget()) + + function failThenThrow( + second: (config: InternalAxiosRequestConfig) => never + ) { + return vi + .fn() + .mockImplementationOnce(async (config) => { + throw new AxiosError( + 'Request failed with status code 503', + AxiosError.ERR_BAD_RESPONSE, + config, + null, + { data: {}, status: 503, statusText: '503', headers: {}, config } + ) + }) + .mockImplementationOnce(async (config) => second(config)) + } + + it('records a request that recovered after a retry', async () => { + const { client } = makeClient([{ status: 503 }, { status: 200 }]) + + await client.get('/api/billing/balance', { + baseURL: 'https://cloud.comfy.org' + }) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith({ + transport: 'axios', + method: 'GET', + host: 'cloud.comfy.org', + route: '/billing/:resource', + attempts: 2, + final_status: 200, + outcome: 'recovered' + }) + }) + + it.for<[string, ScriptedAnswer[], Record]>([ + [ + 'spent its attempts', + [{ status: 503 }], + { attempts: 3, final_status: 503, outcome: 'exhausted' } + ], + [ + 'got a non-retryable error after a retry', + [{ status: 503 }, { status: 404 }], + { attempts: 2, final_status: 404, outcome: 'failed' } + ] + ])('records a request that %s', async ([_name, answers, expected]) => { + const { client } = makeClient(answers) + + await expect(client.get('https://api/x')).rejects.toBeDefined() + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining(expected) + ) + }) + + it('records a request whose wait would overrun its timeout', async () => { + vi.useFakeTimers() + const adapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + vi.setSystemTime(Date.now() + 5_000) + } + }) + + await expect( + client.get('https://api/x', { timeout: 1_000 }) + ).rejects.toBeDefined() + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + attempts: 1, + final_status: 503, + outcome: 'timeout_budget' + }) + ) + }) + + it('records a retry that hit the request timeout', async () => { + const adapter = failThenThrow((config) => { + throw new AxiosError( + 'timeout of 60000ms exceeded', + AxiosError.ECONNABORTED, + config + ) + }) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + sleep: async () => {}, + random: () => 1 + }) + + await expect( + client.get('https://api/x', { timeout: 60_000 }) + ).rejects.toMatchObject({ code: AxiosError.ECONNABORTED }) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + attempts: 2, + final_status: 503, + outcome: 'timeout_budget' + }) + ) + }) + + it.for([ + ['a cancellation error', AxiosError.ERR_CANCELED, 'canceled'], + ['a browser abort', AxiosError.ECONNABORTED, 'Request aborted'] + ])( + 'records nothing when a retry ends in %s', + async ([_name, code, message]) => { + const adapter = failThenThrow((config) => { + throw new AxiosError(message, code, config) + }) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + sleep: async () => {}, + random: () => 1 + }) + + await expect(client.get('https://api/x')).rejects.toMatchObject({ code }) + + expect(tracked()).not.toHaveBeenCalled() + } + ) + + it('records a request whose own timeout signal fired while it waited to retry', async () => { + const controller = new AbortController() + const timeout = new DOMException('signal timed out', 'TimeoutError') + const adapter = scriptedAdapter([{ status: 503 }, { status: 200 }]) + const client = axios.create({ adapter }) + attachRetryableStatusRetry(client, { + random: () => 1, + sleep: async () => { + controller.abort(timeout) + throw timeout + } + }) + + await expect( + client.get('https://api/x', { signal: controller.signal }) + ).rejects.toBeInstanceOf(axios.CanceledError) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + attempts: 1, + final_status: 503, + outcome: 'timeout_budget' + }) + ) + }) + + it('still returns the response when recording throws', async () => { + tracked().mockImplementation(() => { + throw new Error('telemetry down') + }) + const { client } = makeClient([{ status: 503 }, { status: 200 }]) + + await expect(client.get('https://api/x')).resolves.toMatchObject({ + status: 200 + }) + }) + + it('records nothing for a request that never saw a retryable status', async () => { + const { client } = makeClient([{ status: 404 }]) + + await expect(client.get('https://api/x')).rejects.toBeDefined() + + expect(tracked()).not.toHaveBeenCalled() + }) + + it('records nothing for a read that opted out of the status retry', async () => { + const { client } = makeClient([{ status: 503 }]) + + await expect( + client.get('https://api/x', { __skipStatusRetry: true }) + ).rejects.toBeDefined() + + expect(tracked()).not.toHaveBeenCalled() + }) +}) diff --git a/src/utils/axiosRetryableStatus.ts b/src/utils/axiosRetryableStatus.ts index 4e644cec6cb..c757a204566 100644 --- a/src/utils/axiosRetryableStatus.ts +++ b/src/utils/axiosRetryableStatus.ts @@ -4,10 +4,16 @@ import type { AxiosResponse, InternalAxiosRequestConfig } from 'axios' -import axios from 'axios' +import axios, { AxiosError } from 'axios' import type { RetryPolicy } from '@/utils/retryPolicy' -import { retryDelayMs, sleep as sleepFor } from '@/utils/retryPolicy' +import { nextRetry, sleep as sleepFor } from '@/utils/retryPolicy' +import type { StatusRetryEnd } from '@/utils/statusRetryTelemetry' +import { + abortEnd, + endForStop, + trackStatusRetry +} from '@/utils/statusRetryTelemetry' export interface AxiosRetryableStatusOptions { readonly policy?: RetryPolicy @@ -41,11 +47,8 @@ function cancellationSignal( : controller.signal } -function fitsBudget( - delay: number | undefined, - remaining: number | undefined -): delay is number { - return delay !== undefined && (remaining === undefined || delay < remaining) +function fitsBudget(delay: number, remaining: number | undefined): boolean { + return remaining === undefined || delay < remaining } async function runAttempt( @@ -55,7 +58,9 @@ async function runAttempt( ): Promise { try { const response = await adapter( - remaining === undefined ? config : { ...config, timeout: remaining } + remaining === undefined + ? config + : { ...config, timeout: Math.max(1, remaining) } ) response.config = config return response @@ -65,32 +70,100 @@ async function runAttempt( } } +function isTimeout(error: unknown): boolean { + if (!axios.isAxiosError(error)) return false + if (error.code === AxiosError.ETIMEDOUT) return true + return ( + error.code === AxiosError.ECONNABORTED && /timeout/i.test(error.message) + ) +} + +function noResponseEnd(error: unknown): StatusRetryEnd { + if (isTimeout(error)) return 'timeout' + const cancelled = + axios.isCancel(error) || + (axios.isAxiosError(error) && + (error.code === AxiosError.ERR_CANCELED || + error.code === AxiosError.ECONNABORTED)) + return cancelled ? 'cancelled' : 'network' +} + +interface StatusRetryTracker { + scheduled(status: number): void + settle(end: StatusRetryEnd, attempts: number, status?: number): void +} + +function statusRetryTracker( + config: InternalAxiosRequestConfig +): StatusRetryTracker { + let lastRetryStatus: number | undefined + return { + scheduled(status) { + lastRetryStatus = status + }, + settle(end, attempts, status) { + trackStatusRetry({ + transport: 'axios', + method: config.method ?? 'GET', + url: () => axios.getUri(config), + end, + attempts, + finalStatus: status ?? lastRetryStatus, + retryScheduled: lastRetryStatus !== undefined + }) + } + } +} + +function delayOrRethrow( + error: unknown, + config: InternalAxiosRequestConfig, + attempt: number, + remaining: number | undefined, + tracker: StatusRetryTracker, + { policy, random }: AxiosRetryableStatusOptions +): number { + const failed = axios.isAxiosError(error) ? error.response : undefined + if (!failed) { + tracker.settle(noResponseEnd(error), attempt) + throw error + } + const decision = nextRetry({ + method: config.method ?? 'GET', + status: failed.status, + readRetryAfter: () => retryAfterHeader(failed), + attempt, + policy, + random + }) + if (!decision.retry) { + tracker.settle(endForStop(decision.reason), attempt, failed.status) + throw error + } + tracker.scheduled(failed.status) + if (!fitsBudget(decision.delayMs, remaining)) { + tracker.settle('timeout', attempt) + throw error + } + return decision.delayMs +} + function retryingAdapter( adapter: AxiosAdapter, - { policy, sleep = sleepFor, random }: AxiosRetryableStatusOptions + options: AxiosRetryableStatusOptions ): AxiosAdapter { - function waitBeforeRetry( - error: unknown, - config: InternalAxiosRequestConfig, - attempt: number - ): number | undefined { - const response = axios.isAxiosError(error) ? error.response : undefined - if (!response) return undefined - return retryDelayMs({ - method: config.method ?? 'GET', - status: response.status, - readRetryAfter: () => retryAfterHeader(response), - attempt, - policy, - random - }) - } + const { sleep = sleepFor } = options - async function pause(delay: number, config: InternalAxiosRequestConfig) { + async function pause( + delay: number, + config: InternalAxiosRequestConfig, + settleAbort: (signal: AbortSignal | undefined) => void + ) { const signal = cancellationSignal(config) try { await sleep(delay, signal) } catch (error) { + settleAbort(signal) if (!signal?.aborted) throw error throw new axios.CanceledError('canceled', config) } @@ -99,18 +172,32 @@ function retryingAdapter( return async (config) => { if (config.__skipStatusRetry) return adapter(config) const remainingMs = timeBudget(config.timeout) + const tracker = statusRetryTracker(config) let lastError: unknown for (let attempt = 1; ; attempt++) { const remaining = remainingMs() - if (remaining !== undefined && remaining <= 0) throw lastError + if (attempt > 1 && remaining !== undefined && remaining <= 0) { + tracker.settle('timeout', attempt - 1) + throw lastError + } try { - return await runAttempt(adapter, config, remaining) + const response = await runAttempt(adapter, config, remaining) + tracker.settle('answered', attempt, response.status) + return response } catch (error) { lastError = error - const delay = waitBeforeRetry(error, config, attempt) - if (!fitsBudget(delay, remainingMs())) throw error - await pause(delay, config) + const delay = delayOrRethrow( + error, + config, + attempt, + remainingMs(), + tracker, + options + ) + await pause(delay, config, (signal) => + tracker.settle(abortEnd(signal), attempt) + ) } } } diff --git a/src/utils/fetchRouteTemplate.ts b/src/utils/fetchRouteTemplate.ts new file mode 100644 index 00000000000..fe1ee49e206 --- /dev/null +++ b/src/utils/fetchRouteTemplate.ts @@ -0,0 +1,50 @@ +const TELEMETRY_SAFE_ROUTE_GROUPS = new Set([ + 'assets', + 'billing', + 'customers', + 'embeddings', + 'experiment', + 'extensions', + 'features', + 'files', + 'folder_paths', + 'free', + 'global_subgraphs', + 'history', + 'hub', + 'internal', + 'interrupt', + 'jobs', + 'logs', + 'models', + 'node_replacements', + 'object_info', + 'prompt', + 'providers', + 'queue', + 'secrets', + 'settings', + 'system_stats', + 'upload', + 'user', + 'userdata', + 'users', + 'video_metadata', + 'view', + 'view_metadata', + 'workflow_templates', + 'workflows', + 'workspace', + 'workspaces' +]) + +/** A route reduced to its allowlisted group, safe to send to telemetry. */ +export function getFetchRouteTemplate(route: string): string { + const segments = (route.split(/[?#]/)[0] ?? '').split('/').filter(Boolean) + const routeSegments = segments[0] === 'api' ? segments.slice(1) : segments + const [routeGroup, ...resources] = routeSegments + + if (!routeGroup || !TELEMETRY_SAFE_ROUTE_GROUPS.has(routeGroup)) + return '/other' + return `/${routeGroup}${resources.length ? '/:resource' : ''}` +} diff --git a/src/utils/fetchWithRetry.test.ts b/src/utils/fetchWithRetry.test.ts index 82ccbf4fc5b..73846632573 100644 --- a/src/utils/fetchWithRetry.test.ts +++ b/src/utils/fetchWithRetry.test.ts @@ -1,7 +1,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +import { useTelemetry } from '@/platform/telemetry' + import { fetchWithRetry } from '@/utils/fetchWithRetry' -import { DEFAULT_RETRY_POLICY } from '@/utils/retryPolicy' +import { DEFAULT_RETRY_POLICY, NO_RETRY_POLICY } from '@/utils/retryPolicy' +import { clearStatusRetryBudget } from '@/utils/statusRetryTelemetry' + +vi.mock(import('@/platform/telemetry')) function respond(status: number, headers: Record = {}) { return new Response(status === 200 ? '{}' : null, { status, headers }) @@ -277,3 +282,229 @@ describe('fetchWithRetry', () => { }) }) }) + +describe('fetchWithRetry status retry telemetry', () => { + const sleep = vi.fn<(ms: number) => Promise>(async () => {}) + const random = () => 1 + const balanceUrl = 'https://api.comfy.org/customers/balance' + const tracked = () => vi.mocked(useTelemetry()!.trackFetchStatusRetry) + + beforeEach(() => clearStatusRetryBudget()) + + it('records a request that recovered after a retry', async () => { + const fetchImpl = scriptedFetch(respond(503), respond(200)) + + await fetchWithRetry(balanceUrl, {}, { fetchImpl, sleep, random }) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith({ + transport: 'fetch', + method: 'GET', + host: 'api.comfy.org', + route: '/customers/:resource', + attempts: 2, + final_status: 200, + outcome: 'recovered' + }) + }) + + it.for<[string, Response[], Record]>([ + [ + 'spent its attempts', + [respond(503), respond(502), respond(503)], + { attempts: 3, final_status: 503, outcome: 'exhausted' } + ], + [ + 'gave up because Retry-After was too long', + [respond(503, { 'Retry-After': '30' })], + { attempts: 1, final_status: 503, outcome: 'retry_after_too_long' } + ], + [ + 'got a non-retryable error after a retry', + [respond(503), respond(500)], + { attempts: 2, final_status: 500, outcome: 'failed' } + ] + ])('records a request that %s', async ([_name, responses, expected]) => { + const fetchImpl = scriptedFetch(...responses) + + await fetchWithRetry(balanceUrl, {}, { fetchImpl, sleep, random }) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining(expected) + ) + }) + + it('records a retry that got no response and still rethrows the failure', async () => { + const failure = new TypeError('Failed to fetch') + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(respond(503)) + .mockRejectedValueOnce(failure) + + await expect( + fetchWithRetry(balanceUrl, {}, { fetchImpl, sleep, random }) + ).rejects.toBe(failure) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + attempts: 2, + final_status: 503, + outcome: 'failed' + }) + ) + }) + + it('records a request whose timeout fired while it waited to retry', async () => { + const controller = new AbortController() + const timeout = new DOMException('Fetch timeout', 'TimeoutError') + const fetchImpl = scriptedFetch(respond(503)) + const sleepUntilTimeout = vi.fn(async () => { + controller.abort(timeout) + throw timeout + }) + + await expect( + fetchWithRetry( + balanceUrl, + { signal: controller.signal }, + { fetchImpl, sleep: sleepUntilTimeout, random } + ) + ).rejects.toBe(timeout) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + attempts: 1, + final_status: 503, + outcome: 'timeout_budget' + }) + ) + }) + + it('records nothing when the caller cancels while it waits to retry', async () => { + const controller = new AbortController() + const fetchImpl = scriptedFetch(respond(503)) + const sleepUntilCancel = vi.fn(async () => { + controller.abort('route changed') + throw 'route changed' + }) + + await expect( + fetchWithRetry( + balanceUrl, + { signal: controller.signal }, + { fetchImpl, sleep: sleepUntilCancel, random } + ) + ).rejects.toBe('route changed') + + expect(tracked()).not.toHaveBeenCalled() + }) + + it.for([ + [ + 'https://cloud.comfy.org/api/workspaces/ws_8f2a91/members?token=secret', + '/workspaces/:resource' + ], + [ + 'https://cloud.comfy.org/api/userdata/my-secret-project', + '/userdata/:resource' + ], + ['https://cloud.comfy.org/api/someone/alice', '/other'] + ])('reports %s only by its route group', async ([url, route]) => { + const fetchImpl = scriptedFetch(respond(503), respond(200)) + + await fetchWithRetry(url, {}, { fetchImpl, sleep, random }) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ host: 'cloud.comfy.org', route }) + ) + }) + + it('records at most 20 events per route and outcome each minute', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + const recoverOnce = () => + fetchWithRetry( + 'https://cloud.comfy.org/api/queue', + {}, + { fetchImpl: scriptedFetch(respond(503), respond(200)), sleep, random } + ) + + await Promise.all(Array.from({ length: 25 }, recoverOnce)) + expect(tracked()).toHaveBeenCalledTimes(20) + + vi.advanceTimersByTime(60_000) + await recoverOnce() + + expect(tracked()).toHaveBeenCalledTimes(21) + }) + + it('does not spend the budget before telemetry is ready', async () => { + vi.mocked(useTelemetry).mockReturnValueOnce(null) + await fetchWithRetry( + balanceUrl, + {}, + { fetchImpl: scriptedFetch(respond(503), respond(200)), sleep, random } + ) + + await fetchWithRetry( + balanceUrl, + {}, + { fetchImpl: scriptedFetch(respond(503), respond(200)), sleep, random } + ) + + expect(tracked()).toHaveBeenCalledOnce() + }) + + it('reports a host outside Comfy as other', async () => { + const fetchImpl = scriptedFetch(respond(503), respond(200)) + + await fetchWithRetry( + 'https://10.0.0.7:8443/api/queue', + {}, + { fetchImpl, sleep, random } + ) + + expect(tracked()).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ host: 'other', route: '/queue' }) + ) + }) + + it('still retries when discarding the retryable body fails', async () => { + const broken = new Response( + new ReadableStream({ + cancel: () => Promise.reject(new Error('stream errored')) + }), + { status: 503 } + ) + const fetchImpl = scriptedFetch(broken, respond(200)) + + const result = await fetchWithRetry( + balanceUrl, + {}, + { fetchImpl, sleep, random } + ) + + expect(result.status).toBe(200) + expect(tracked()).toHaveBeenCalledWith( + expect.objectContaining({ outcome: 'recovered' }) + ) + }) + + it('records nothing for a request that never saw a retryable status', async () => { + const fetchImpl = scriptedFetch(respond(500)) + + await fetchWithRetry(balanceUrl, {}, { fetchImpl, sleep, random }) + + expect(tracked()).not.toHaveBeenCalled() + }) + + it('records nothing for a read that opted out of the status retry', async () => { + const fetchImpl = scriptedFetch(respond(503, { 'Retry-After': '30' })) + + await fetchWithRetry( + balanceUrl, + {}, + { policy: NO_RETRY_POLICY, fetchImpl, sleep, random } + ) + + expect(tracked()).not.toHaveBeenCalled() + }) +}) diff --git a/src/utils/fetchWithRetry.ts b/src/utils/fetchWithRetry.ts index aa2164317f8..e0fe51d1d09 100644 --- a/src/utils/fetchWithRetry.ts +++ b/src/utils/fetchWithRetry.ts @@ -1,9 +1,15 @@ import type { RetryPolicy } from '@/utils/retryPolicy' import { canRetryMethod, - retryDelayMs, + nextRetry, sleep as sleepFor } from '@/utils/retryPolicy' +import type { StatusRetryEnd } from '@/utils/statusRetryTelemetry' +import { + abortEnd, + endForStop, + trackStatusRetry +} from '@/utils/statusRetryTelemetry' type FetchLike = ( input: RequestInfo | URL, @@ -51,9 +57,27 @@ export async function fetchWithRetry( const template = input instanceof Request && input.body !== null ? input.clone() : undefined + let lastRetryStatus: number | undefined + const settle = (end: StatusRetryEnd, attempts: number, status?: number) => + trackStatusRetry({ + transport: 'fetch', + method, + url: () => (input instanceof Request ? input.url : String(input)), + end, + attempts, + finalStatus: status ?? lastRetryStatus, + retryScheduled: lastRetryStatus !== undefined + }) + for (let attempt = 1; ; attempt++) { - const response = await fetchImpl(template?.clone() ?? input, init) - const delay = retryDelayMs({ + let response: Response + try { + response = await fetchImpl(template?.clone() ?? input, init) + } catch (error) { + settle(abortEnd(signal), attempt) + throw error + } + const decision = nextRetry({ method, status: response.status, readRetryAfter: () => response.headers.get('Retry-After'), @@ -61,9 +85,18 @@ export async function fetchWithRetry( policy, random }) - if (delay === undefined) return response - await sleep(delay, signal) - await response.body?.cancel() + if (!decision.retry) { + settle(endForStop(decision.reason), attempt, response.status) + return response + } + lastRetryStatus = response.status + await response.body?.cancel().catch(() => undefined) + try { + await sleep(decision.delayMs, signal) + } catch (error) { + settle(abortEnd(signal), attempt) + throw error + } } } diff --git a/src/utils/retryPolicy.test.ts b/src/utils/retryPolicy.test.ts index ddacf467f99..64866cf2ab9 100644 --- a/src/utils/retryPolicy.test.ts +++ b/src/utils/retryPolicy.test.ts @@ -3,8 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { RetryPolicy } from '@/utils/retryPolicy' import { DEFAULT_RETRY_POLICY, + NO_RETRY_POLICY, + nextRetry, parseRetryAfterMs, - retryDelayMs, sleep } from '@/utils/retryPolicy' @@ -33,11 +34,11 @@ describe('parseRetryAfterMs', () => { }) }) -describe('retryDelayMs', () => { - const decide = ( - overrides: Partial[0]> = {} - ) => - retryDelayMs({ +describe('nextRetry delay', () => { + const scheduledDelayMs = ( + overrides: Partial[0]> = {} + ) => { + const decision = nextRetry({ method: 'GET', status: 503, readRetryAfter: () => null, @@ -46,26 +47,28 @@ describe('retryDelayMs', () => { now: () => NOW, ...overrides }) + return decision.retry ? decision.delayMs : undefined + } it.for([429, 502, 503, 504])('retries status %i', (status) => { - expect(decide({ status })).toBeDefined() + expect(scheduledDelayMs({ status })).toBeDefined() }) it.for([200, 400, 401, 403, 404, 409, 500, 501])( 'does not retry status %i', (status) => { - expect(decide({ status })).toBeUndefined() + expect(scheduledDelayMs({ status })).toBeUndefined() } ) it.for(['GET', 'get', 'HEAD', 'OPTIONS'])('retries %s', (method) => { - expect(decide({ method })).toBeDefined() + expect(scheduledDelayMs({ method })).toBeDefined() }) it.for(['POST', 'PUT', 'PATCH', 'DELETE'])( 'does not retry %s by default', (method) => { - expect(decide({ method })).toBeUndefined() + expect(scheduledDelayMs({ method })).toBeUndefined() } ) @@ -74,13 +77,13 @@ describe('retryDelayMs', () => { ...DEFAULT_RETRY_POLICY, retryNonIdempotent: true } - expect(decide({ method: 'POST', policy })).toBeDefined() + expect(scheduledDelayMs({ method: 'POST', policy })).toBeDefined() }) it('stops once the attempts are spent', () => { const { maxAttempts } = DEFAULT_RETRY_POLICY - expect(decide({ attempt: maxAttempts - 1 })).toBeDefined() - expect(decide({ attempt: maxAttempts })).toBeUndefined() + expect(scheduledDelayMs({ attempt: maxAttempts - 1 })).toBeDefined() + expect(scheduledDelayMs({ attempt: maxAttempts })).toBeUndefined() }) it('backs off exponentially up to the cap', () => { @@ -91,7 +94,7 @@ describe('retryDelayMs', () => { retryNonIdempotent: false } const delays = [1, 2, 3, 4, 5].map((attempt) => - decide({ attempt, policy, random: () => 1 }) + scheduledDelayMs({ attempt, policy, random: () => 1 }) ) expect(delays).toEqual([100, 200, 400, 500, 500]) }) @@ -99,7 +102,7 @@ describe('retryDelayMs', () => { it.for([0, 0.25, 0.5, 0.99, 1])( 'keeps jitter within half to full backoff (random=%f)', (random) => { - const delay = decide({ attempt: 2, random: () => random }) + const delay = scheduledDelayMs({ attempt: 2, random: () => random }) const backoff = DEFAULT_RETRY_POLICY.baseDelayMs * 2 expect(delay).toBeGreaterThanOrEqual(backoff / 2) expect(delay).toBeLessThanOrEqual(backoff) @@ -112,15 +115,15 @@ describe('retryDelayMs', () => { ])( 'adds jitter on top of Retry-After, never below it (random=%f)', ([random, expected]) => { - expect(decide({ readRetryAfter: () => '4', random: () => random })).toBe( - expected - ) + expect( + scheduledDelayMs({ readRetryAfter: () => '4', random: () => random }) + ).toBe(expected) } ) it('reads an HTTP-date Retry-After against the clock', () => { expect( - decide({ + scheduledDelayMs({ readRetryAfter: () => 'Sun, 18 Oct 2026 12:00:03 GMT', random: () => 0 }) @@ -129,20 +132,20 @@ describe('retryDelayMs', () => { it('gives up when Retry-After exceeds the cap', () => { const tooLong = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000 + 1) - expect(decide({ readRetryAfter: () => tooLong })).toBeUndefined() + expect(scheduledDelayMs({ readRetryAfter: () => tooLong })).toBeUndefined() }) it('accepts Retry-After exactly at the cap', () => { const atCap = String(DEFAULT_RETRY_POLICY.maxDelayMs / 1000) - expect(decide({ readRetryAfter: () => atCap })).toBe( + expect(scheduledDelayMs({ readRetryAfter: () => atCap })).toBe( DEFAULT_RETRY_POLICY.maxDelayMs ) }) it('does not read the header for a response it will not retry', () => { const readRetryAfter = vi.fn(() => '1') - decide({ status: 500, readRetryAfter }) - decide({ method: 'POST', readRetryAfter }) + scheduledDelayMs({ status: 500, readRetryAfter }) + scheduledDelayMs({ method: 'POST', readRetryAfter }) expect(readRetryAfter).not.toHaveBeenCalled() }) }) @@ -183,3 +186,34 @@ describe('sleep', () => { await expect(sleep(1_000, controller.signal)).rejects.toBe('stop') }) }) + +describe('nextRetry', () => { + const decide = (overrides: Partial[0]> = {}) => + nextRetry({ + method: 'GET', + status: 503, + readRetryAfter: () => null, + attempt: 1, + random: () => 1, + now: () => NOW, + ...overrides + }) + + it.for<[string, Partial[0]>, string]>([ + ['a non-retryable status', { status: 500 }, 'not_retryable'], + ['a write', { method: 'POST' }, 'not_retryable'], + ['the last attempt', { attempt: 3 }, 'attempts_spent'], + ['a no-retry policy', { policy: NO_RETRY_POLICY }, 'attempts_spent'], + [ + 'a Retry-After past the cap', + { readRetryAfter: () => '30' }, + 'retry_after_too_long' + ] + ])('stops on %s and says why', ([_name, overrides, reason]) => { + expect(decide(overrides)).toEqual({ retry: false, reason }) + }) + + it('schedules the next attempt after the backoff delay', () => { + expect(decide()).toEqual({ retry: true, delayMs: 500 }) + }) +}) diff --git a/src/utils/retryPolicy.ts b/src/utils/retryPolicy.ts index 56d55fdb238..77928f0b491 100644 --- a/src/utils/retryPolicy.ts +++ b/src/utils/retryPolicy.ts @@ -63,8 +63,18 @@ interface RetryDecisionInput { readonly now?: () => number } -/** Delay before the next attempt, or `undefined` to stop retrying. */ -export function retryDelayMs({ +/** Why {@link nextRetry} stopped instead of scheduling another attempt. */ +export type RetryStopReason = + | 'not_retryable' + | 'attempts_spent' + | 'retry_after_too_long' + +export type RetryDecision = + | { readonly retry: true; readonly delayMs: number } + | { readonly retry: false; readonly reason: RetryStopReason } + +/** Whether to send another attempt, and after how long. */ +export function nextRetry({ method, status, readRetryAfter, @@ -72,14 +82,17 @@ export function retryDelayMs({ policy = DEFAULT_RETRY_POLICY, random = Math.random, now = Date.now -}: RetryDecisionInput): number | undefined { - if (attempt >= policy.maxAttempts) return undefined - if (!isRetryableStatus(status)) return undefined - if (!canRetryMethod(method, policy)) return undefined +}: RetryDecisionInput): RetryDecision { + if (!isRetryableStatus(status) || !canRetryMethod(method, policy)) { + return { retry: false, reason: 'not_retryable' } + } + if (attempt >= policy.maxAttempts) { + return { retry: false, reason: 'attempts_spent' } + } const retryAfterMs = parseRetryAfterMs(readRetryAfter(), now()) if (retryAfterMs !== undefined && retryAfterMs > policy.maxDelayMs) { - return undefined + return { retry: false, reason: 'retry_after_too_long' } } const ceiling = Math.min( @@ -87,9 +100,15 @@ export function retryDelayMs({ policy.baseDelayMs * 2 ** (attempt - 1) ) const jittered = ceiling / 2 + (random() * ceiling) / 2 - if (retryAfterMs === undefined) return jittered + if (retryAfterMs === undefined) return { retry: true, delayMs: jittered } const spread = (random() * ceiling) / 2 - return Math.min(policy.maxDelayMs, Math.max(jittered, retryAfterMs + spread)) + return { + retry: true, + delayMs: Math.min( + policy.maxDelayMs, + Math.max(jittered, retryAfterMs + spread) + ) + } } export function sleep(ms: number, signal?: AbortSignal): Promise { diff --git a/src/utils/statusRetryTelemetry.test.ts b/src/utils/statusRetryTelemetry.test.ts new file mode 100644 index 00000000000..33f712e60e8 --- /dev/null +++ b/src/utils/statusRetryTelemetry.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest' + +import type { StatusRetrySettlement } from '@/utils/statusRetryTelemetry' +import { abortEnd, classifyStatusRetry } from '@/utils/statusRetryTelemetry' + +describe('classifyStatusRetry', () => { + it.for<[string, StatusRetrySettlement, string | undefined]>([ + [ + 'a first answer', + { end: 'answered', attempts: 1, finalStatus: 200, retryScheduled: false }, + undefined + ], + [ + 'a success after a retry', + { end: 'answered', attempts: 2, finalStatus: 204, retryScheduled: true }, + 'recovered' + ], + [ + 'an error status after a retry', + { end: 'answered', attempts: 2, finalStatus: 500, retryScheduled: true }, + 'failed' + ], + [ + 'a redirect the caller asked to see after a retry', + { end: 'answered', attempts: 2, finalStatus: 302, retryScheduled: true }, + 'failed' + ], + [ + 'an opaque answer after a retry', + { end: 'answered', attempts: 2, finalStatus: 0, retryScheduled: true }, + 'failed' + ], + [ + 'spent attempts', + { + end: 'attempts_spent', + attempts: 3, + finalStatus: 503, + retryScheduled: true + }, + 'exhausted' + ], + [ + 'a no-retry read', + { + end: 'attempts_spent', + attempts: 1, + finalStatus: 503, + retryScheduled: false + }, + undefined + ], + [ + 'a long Retry-After', + { + end: 'retry_after_too_long', + attempts: 1, + finalStatus: 503, + retryScheduled: false + }, + 'retry_after_too_long' + ], + [ + 'a timeout after a scheduled retry', + { end: 'timeout', attempts: 1, finalStatus: 503, retryScheduled: true }, + 'timeout_budget' + ], + [ + 'a timeout on the first request', + { end: 'timeout', attempts: 1, retryScheduled: false }, + undefined + ], + [ + 'no response after a retry', + { end: 'network', attempts: 2, finalStatus: 503, retryScheduled: true }, + 'failed' + ], + [ + 'no response on the first request', + { end: 'network', attempts: 1, retryScheduled: false }, + undefined + ], + [ + 'a caller cancel', + { end: 'cancelled', attempts: 1, finalStatus: 503, retryScheduled: true }, + undefined + ] + ])('classifies %s', ([_name, settlement, outcome]) => { + expect(classifyStatusRetry(settlement)).toBe(outcome) + }) +}) + +describe('abortEnd', () => { + it('treats an unaborted failure as a network failure', () => { + expect(abortEnd(new AbortController().signal)).toBe('network') + expect(abortEnd(undefined)).toBe('network') + }) + + it('tells the request timeout from a caller abort', () => { + const timedOut = new AbortController() + timedOut.abort(new DOMException('Fetch timeout', 'TimeoutError')) + const cancelled = new AbortController() + cancelled.abort() + + expect(abortEnd(timedOut.signal)).toBe('timeout') + expect(abortEnd(cancelled.signal)).toBe('cancelled') + }) +}) diff --git a/src/utils/statusRetryTelemetry.ts b/src/utils/statusRetryTelemetry.ts new file mode 100644 index 00000000000..b000ff53150 --- /dev/null +++ b/src/utils/statusRetryTelemetry.ts @@ -0,0 +1,141 @@ +import { useTelemetry } from '@/platform/telemetry' +import type { + FetchStatusRetryMetadata, + FetchStatusRetryOutcome +} from '@/platform/telemetry/types' + +import { getFetchRouteTemplate } from '@/utils/fetchRouteTemplate' +import type { RetryStopReason } from '@/utils/retryPolicy' + +/** How a request's retry loop ended. */ +export type StatusRetryEnd = + | 'answered' + | Exclude + | 'timeout' + | 'network' + | 'cancelled' + +export interface StatusRetrySettlement { + readonly end: StatusRetryEnd + /** Requests sent, including the first. */ + readonly attempts: number + readonly finalStatus?: number + /** Whether an earlier answer was retryable and a retry was scheduled. */ + readonly retryScheduled: boolean +} + +function isSuccessfulStatus(status: number | undefined): boolean { + return status !== undefined && status >= 200 && status < 300 +} + +/** The outcome to record for a settled request, or `undefined` for none. */ +export function classifyStatusRetry({ + end, + attempts, + finalStatus, + retryScheduled +}: StatusRetrySettlement): FetchStatusRetryOutcome | undefined { + switch (end) { + case 'cancelled': + return undefined + case 'retry_after_too_long': + return end + case 'attempts_spent': + return attempts > 1 ? 'exhausted' : undefined + case 'timeout': + return retryScheduled ? 'timeout_budget' : undefined + case 'network': + return retryScheduled ? 'failed' : undefined + case 'answered': + if (!retryScheduled) return undefined + return isSuccessfulStatus(finalStatus) ? 'recovered' : 'failed' + } +} + +/** How the loop ended when it stopped on an answer. */ +export function endForStop(reason: RetryStopReason): StatusRetryEnd { + return reason === 'not_retryable' ? 'answered' : reason +} + +const MAX_EVENTS_PER_KEY_PER_WINDOW = 20 +const BUDGET_WINDOW_MS = 60_000 +const budgetPerKey = new Map() + +/** Forgets how many events each key has sent. */ +export function clearStatusRetryBudget(): void { + budgetPerKey.clear() +} + +function spendBudget(key: string): boolean { + const now = Date.now() + const budget = budgetPerKey.get(key) + if (!budget || now - budget.windowStart >= BUDGET_WINDOW_MS) { + budgetPerKey.set(key, { windowStart: now, sent: 1 }) + return true + } + if (budget.sent >= MAX_EVENTS_PER_KEY_PER_WINDOW) return false + budget.sent += 1 + return true +} + +function telemetryHost(host: string): string { + const knownHost = + host === location.host || + host === 'comfy.org' || + host.endsWith('.comfy.org') + return knownHost ? host : 'other' +} + +function describeUrl( + url: string +): Pick { + const parsed = new URL(url, location.href) + return { + host: telemetryHost(parsed.host), + route: getFetchRouteTemplate(parsed.pathname) + } +} + +interface StatusRetryReport extends StatusRetrySettlement { + readonly transport: FetchStatusRetryMetadata['transport'] + readonly method: string + readonly url: () => string +} + +function recordStatusRetry(report: StatusRetryReport): void { + const outcome = classifyStatusRetry(report) + const telemetry = useTelemetry() + if (!outcome || !telemetry) return + const { host, route } = describeUrl(report.url()) + if (!spendBudget(`${host} ${route} ${outcome}`)) return + + telemetry.trackFetchStatusRetry({ + transport: report.transport, + method: report.method.toUpperCase(), + host, + route, + attempts: report.attempts, + ...(report.finalStatus !== undefined && { + final_status: report.finalStatus + }), + outcome + }) +} + +/** Records one settled request that hit a retryable status; never throws. */ +export function trackStatusRetry(report: StatusRetryReport): void { + try { + recordStatusRetry(report) + } catch (error) { + console.warn('[telemetry] status retry event dropped', error) + } +} + +/** Whether a failed or aborted fetch was the request timeout, the caller, or the network. */ +export function abortEnd(signal: AbortSignal | undefined): StatusRetryEnd { + if (!signal?.aborted) return 'network' + const reason: unknown = signal.reason + return reason instanceof DOMException && reason.name === 'TimeoutError' + ? 'timeout' + : 'cancelled' +}