Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ import type {
TabCountMetadata,
TelemetryDispatcher,
TelemetryProvider,
FetchStatusRetryMetadata,
FetchTimeoutMetadata,
TemplateFilterMetadata,
TemplateLibraryClosedMetadata,
Expand Down Expand Up @@ -601,4 +602,9 @@ export class TelemetryRegistry implements TelemetryDispatcher {
trackFetchTimeout(metadata: FetchTimeoutMetadata): void {
this.dispatch((provider) => provider.trackFetchTimeout?.(metadata))
}

// fallow-ignore-next-line unused-class-member

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit-pick (non-blocking): Please copy the external-tool constraint from commit fce3e34 into the PR description: Fallow cannot credit this concrete method through useTelemetry()’s TelemetryDispatcher return type, while the real registry/provider path is reachable. AGENTS.md asks that a justified rare suppression’s constraint be recorded there. Keep the machine directive itself free of prose as it is now.

trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void {
this.dispatch((provider) => provider.trackFetchStatusRetry?.(metadata))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,25 @@ describe('DatadogRumTelemetryProvider', () => {
})
})

it('records status retries as RUM actions', () => {
const metadata = {
transport: 'axios',
method: 'GET',
host: 'cloud.comfy.org',
route: '/api/billing/balance',
attempts: 3,
final_status: 503,
outcome: 'exhausted'
} as const

new DatadogRumTelemetryProvider().trackFetchStatusRetry(metadata)

expect(addAction).toHaveBeenCalledExactlyOnceWith(
TelemetryEvents.FETCH_STATUS_RETRY,
metadata
)
})

it('records fetch timeouts as RUM actions', () => {
new DatadogRumTelemetryProvider().trackFetchTimeout({
route: '/userdata/:resource',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { useCurrentUser } from '@/composables/auth/useCurrentUser'
import type {
AuthMetadata,
ExecutionOutcomeMetadata,
FetchStatusRetryMetadata,
FetchTimeoutMetadata,
ImageLoadFailureMetadata,
TelemetryProvider,
Expand Down Expand Up @@ -56,6 +57,10 @@ export class DatadogRumTelemetryProvider implements TelemetryProvider {
datadogRum.addAction(TelemetryEvents.FETCH_TIMEOUT, metadata)
}

trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void {
datadogRum.addAction(TelemetryEvents.FETCH_STATUS_RETRY, metadata)
}

trackUnifiedAuthRetry(metadata: UnifiedAuthRetryMetadata): void {
datadogRum.addAction(
metadata.outcome === 'succeeded'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,4 +148,32 @@ describe('SentryTelemetryProvider', () => {
node_type: 'KSampler'
})
})

it.for([
['recovered', 'info'],
['exhausted', 'warning'],
['retry_after_too_long', 'warning']
] as const)(
'records a %s status retry as a %s breadcrumb',
([outcome, level]) => {
const metadata = {
transport: 'fetch',
method: 'GET',
host: 'api.comfy.org',
route: '/customers/balance',
attempts: 2,
final_status: 503,
outcome
} as const

new SentryTelemetryProvider().trackFetchStatusRetry(metadata)

expect(mocks.addBreadcrumb).toHaveBeenCalledExactlyOnceWith({
category: 'fetch.status_retry',
message: outcome,
level,
data: metadata
})
}
)
})
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { useWorkflowStore } from '@/platform/workflow/management/stores/workflow
import type {
AuthMetadata,
ExecutionErrorMetadata,
FetchStatusRetryMetadata,
ExecutionSuccessMetadata,
NodeAddedMetadata,
ShellLayoutMetadata,
Expand All @@ -30,6 +31,15 @@ export class SentryTelemetryProvider implements TelemetryProvider {
this.updateAppContext()
}

trackFetchStatusRetry(metadata: FetchStatusRetryMetadata): void {
addBreadcrumb({
category: 'fetch.status_retry',
message: metadata.outcome,
level: metadata.outcome === 'recovered' ? 'info' : 'warning',
data: { ...metadata }
})
}

trackNodeAdded({ node_type, source }: NodeAddedMetadata): void {
addBreadcrumb({
category: 'workflow',
Expand Down
25 changes: 25 additions & 0 deletions src/platform/telemetry/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1424,6 +1424,28 @@ export interface FetchTimeoutMetadata {
timeout_ms: number
}

/** How a request that got a retryable status (429/502/503/504) ended. */
export type FetchStatusRetryOutcome =
| 'recovered'
| 'exhausted'
| 'retry_after_too_long'
| 'timeout_budget'
| 'failed'

/** One request that hit a retryable status in the shared status retry. */
export interface FetchStatusRetryMetadata {
transport: 'fetch' | 'axios'
method: string
host: string
/** Route group only, as in `app:fetch_timeout`, e.g. `/billing/:resource`. */
route: string
/** Requests sent, including the first. */
attempts: number
/** The last status the request got, if any. */
final_status?: number
outcome: FetchStatusRetryOutcome
}

/**
* Telemetry provider interface for individual providers.
* All methods are optional - providers only implement what they need.
Expand Down Expand Up @@ -1610,6 +1632,7 @@ export interface TelemetryProvider {

// Network error events
trackFetchTimeout?(metadata: FetchTimeoutMetadata): void
trackFetchStatusRetry?(metadata: FetchStatusRetryMetadata): void
}

/**
Expand Down Expand Up @@ -1790,6 +1813,7 @@ export const TelemetryEvents = {

// Network
FETCH_TIMEOUT: 'app:fetch_timeout',
FETCH_STATUS_RETRY: 'app:fetch_status_retry',

WORKSPACE_SCOPE_MISMATCH: 'workspace_scope_mismatch'
} as const
Expand Down Expand Up @@ -1896,4 +1920,5 @@ export type TelemetryEventProperties =
| AgentPaywallCtaMetadata
| AgentCreditTransitionNoticeMetadata
| FetchTimeoutMetadata
| FetchStatusRetryMetadata
| WorkspaceScopeMismatchMetadata
47 changes: 1 addition & 46 deletions src/scripts/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ import type {
import type { ComfyNodeDef } from '@/schemas/nodeDefSchema'
import type { AuthCredential, AuthScheme } from '@/types/authTypes'
import type { NodeExecutionId } from '@/types/nodeIdentification'
import { getFetchRouteTemplate } from '@/utils/fetchRouteTemplate'
import { retryingSend } from '@/utils/fetchWithRetry'
import { NO_RETRY_POLICY } from '@/utils/retryPolicy'
import type { RetryPolicy } from '@/utils/retryPolicy'
Expand Down Expand Up @@ -161,52 +162,6 @@ interface FetchApiOptions extends RequestInit {
skipStatusRetry?: boolean
}

const FETCH_ROUTE_GROUPS = new Set([
'assets',
'embeddings',
'experiment',
'extensions',
'features',
'files',
'folder_paths',
'free',
'global_subgraphs',
'history',
'hub',
'internal',
'interrupt',
'jobs',
'logs',
'models',
'node_replacements',
'object_info',
'prompt',
'providers',
'queue',
'secrets',
'settings',
'system_stats',
'upload',
'user',
'userdata',
'users',
'video_metadata',
'view',
'view_metadata',
'workflow_templates',
'workflows',
'workspace'
])

function getFetchRouteTemplate(route: string): string {
const segments = (route.split(/[?#]/)[0] ?? '').split('/').filter(Boolean)
const routeSegments = segments[0] === 'api' ? segments.slice(1) : segments
const [routeGroup, ...resources] = routeSegments

if (!routeGroup || !FETCH_ROUTE_GROUPS.has(routeGroup)) return '/other'
return `/${routeGroup}${resources.length ? '/:resource' : ''}`
}

/**
* Options for queuePrompt method
*/
Expand Down
Loading
Loading