diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index f79bb5b8143..c81e73c7631 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -13,6 +13,7 @@ import type { AgentPanelOpenedMetadata, AgentPaywallCtaMetadata, AgentPaywallShownMetadata, + AgentSendFailedMetadata, AgentWorkflowAppliedMetadata, BillingTelemetryEvent, CheckoutJourneyTelemetryEvent, @@ -300,6 +301,14 @@ describe('TelemetryRegistry', () => { client_message_id: 'cm-1', input_method: 'typed' } satisfies AgentMessageSentMetadata + const sendFailedMetadata = { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: 'th-1', + client_message_id: 'cm-1' + } satisfies AgentSendFailedMetadata const consentShownMetadata = { trigger: 'button_click' } satisfies AgentConsentShownMetadata @@ -364,6 +373,11 @@ describe('TelemetryRegistry', () => { invoke: (registry) => registry.trackAgentMessageSent(messageSentMetadata) }, + { + method: 'trackAgentSendFailed', + expected: { ...sendFailedMetadata }, + invoke: (registry) => registry.trackAgentSendFailed(sendFailedMetadata) + }, { method: 'trackAgentConsentShown', expected: { ...consentShownMetadata }, diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index c92b6344ed8..3b0cd16f14e 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -17,6 +17,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -419,6 +420,10 @@ export class TelemetryRegistry implements TelemetryDispatcher { this.dispatch((provider) => provider.trackAgentMessageSent?.(metadata)) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.dispatch((provider) => provider.trackAgentSendFailed?.(metadata)) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.dispatch((provider) => provider.trackAgentNodeTagged?.(metadata)) } diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts index 2afd588089d..795957cf080 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts @@ -96,6 +96,29 @@ describe('DatadogRumTelemetryProvider', () => { ) }) + it('records failed agent sends as RUM actions so they can be monitored', () => { + new DatadogRumTelemetryProvider().trackAgentSendFailed({ + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-1' + }) + + expect(addAction).toHaveBeenCalledExactlyOnceWith( + TelemetryEvents.AGENT_SEND_FAILED, + { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-1' + } + ) + }) + it('records terminal unified auth retry outcomes without request data', () => { new DatadogRumTelemetryProvider().trackUnifiedAuthRetry({ transport: 'axios', diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts index 1de4a8f38ae..ae5f388fde5 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts @@ -4,6 +4,7 @@ import { datadogRum } from '@datadog/browser-rum' import { useCurrentUser } from '@/composables/auth/useCurrentUser' import type { + AgentSendFailedMetadata, AuthMetadata, BillingTelemetryEvent, CheckoutJourneyTelemetryEvent, @@ -66,6 +67,14 @@ export class DatadogRumTelemetryProvider implements TelemetryProvider { ) } + /** + * A funnel drop-off worth a monitor, so ADR-TELEMETRY-ROUTING-0013 puts it + * here as well as in PostHog rather than only in the growth layer. + */ + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + datadogRum.addAction(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackImageLoadFailed(metadata: ImageLoadFailureMetadata): void { datadogRum.addAction(TelemetryEvents.IMAGE_LOAD_FAILED, metadata) } diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index 4bb635c90b7..6fd15b236a8 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -478,6 +478,32 @@ describe('PostHogTelemetryProvider', () => { ) }) + it('captures a failed send against the attempt it belongs to', async () => { + const provider = createProvider() + await vi.dynamicImportSettled() + + provider.trackAgentSendFailed({ + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: null, + client_message_id: 'client-message-1' + }) + + expect(hoisted.mockCapture).toHaveBeenCalledWith( + TelemetryEvents.AGENT_SEND_FAILED, + { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: null, + client_message_id: 'client-message-1' + } + ) + }) + it('captures link dedup drop events with metadata', async () => { const provider = createProvider() await vi.dynamicImportSettled() diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index 331cfff0fd5..48e65f3bebc 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -31,6 +31,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -786,6 +787,10 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_MESSAGE_SENT, metadata) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.trackEvent(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.trackEvent(TelemetryEvents.AGENT_NODE_TAGGED, metadata) } diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index 7a20a9d89f2..b3d791cb458 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -308,6 +308,26 @@ describe('HostTelemetrySink', () => { input_method: 'typed' } }, + { + name: TelemetryEvents.AGENT_SEND_FAILED, + track: (sink: HostTelemetrySink) => + sink.trackAgentSendFailed({ + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-3' + }), + properties: { + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-3' + } + }, { name: TelemetryEvents.AGENT_CONSENT_SHOWN, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.ts index 39fa501b585..32397f1e09d 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.ts @@ -21,6 +21,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -364,6 +365,10 @@ export class HostTelemetrySink implements TelemetryProvider { this.capture(TelemetryEvents.AGENT_MESSAGE_SENT, metadata) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.capture(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.capture(TelemetryEvents.AGENT_NODE_TAGGED, metadata) } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 2da6ef945b3..31d62141bea 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -666,6 +666,105 @@ export interface AgentMessageSentMetadata extends Record { client_message_id: string input_method: AgentInputMethod } +/** + * How far a send got before it failed. `agent_message_sent` counts attempts, so + * it fires for every send in this list too; the pair is what separates an + * attempt the server never saw from one it refused. + * + * - `not_dispatched` — the client stopped the send itself; no request was issued. + * - `no_response` — the request was issued and no HTTP status came back. It does + * **not** prove the server ignored it: a lost response and a refused request + * look the same from here, so a turn may exist. + * - `refused` — an HTTP error status came back, carried in `http_status`. + * - `unreadable_response` — a success status came back whose body the client + * could not parse, so the turn probably did start. + */ +type AgentSendFailureStage = + | 'not_dispatched' + | 'no_response' + | 'refused' + | 'unreadable_response' +/** + * `send_in_flight`: an earlier send of this session had not settled. + * `target_changed`: the workflow the send was composed against stopped being + * the target while the client resolved it. `session_reset`: a new chat or a + * thread load replaced the session mid-send. + */ +export type AgentSendNotDispatchedReason = + | 'send_in_flight' + | 'target_changed' + | 'session_reset' +/** + * `timeout` is the api layer's own response-header deadline, `aborted` any other + * cancellation, `network_error` a request the browser could not complete, and + * `unknown` anything else thrown on the send path — kept separate so a client + * bug cannot be read as a network failure. + */ +export type AgentSendNoResponseReason = + | 'timeout' + | 'aborted' + | 'network_error' + | 'unknown' +/** `admission_denied` is the pre-turn spend gate; `http_error` every other status. */ +type AgentSendRefusedReason = 'admission_denied' | 'http_error' +type AgentSendFailureReason = + | AgentSendNotDispatchedReason + | AgentSendNoResponseReason + | AgentSendRefusedReason + | 'malformed_body' +/** + * The pre-turn admission gate's own reason for declining, mirroring + * `AgentAdmissionError['error']['reason']`. Narrowed on purpose: a new value on + * that generated enum should fail to compile here rather than reach a dashboard + * unnoticed. + */ +type AgentAdmissionReason = 'no_funds' | 'manual_block' | 'funds_unavailable' +/** + * A failed send, classified by where it stopped. The fields that cannot apply to + * a stage are `null` rather than absent, so a query never has to distinguish a + * missing property from a real one. + */ +export type AgentSendFailure = + | { + stage: 'not_dispatched' + reason: AgentSendNotDispatchedReason + http_status: null + admission_reason: null + } + | { + stage: 'no_response' + reason: AgentSendNoResponseReason + http_status: null + admission_reason: null + } + | { + stage: 'refused' + reason: AgentSendRefusedReason + http_status: number + admission_reason: AgentAdmissionReason | null + } + | { + stage: 'unreadable_response' + reason: 'malformed_body' + http_status: null + admission_reason: null + } +/** + * Carries no server message and no request body: the reason enum is what a + * funnel can group by, and free text on a refusal is the one field that could + * carry user content (ADR-AGENT-OBSERVABILITY-0034 forbids raw errors in + * correlated fields). + */ +export interface AgentSendFailedMetadata extends Record { + stage: AgentSendFailureStage + reason: AgentSendFailureReason + http_status: number | null + admission_reason: AgentAdmissionReason | null + /** The thread the attempt was posted into, `null` when it opened a new one. */ + thread_id: string | null + /** The `agent_message_sent` attempt this failure belongs to. */ + client_message_id: string +} export interface AgentNodeTaggedMetadata extends Record { source: 'mention_picker' } @@ -1439,6 +1538,7 @@ export interface TelemetryProvider { trackAgentOnboardingShown?(): void trackAgentOnboardingStep?(metadata: AgentOnboardingStepMetadata): void trackAgentMessageSent?(metadata: AgentMessageSentMetadata): void + trackAgentSendFailed?(metadata: AgentSendFailedMetadata): void trackAgentNodeTagged?(metadata: AgentNodeTaggedMetadata): void trackAgentAttachButtonClicked?( metadata: AgentAttachButtonClickedMetadata @@ -1628,6 +1728,7 @@ export const TelemetryEvents = { AGENT_ONBOARDING_SHOWN: 'app:agent_onboarding_shown', AGENT_ONBOARDING_STEP: 'app:agent_onboarding_step', AGENT_MESSAGE_SENT: 'app:agent_message_sent', + AGENT_SEND_FAILED: 'app:agent_send_failed', AGENT_NODE_TAGGED: 'app:agent_node_tagged', AGENT_ATTACH_BUTTON_CLICKED: 'app:agent_attach_button_clicked', AGENT_WORKFLOW_APPLIED: 'app:agent_workflow_applied', diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index e936df7063f..4ab198faeac 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -478,6 +478,18 @@ async function sendFromComposer(text: string): Promise { await screen.findByRole('button', { name: 'Stop' }) } +/** + * A send that will not open a turn, so there is no Stop button to wait for; + * the caller waits on whatever the failure is expected to produce. + */ +async function sendFromComposerExpectingFailure(text: string): Promise { + const textbox = screen.getByRole('textbox') + await userEvent.click(textbox) + await userEvent.keyboard('{ArrowRight}') + await userEvent.paste(text) + await userEvent.click(screen.getByRole('button', { name: 'Send' })) +} + async function renderAndSend(text: string): Promise { renderWithSelectedTarget() await sendFromComposer(text) @@ -9043,4 +9055,94 @@ describe('AgentPanelRoot workflow binding', () => { toRootGraphId('wf-42-rotated') ) }) + + it.for([ + { + name: 'a credit refusal', + response: () => + json(402, { + error: { + message: 'Out of credits', + reason: 'no_funds', + type: 'PAYMENT_REQUIRED' + } + }), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + // gc-7: ingest hides the agent behind a 404 when its own flag read says + // no, so this send is refused by a surface with no other telemetry. + name: 'a server-side flag miss', + response: () => json(404, { error: 'not found' }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null + } + }, + { + name: 'a request the browser could not complete', + response: () => { + throw new TypeError('Failed to fetch') + }, + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + } + ])( + 'reports $name against the attempt the funnel counted', + async ({ response, expected }) => { + makeTab('wf-42') + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string, init?: RequestInit) => { + if (url.includes('/messages') && init?.method === 'POST') + return response() + if (url.includes('/messages')) return json(200, []) + if (url.includes('/agent/threads')) + return json(200, agentThreadList([])) + return new Response('{}', { status: 200 }) + }) + ) + telemetry.trackAgentMessageSent.mockClear() + telemetry.trackAgentSendFailed.mockClear() + renderWithSelectedTarget() + await screen.findByRole('textbox') + + await sendFromComposerExpectingFailure('build me a workflow') + + await waitFor(() => + expect(telemetry.trackAgentSendFailed).toHaveBeenCalledWith({ + ...expected, + thread_id: null, + client_message_id: 'client-message-1' + }) + ) + // The attempt event is consumed by the workflow refresh long before the + // POST resolves; both still name the same attempt. + expect( + telemetry.trackAgentMessageSent.mock.calls[0][0].client_message_id + ).toBe('client-message-1') + expect(telemetry.trackAgentSendFailed).toHaveBeenCalledOnce() + } + ) + + it('reports no send failure when the turn is accepted', async () => { + makeTab('wf-42') + mockMessagesEndpoint('wf-42') + telemetry.trackAgentSendFailed.mockClear() + + await renderAndSend('build me a workflow') + + expect(telemetry.trackAgentSendFailed).not.toHaveBeenCalled() + }) }) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 4092543e9d3..5417c7ab53d 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -21,6 +21,7 @@ import { useTelemetry } from '@/platform/telemetry' import type { AgentMessageSentMetadata, AgentRunApprovalDecision, + AgentSendFailure, AgentStopMethod } from '@/platform/telemetry/types' import { useSettingStore } from '@/platform/settings/settingStore' @@ -696,6 +697,18 @@ function reportPendingSend(): void { }) } +/** + * Reports the failure of the send currently in flight against the attempt + * `app:agent_message_sent` counted. Held as a closure rather than as the + * attempt's identity, because the workflow refresh consumes `pendingSend` + * before the POST is even issued: by the time a send fails there is no pending + * report left to read the id from, and an attempt-less failure event would be + * unjoinable. Scoped to one `sendMessage` call like `pendingSend`, and single + * flight for the same reason: the composer refuses a submit while one is + * pending. + */ +let reportSendFailure: ((failure: AgentSendFailure) => void) | null = null + const { sendMessage, stopTurn, @@ -721,6 +734,7 @@ const { events, onThreadStarted: (source) => useTelemetry()?.trackAgentThreadStarted({ source }), + onSendFailed: (failure) => reportSendFailure?.(failure), onAskResolved: forgetApproval, workflow: { initialize: agentPanelStore.initializeTargetTracking, @@ -1260,6 +1274,13 @@ const { submit: onSend } = useAgentDraftSubmission({ originContext === undefined ? null : { tabPath: originContext.tabPath } } const selectionWorkflow = selectedTarget.value + const attemptThreadId = threadId.value + reportSendFailure = (failure) => + useTelemetry()?.trackAgentSendFailed({ + ...failure, + thread_id: attemptThreadId, + client_message_id: meta.clientMessageId + }) try { return await sendMessage(text, attachments, nodes, references, () => selectionWorkflow ? cloudIdFor(selectionWorkflow) : undefined @@ -1268,6 +1289,7 @@ const { submit: onSend } = useAgentDraftSubmission({ // Normally already consumed by the refresh. A send rejected before it // gets that far still reports here, so the funnel counts the attempt. reportPendingSend() + reportSendFailure = null } } }) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index a73a1ba31e1..c9afcf27b5b 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -2966,3 +2966,212 @@ describe('thread resume (B17)', () => { expect(threads[0]).toMatchObject({ id: 'th-9', title: 'build a duck' }) }) }) + +describe('failed send reporting', () => { + it.for([ + { + name: 'a credit refusal, with the gate reason attached', + error: () => admissionError('no_funds', 'Out of credits'), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + name: 'a server error, which no gate reason explains', + error: () => new AgentApiError('server exploded', 500, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + }, + { + name: 'a request that never reached the service', + error: () => new TypeError('Failed to fetch'), + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + } + ])('reports $name', async ({ error, expected }) => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi + .fn() + .mockRejectedValue(error()) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed.mock.calls).toEqual([[expected]]) + }) + + it('reports nothing when the turn is accepted', async () => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest(), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(true) + + expect(onSendFailed).not.toHaveBeenCalled() + }) + + it('reports a send refused while an earlier one is still in flight', async () => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest(), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + const first = session.sendMessage('first') + expect(await session.sendMessage('second')).toBe(false) + expect(await first).toBe(true) + + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'send_in_flight', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a send abandoned because its target stopped being reachable', async () => { + const onSendFailed = vi.fn() + let reachable = true + const postMessage = vi.fn() + const session = useAgentSession({ + rest: fakeRest({ postMessage }), + events: fakeEvents().source, + onSendFailed, + workflow: { + current: (origin) => + origin !== undefined && !reachable + ? undefined + : { id: 'wf-a', tabPath: 'tab-a' }, + adopted: vi.fn(), + prepare: async () => { + reachable = false + } + } + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(postMessage).not.toHaveBeenCalled() + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'target_changed', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a send the user replaced with a new chat before it was posted', async () => { + const onSendFailed = vi.fn() + const postMessage = vi.fn() + let reset: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ postMessage }), + events: fakeEvents().source, + onSendFailed, + workflow: { + current: () => undefined, + adopted: vi.fn(), + prepare: async () => reset() + } + }) + session.start() + reset = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(postMessage).not.toHaveBeenCalled() + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'session_reset', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a refusal that lands after the session moved on, because the attempt was still counted', async () => { + const onSendFailed = vi.fn() + let abandon: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi.fn(async () => { + abandon() + throw new AgentApiError('server exploded', 500, undefined) + }) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + abandon = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + ] + ]) + }) + + it('reports nothing for an acknowledgement dropped after the session moved on, because the turn exists', async () => { + const onSendFailed = vi.fn() + let abandon: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi.fn(async () => { + abandon() + return { thread_id: 'th-1', message_id: 'msg-1' } + }) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + abandon = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed).not.toHaveBeenCalled() + }) +}) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index 98bc60bcfa4..e951836c1c0 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -4,6 +4,7 @@ import { i18n } from '@/i18n' import { useTelemetry } from '@/platform/telemetry' import { reportError } from '@/platform/telemetry/reportError' import type { + AgentSendFailure, AgentStopClickedMetadata, AgentStopMethod, AgentThreadStartSource, @@ -25,7 +26,6 @@ import { isAgentEvent, parseAgentWsEvent, toTurnId, - zAgentAdmissionError, zDisownedWorkflowError } from '../../schemas/agentApiSchema' import { AgentApiError } from '../../services/agent/agentRestClient' @@ -38,6 +38,11 @@ import type { import { useAgentConversationStore } from '../../stores/agent/agentConversationStore' import { useAgentWorkflowTabBindingStore } from '../../stores/agent/agentWorkflowTabBindingStore' import type { WorkflowReference } from '../../types/workflowReference' +import { + classifyAgentSendError, + parseAdmissionError, + sendNotDispatched +} from '../../utils/agentSendFailure' import { serializeWorkflowReferences } from '../../utils/workflowReferenceText' export interface AgentEventSource { @@ -93,6 +98,13 @@ export interface AgentSessionDeps { events: AgentEventSource onThreadStarted?: (source: AgentSessionThreadStartSource) => void onAskResolved?: (askId: string) => void + /** + * Once per send attempt that produced no turn. A turn the server accepted + * after the session moved on is not reported here: the ack was dropped by the + * client, but the turn exists, and counting it as a failed send would + * overstate the loss the event is measuring. + */ + onSendFailed?: (failure: AgentSendFailure) => void workflow?: { /** Resolve fresh versus restored startup before asynchronous hydration. */ initialize?(hasThread: boolean): void @@ -147,16 +159,6 @@ function consumeStopPendingAck() { return pending } -function parseAdmissionError(error: unknown) { - if (!(error instanceof AgentApiError)) return undefined - const parsed = zAgentAdmissionError.safeParse(error.body) - if (!parsed.success) return undefined - const expectedStatus = - parsed.data.error.type === 'PAYMENT_REQUIRED' ? 402 : 503 - if (error.status !== expectedStatus) return undefined - return { ...parsed.data.error, retryAfterSeconds: error.retryAfterSeconds } -} - function disownsWorkflow(error: unknown): boolean { return ( error instanceof AgentApiError && @@ -166,7 +168,14 @@ function disownsWorkflow(error: unknown): boolean { } export function useAgentSession(deps: AgentSessionDeps) { - const { rest, events, onThreadStarted, onAskResolved, workflow } = deps + const { + rest, + events, + onThreadStarted, + onAskResolved, + onSendFailed, + workflow + } = deps const threadStorageKey = StorageKeys.agentThread(getWorkspaceId()) clearLegacyAgentStorage() @@ -533,6 +542,10 @@ export function useAgentSession(deps: AgentSessionDeps) { if (pendingStop !== null) void stopTurn(pendingStop.method) } + function reportSendFailure(failure: AgentSendFailure): void { + onSendFailed?.(failure) + } + function recordSendError(error: unknown, text: string): void { const admission = parseAdmissionError(error) if (admission?.reason === 'no_funds') { @@ -604,10 +617,14 @@ export function useAgentSession(deps: AgentSessionDeps) { let sentContext: WorkflowTurnContext | undefined try { await prepareWorkflow() - if (generation !== loadGeneration) return false + if (generation !== loadGeneration) { + reportSendFailure(sendNotDispatched('session_reset')) + return false + } const wfContext = workflow?.current(origin) if (workflowTargetChanged(originContext, wfContext)) { recordUnavailableTarget(text) + reportSendFailure(sendNotDispatched('target_changed')) return false } sentContext = wfContext @@ -621,6 +638,7 @@ export function useAgentSession(deps: AgentSessionDeps) { workflowReferences, selectionWorkflowId ) + // A dropped ack is not a failed send - see onSendFailed's contract. if (generation !== loadGeneration) return false acceptTurn(ack, text, wfContext, attachments, tags, workflowReferences) return true @@ -629,6 +647,9 @@ export function useAgentSession(deps: AgentSessionDeps) { // persisted, so a refusal that lands after newChat()/loadThread() has // moved on still has to release, or the dead id survives the reload. releaseDisownedWorkflow(sentContext, error) + // Also before it: the attempt was counted when it was made, so the + // refusal has to be counted even when nothing is left to render it to. + reportSendFailure(classifyAgentSendError(error)) if (generation !== loadGeneration) return false recordSendError(error, text) return false @@ -656,6 +677,7 @@ export function useAgentSession(deps: AgentSessionDeps) { text, i18n.global.t('agent.sendBusy') ) + reportSendFailure(sendNotDispatched('send_in_flight')) return false } promptEditState.value = { phase: 'idle' } diff --git a/src/workbench/extensions/agent/utils/agentSendFailure.test.ts b/src/workbench/extensions/agent/utils/agentSendFailure.test.ts new file mode 100644 index 00000000000..811d32cf914 --- /dev/null +++ b/src/workbench/extensions/agent/utils/agentSendFailure.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +import type { AgentSendFailure } from '@/platform/telemetry/types' + +import { zAgentAdmissionError } from '../schemas/agentApiSchema' +import { AgentApiError } from '../services/agent/agentRestClient' + +import { classifyAgentSendError, sendNotDispatched } from './agentSendFailure' + +function admission( + reason: 'no_funds' | 'manual_block' | 'funds_unavailable', + status: number +): AgentApiError { + const body = zAgentAdmissionError.parse({ + error: { + message: 'declined', + reason, + type: + reason === 'funds_unavailable' + ? 'SERVICE_UNAVAILABLE' + : 'PAYMENT_REQUIRED' + } + }) + return new AgentApiError('declined', status, body) +} + +/** What the turn schema actually throws when the ack body is the wrong shape. */ +function schemaError(): unknown { + const result = z.object({ thread_id: z.string() }).safeParse({}) + return result.success ? undefined : result.error +} + +describe('classifyAgentSendError', () => { + it.for([ + { + name: 'a credit refusal', + error: () => admission('no_funds', 402), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + name: 'a billing outage', + error: () => admission('funds_unavailable', 503), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 503, + admission_reason: 'funds_unavailable' + } + }, + { + name: 'a blocked workspace', + error: () => admission('manual_block', 402), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'manual_block' + } + }, + { + // The server-side flag miss gc-7 named: ingest hides the feature behind a + // 404, so the status is the only evidence the send was refused at all. + name: 'a server-side flag miss', + error: () => new AgentApiError('not found', 404, { error: 'not found' }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null + } + }, + { + name: 'a single-active-turn conflict', + error: () => new AgentApiError('conflict', 409, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 409, + admission_reason: null + } + }, + { + name: 'an agent service outage', + error: () => new AgentApiError('unavailable', 502, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 502, + admission_reason: null + } + }, + { + // A 402 whose body does not parse is still a refusal; it just cannot be + // attributed to the admission gate's reason. + name: 'a 402 with no admission body', + error: () => + new AgentApiError('payment required', 402, { + error: { message: 'payment required', type: 'PAYMENT_REQUIRED' } + }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 402, + admission_reason: null + } + }, + { + name: 'an admission body on the wrong status', + error: () => admission('no_funds', 500), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + }, + { + name: 'a request the browser could not complete', + error: () => new TypeError('Failed to fetch'), + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + }, + { + name: "the api layer's own response deadline", + error: () => new DOMException('Fetch timeout', 'TimeoutError'), + expected: { + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null + } + }, + { + name: 'a cancelled request', + error: () => new DOMException('Aborted', 'AbortError'), + expected: { + stage: 'no_response', + reason: 'aborted', + http_status: null, + admission_reason: null + } + }, + { + // Kept out of network_error on purpose: a bug on the send path must not + // read as connectivity. + name: 'an unexpected throw on the send path', + error: () => new RangeError('bad index'), + expected: { + stage: 'no_response', + reason: 'unknown', + http_status: null, + admission_reason: null + } + }, + { + name: 'a non-error rejection', + error: () => 'nope', + expected: { + stage: 'no_response', + reason: 'unknown', + http_status: null, + admission_reason: null + } + }, + { + name: 'an acknowledgement that does not match the schema', + error: schemaError, + expected: { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + }, + { + name: 'an acknowledgement that is not json', + error: () => new SyntaxError('Unexpected token < in JSON'), + expected: { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + } + ])('classifies $name', ({ error, expected }) => { + expect(classifyAgentSendError(error())).toEqual(expected) + }) +}) + +describe('sendNotDispatched', () => { + it.for(['send_in_flight', 'target_changed', 'session_reset'] as const)( + 'reports %s with no status, because no request was issued', + (reason) => { + const expected: AgentSendFailure = { + stage: 'not_dispatched', + reason, + http_status: null, + admission_reason: null + } + expect(sendNotDispatched(reason)).toEqual(expected) + } + ) +}) diff --git a/src/workbench/extensions/agent/utils/agentSendFailure.ts b/src/workbench/extensions/agent/utils/agentSendFailure.ts new file mode 100644 index 00000000000..1fd0e2fdc57 --- /dev/null +++ b/src/workbench/extensions/agent/utils/agentSendFailure.ts @@ -0,0 +1,86 @@ +import type { + AgentSendFailure, + AgentSendNoResponseReason, + AgentSendNotDispatchedReason +} from '@/platform/telemetry/types' + +import { zAgentAdmissionError } from '../schemas/agentApiSchema' +import { AgentApiError } from '../services/agent/agentRestClient' + +export function parseAdmissionError(error: unknown) { + if (!(error instanceof AgentApiError)) return undefined + const parsed = zAgentAdmissionError.safeParse(error.body) + if (!parsed.success) return undefined + const expectedStatus = + parsed.data.error.type === 'PAYMENT_REQUIRED' ? 402 : 503 + if (error.status !== expectedStatus) return undefined + return { ...parsed.data.error, retryAfterSeconds: error.retryAfterSeconds } +} + +export function sendNotDispatched( + reason: AgentSendNotDispatchedReason +): AgentSendFailure { + return { + stage: 'not_dispatched', + reason, + http_status: null, + admission_reason: null + } +} + +/** + * Errors are matched on `name`, never on class identity: the response schema + * comes from `@comfyorg/ingest-types`, which declares its own zod dependency, so + * a second copy of zod in the tree would make `instanceof ZodError` silently + * false and reclassify a parse failure as a network one. + */ +function errorName(error: unknown): string | undefined { + return error instanceof Error ? error.name : undefined +} + +/** + * Zod and `Response.json()` both reject only after the status line was read and + * found ok, so these are the two throws that mean "the turn may well have + * started and the client cannot tell". + */ +function isUnreadableResponse(error: unknown): boolean { + const name = errorName(error) + return name === 'ZodError' || name === 'SyntaxError' +} + +function noResponseReason(error: unknown): AgentSendNoResponseReason { + const name = errorName(error) + if (name === 'TimeoutError') return 'timeout' + if (name === 'AbortError') return 'aborted' + return name === 'TypeError' ? 'network_error' : 'unknown' +} + +/** + * The turn POST rejected. Only an `AgentApiError` proves a status line came + * back, so every other throw is classified from what it says about itself + * rather than assumed to be the network. + */ +export function classifyAgentSendError(error: unknown): AgentSendFailure { + if (error instanceof AgentApiError) { + const admission = parseAdmissionError(error) + return { + stage: 'refused', + reason: admission === undefined ? 'http_error' : 'admission_denied', + http_status: error.status, + admission_reason: admission?.reason ?? null + } + } + if (isUnreadableResponse(error)) + return { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + return { + stage: 'no_response', + reason: noResponseReason(error), + http_status: null, + admission_reason: null + } +}