From 199474d56f81eacfc23acc06a7557e1655645241 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 26 Sep 2026 02:46:30 -0700 Subject: [PATCH] feat(telemetry): report every failed agent send A successful send is instrumented in six places; a failed one reported to nothing - no PostHog event and no Sentry - so the largest single-step loss in the activation funnel (44% of senders never reach a turn) could only be inferred from the absence of backend events. `app:agent_send_failed` now fires once per send attempt that produces no turn, carrying where it stopped, why, the HTTP status when one came back, the admission gate's reason when the body has one, and the `client_message_id` of the `app:agent_message_sent` attempt it belongs to. Four stages rather than the usual transport/server split, because the send path has two ambiguous outcomes worth separating: `no_response` cannot prove the server did not accept the turn, and `unreadable_response` (a 2xx whose body the client rejected) probably means the turn started while the user saw a failure. `http_status` is a raw number - the generated error map documents 400/401/402/403/500/502/503 and omits the 404 a server-side flag miss produces and the 409 from the single-active-turn race. Observation only: no change to send, retry or error copy, no new surface, no server message or request body in the event. Dual-emitted to Datadog RUM per ADR-TELEMETRY-ROUTING-0013, which routes funnel-drop-off signals there alongside PostHog. --- .../telemetry/TelemetryRegistry.test.ts | 14 ++ src/platform/telemetry/TelemetryRegistry.ts | 5 + .../cloud/DatadogRumTelemetryProvider.test.ts | 23 ++ .../cloud/DatadogRumTelemetryProvider.ts | 9 + .../cloud/PostHogTelemetryProvider.test.ts | 26 +++ .../cloud/PostHogTelemetryProvider.ts | 5 + .../providers/host/HostTelemetrySink.test.ts | 20 ++ .../providers/host/HostTelemetrySink.ts | 5 + src/platform/telemetry/types.ts | 101 +++++++++ .../extensions/agent/AgentPanelRoot.test.ts | 102 +++++++++ .../extensions/agent/AgentPanelRoot.vue | 22 ++ .../composables/agent/useAgentSession.test.ts | 209 +++++++++++++++++ .../composables/agent/useAgentSession.ts | 48 ++-- .../agent/utils/agentSendFailure.test.ts | 213 ++++++++++++++++++ .../agent/utils/agentSendFailure.ts | 86 +++++++ 15 files changed, 875 insertions(+), 13 deletions(-) create mode 100644 src/workbench/extensions/agent/utils/agentSendFailure.test.ts create mode 100644 src/workbench/extensions/agent/utils/agentSendFailure.ts diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index f79bb5b8143..c81e73c7631 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -13,6 +13,7 @@ import type { AgentPanelOpenedMetadata, AgentPaywallCtaMetadata, AgentPaywallShownMetadata, + AgentSendFailedMetadata, AgentWorkflowAppliedMetadata, BillingTelemetryEvent, CheckoutJourneyTelemetryEvent, @@ -300,6 +301,14 @@ describe('TelemetryRegistry', () => { client_message_id: 'cm-1', input_method: 'typed' } satisfies AgentMessageSentMetadata + const sendFailedMetadata = { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: 'th-1', + client_message_id: 'cm-1' + } satisfies AgentSendFailedMetadata const consentShownMetadata = { trigger: 'button_click' } satisfies AgentConsentShownMetadata @@ -364,6 +373,11 @@ describe('TelemetryRegistry', () => { invoke: (registry) => registry.trackAgentMessageSent(messageSentMetadata) }, + { + method: 'trackAgentSendFailed', + expected: { ...sendFailedMetadata }, + invoke: (registry) => registry.trackAgentSendFailed(sendFailedMetadata) + }, { method: 'trackAgentConsentShown', expected: { ...consentShownMetadata }, diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index c92b6344ed8..3b0cd16f14e 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -17,6 +17,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -419,6 +420,10 @@ export class TelemetryRegistry implements TelemetryDispatcher { this.dispatch((provider) => provider.trackAgentMessageSent?.(metadata)) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.dispatch((provider) => provider.trackAgentSendFailed?.(metadata)) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.dispatch((provider) => provider.trackAgentNodeTagged?.(metadata)) } diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts index 2afd588089d..795957cf080 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.test.ts @@ -96,6 +96,29 @@ describe('DatadogRumTelemetryProvider', () => { ) }) + it('records failed agent sends as RUM actions so they can be monitored', () => { + new DatadogRumTelemetryProvider().trackAgentSendFailed({ + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-1' + }) + + expect(addAction).toHaveBeenCalledExactlyOnceWith( + TelemetryEvents.AGENT_SEND_FAILED, + { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-1' + } + ) + }) + it('records terminal unified auth retry outcomes without request data', () => { new DatadogRumTelemetryProvider().trackUnifiedAuthRetry({ transport: 'axios', diff --git a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts index 1de4a8f38ae..ae5f388fde5 100644 --- a/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/DatadogRumTelemetryProvider.ts @@ -4,6 +4,7 @@ import { datadogRum } from '@datadog/browser-rum' import { useCurrentUser } from '@/composables/auth/useCurrentUser' import type { + AgentSendFailedMetadata, AuthMetadata, BillingTelemetryEvent, CheckoutJourneyTelemetryEvent, @@ -66,6 +67,14 @@ export class DatadogRumTelemetryProvider implements TelemetryProvider { ) } + /** + * A funnel drop-off worth a monitor, so ADR-TELEMETRY-ROUTING-0013 puts it + * here as well as in PostHog rather than only in the growth layer. + */ + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + datadogRum.addAction(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackImageLoadFailed(metadata: ImageLoadFailureMetadata): void { datadogRum.addAction(TelemetryEvents.IMAGE_LOAD_FAILED, metadata) } diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index 4bb635c90b7..6fd15b236a8 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -478,6 +478,32 @@ describe('PostHogTelemetryProvider', () => { ) }) + it('captures a failed send against the attempt it belongs to', async () => { + const provider = createProvider() + await vi.dynamicImportSettled() + + provider.trackAgentSendFailed({ + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: null, + client_message_id: 'client-message-1' + }) + + expect(hoisted.mockCapture).toHaveBeenCalledWith( + TelemetryEvents.AGENT_SEND_FAILED, + { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds', + thread_id: null, + client_message_id: 'client-message-1' + } + ) + }) + it('captures link dedup drop events with metadata', async () => { const provider = createProvider() await vi.dynamicImportSettled() diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index 331cfff0fd5..48e65f3bebc 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -31,6 +31,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -786,6 +787,10 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_MESSAGE_SENT, metadata) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.trackEvent(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.trackEvent(TelemetryEvents.AGENT_NODE_TAGGED, metadata) } diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index 7a20a9d89f2..b3d791cb458 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -308,6 +308,26 @@ describe('HostTelemetrySink', () => { input_method: 'typed' } }, + { + name: TelemetryEvents.AGENT_SEND_FAILED, + track: (sink: HostTelemetrySink) => + sink.trackAgentSendFailed({ + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-3' + }), + properties: { + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null, + thread_id: 'thread-1', + client_message_id: 'client-message-3' + } + }, { name: TelemetryEvents.AGENT_CONSENT_SHOWN, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.ts index 39fa501b585..32397f1e09d 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.ts @@ -21,6 +21,7 @@ import type { AgentRunApprovalResolvedMetadata, AgentRunApprovalShownMetadata, AgentRunModeChangedMetadata, + AgentSendFailedMetadata, AgentStopClickedMetadata, AgentThreadStartedMetadata, AgentWorkflowBoundMetadata, @@ -364,6 +365,10 @@ export class HostTelemetrySink implements TelemetryProvider { this.capture(TelemetryEvents.AGENT_MESSAGE_SENT, metadata) } + trackAgentSendFailed(metadata: AgentSendFailedMetadata): void { + this.capture(TelemetryEvents.AGENT_SEND_FAILED, metadata) + } + trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void { this.capture(TelemetryEvents.AGENT_NODE_TAGGED, metadata) } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 2da6ef945b3..31d62141bea 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -666,6 +666,105 @@ export interface AgentMessageSentMetadata extends Record { client_message_id: string input_method: AgentInputMethod } +/** + * How far a send got before it failed. `agent_message_sent` counts attempts, so + * it fires for every send in this list too; the pair is what separates an + * attempt the server never saw from one it refused. + * + * - `not_dispatched` — the client stopped the send itself; no request was issued. + * - `no_response` — the request was issued and no HTTP status came back. It does + * **not** prove the server ignored it: a lost response and a refused request + * look the same from here, so a turn may exist. + * - `refused` — an HTTP error status came back, carried in `http_status`. + * - `unreadable_response` — a success status came back whose body the client + * could not parse, so the turn probably did start. + */ +type AgentSendFailureStage = + | 'not_dispatched' + | 'no_response' + | 'refused' + | 'unreadable_response' +/** + * `send_in_flight`: an earlier send of this session had not settled. + * `target_changed`: the workflow the send was composed against stopped being + * the target while the client resolved it. `session_reset`: a new chat or a + * thread load replaced the session mid-send. + */ +export type AgentSendNotDispatchedReason = + | 'send_in_flight' + | 'target_changed' + | 'session_reset' +/** + * `timeout` is the api layer's own response-header deadline, `aborted` any other + * cancellation, `network_error` a request the browser could not complete, and + * `unknown` anything else thrown on the send path — kept separate so a client + * bug cannot be read as a network failure. + */ +export type AgentSendNoResponseReason = + | 'timeout' + | 'aborted' + | 'network_error' + | 'unknown' +/** `admission_denied` is the pre-turn spend gate; `http_error` every other status. */ +type AgentSendRefusedReason = 'admission_denied' | 'http_error' +type AgentSendFailureReason = + | AgentSendNotDispatchedReason + | AgentSendNoResponseReason + | AgentSendRefusedReason + | 'malformed_body' +/** + * The pre-turn admission gate's own reason for declining, mirroring + * `AgentAdmissionError['error']['reason']`. Narrowed on purpose: a new value on + * that generated enum should fail to compile here rather than reach a dashboard + * unnoticed. + */ +type AgentAdmissionReason = 'no_funds' | 'manual_block' | 'funds_unavailable' +/** + * A failed send, classified by where it stopped. The fields that cannot apply to + * a stage are `null` rather than absent, so a query never has to distinguish a + * missing property from a real one. + */ +export type AgentSendFailure = + | { + stage: 'not_dispatched' + reason: AgentSendNotDispatchedReason + http_status: null + admission_reason: null + } + | { + stage: 'no_response' + reason: AgentSendNoResponseReason + http_status: null + admission_reason: null + } + | { + stage: 'refused' + reason: AgentSendRefusedReason + http_status: number + admission_reason: AgentAdmissionReason | null + } + | { + stage: 'unreadable_response' + reason: 'malformed_body' + http_status: null + admission_reason: null + } +/** + * Carries no server message and no request body: the reason enum is what a + * funnel can group by, and free text on a refusal is the one field that could + * carry user content (ADR-AGENT-OBSERVABILITY-0034 forbids raw errors in + * correlated fields). + */ +export interface AgentSendFailedMetadata extends Record { + stage: AgentSendFailureStage + reason: AgentSendFailureReason + http_status: number | null + admission_reason: AgentAdmissionReason | null + /** The thread the attempt was posted into, `null` when it opened a new one. */ + thread_id: string | null + /** The `agent_message_sent` attempt this failure belongs to. */ + client_message_id: string +} export interface AgentNodeTaggedMetadata extends Record { source: 'mention_picker' } @@ -1439,6 +1538,7 @@ export interface TelemetryProvider { trackAgentOnboardingShown?(): void trackAgentOnboardingStep?(metadata: AgentOnboardingStepMetadata): void trackAgentMessageSent?(metadata: AgentMessageSentMetadata): void + trackAgentSendFailed?(metadata: AgentSendFailedMetadata): void trackAgentNodeTagged?(metadata: AgentNodeTaggedMetadata): void trackAgentAttachButtonClicked?( metadata: AgentAttachButtonClickedMetadata @@ -1628,6 +1728,7 @@ export const TelemetryEvents = { AGENT_ONBOARDING_SHOWN: 'app:agent_onboarding_shown', AGENT_ONBOARDING_STEP: 'app:agent_onboarding_step', AGENT_MESSAGE_SENT: 'app:agent_message_sent', + AGENT_SEND_FAILED: 'app:agent_send_failed', AGENT_NODE_TAGGED: 'app:agent_node_tagged', AGENT_ATTACH_BUTTON_CLICKED: 'app:agent_attach_button_clicked', AGENT_WORKFLOW_APPLIED: 'app:agent_workflow_applied', diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index e936df7063f..4ab198faeac 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -478,6 +478,18 @@ async function sendFromComposer(text: string): Promise { await screen.findByRole('button', { name: 'Stop' }) } +/** + * A send that will not open a turn, so there is no Stop button to wait for; + * the caller waits on whatever the failure is expected to produce. + */ +async function sendFromComposerExpectingFailure(text: string): Promise { + const textbox = screen.getByRole('textbox') + await userEvent.click(textbox) + await userEvent.keyboard('{ArrowRight}') + await userEvent.paste(text) + await userEvent.click(screen.getByRole('button', { name: 'Send' })) +} + async function renderAndSend(text: string): Promise { renderWithSelectedTarget() await sendFromComposer(text) @@ -9043,4 +9055,94 @@ describe('AgentPanelRoot workflow binding', () => { toRootGraphId('wf-42-rotated') ) }) + + it.for([ + { + name: 'a credit refusal', + response: () => + json(402, { + error: { + message: 'Out of credits', + reason: 'no_funds', + type: 'PAYMENT_REQUIRED' + } + }), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + // gc-7: ingest hides the agent behind a 404 when its own flag read says + // no, so this send is refused by a surface with no other telemetry. + name: 'a server-side flag miss', + response: () => json(404, { error: 'not found' }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null + } + }, + { + name: 'a request the browser could not complete', + response: () => { + throw new TypeError('Failed to fetch') + }, + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + } + ])( + 'reports $name against the attempt the funnel counted', + async ({ response, expected }) => { + makeTab('wf-42') + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string, init?: RequestInit) => { + if (url.includes('/messages') && init?.method === 'POST') + return response() + if (url.includes('/messages')) return json(200, []) + if (url.includes('/agent/threads')) + return json(200, agentThreadList([])) + return new Response('{}', { status: 200 }) + }) + ) + telemetry.trackAgentMessageSent.mockClear() + telemetry.trackAgentSendFailed.mockClear() + renderWithSelectedTarget() + await screen.findByRole('textbox') + + await sendFromComposerExpectingFailure('build me a workflow') + + await waitFor(() => + expect(telemetry.trackAgentSendFailed).toHaveBeenCalledWith({ + ...expected, + thread_id: null, + client_message_id: 'client-message-1' + }) + ) + // The attempt event is consumed by the workflow refresh long before the + // POST resolves; both still name the same attempt. + expect( + telemetry.trackAgentMessageSent.mock.calls[0][0].client_message_id + ).toBe('client-message-1') + expect(telemetry.trackAgentSendFailed).toHaveBeenCalledOnce() + } + ) + + it('reports no send failure when the turn is accepted', async () => { + makeTab('wf-42') + mockMessagesEndpoint('wf-42') + telemetry.trackAgentSendFailed.mockClear() + + await renderAndSend('build me a workflow') + + expect(telemetry.trackAgentSendFailed).not.toHaveBeenCalled() + }) }) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 4092543e9d3..5417c7ab53d 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -21,6 +21,7 @@ import { useTelemetry } from '@/platform/telemetry' import type { AgentMessageSentMetadata, AgentRunApprovalDecision, + AgentSendFailure, AgentStopMethod } from '@/platform/telemetry/types' import { useSettingStore } from '@/platform/settings/settingStore' @@ -696,6 +697,18 @@ function reportPendingSend(): void { }) } +/** + * Reports the failure of the send currently in flight against the attempt + * `app:agent_message_sent` counted. Held as a closure rather than as the + * attempt's identity, because the workflow refresh consumes `pendingSend` + * before the POST is even issued: by the time a send fails there is no pending + * report left to read the id from, and an attempt-less failure event would be + * unjoinable. Scoped to one `sendMessage` call like `pendingSend`, and single + * flight for the same reason: the composer refuses a submit while one is + * pending. + */ +let reportSendFailure: ((failure: AgentSendFailure) => void) | null = null + const { sendMessage, stopTurn, @@ -721,6 +734,7 @@ const { events, onThreadStarted: (source) => useTelemetry()?.trackAgentThreadStarted({ source }), + onSendFailed: (failure) => reportSendFailure?.(failure), onAskResolved: forgetApproval, workflow: { initialize: agentPanelStore.initializeTargetTracking, @@ -1260,6 +1274,13 @@ const { submit: onSend } = useAgentDraftSubmission({ originContext === undefined ? null : { tabPath: originContext.tabPath } } const selectionWorkflow = selectedTarget.value + const attemptThreadId = threadId.value + reportSendFailure = (failure) => + useTelemetry()?.trackAgentSendFailed({ + ...failure, + thread_id: attemptThreadId, + client_message_id: meta.clientMessageId + }) try { return await sendMessage(text, attachments, nodes, references, () => selectionWorkflow ? cloudIdFor(selectionWorkflow) : undefined @@ -1268,6 +1289,7 @@ const { submit: onSend } = useAgentDraftSubmission({ // Normally already consumed by the refresh. A send rejected before it // gets that far still reports here, so the funnel counts the attempt. reportPendingSend() + reportSendFailure = null } } }) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index a73a1ba31e1..c9afcf27b5b 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -2966,3 +2966,212 @@ describe('thread resume (B17)', () => { expect(threads[0]).toMatchObject({ id: 'th-9', title: 'build a duck' }) }) }) + +describe('failed send reporting', () => { + it.for([ + { + name: 'a credit refusal, with the gate reason attached', + error: () => admissionError('no_funds', 'Out of credits'), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + name: 'a server error, which no gate reason explains', + error: () => new AgentApiError('server exploded', 500, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + }, + { + name: 'a request that never reached the service', + error: () => new TypeError('Failed to fetch'), + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + } + ])('reports $name', async ({ error, expected }) => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi + .fn() + .mockRejectedValue(error()) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed.mock.calls).toEqual([[expected]]) + }) + + it('reports nothing when the turn is accepted', async () => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest(), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(true) + + expect(onSendFailed).not.toHaveBeenCalled() + }) + + it('reports a send refused while an earlier one is still in flight', async () => { + const onSendFailed = vi.fn() + const session = useAgentSession({ + rest: fakeRest(), + events: fakeEvents().source, + onSendFailed + }) + session.start() + + const first = session.sendMessage('first') + expect(await session.sendMessage('second')).toBe(false) + expect(await first).toBe(true) + + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'send_in_flight', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a send abandoned because its target stopped being reachable', async () => { + const onSendFailed = vi.fn() + let reachable = true + const postMessage = vi.fn() + const session = useAgentSession({ + rest: fakeRest({ postMessage }), + events: fakeEvents().source, + onSendFailed, + workflow: { + current: (origin) => + origin !== undefined && !reachable + ? undefined + : { id: 'wf-a', tabPath: 'tab-a' }, + adopted: vi.fn(), + prepare: async () => { + reachable = false + } + } + }) + session.start() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(postMessage).not.toHaveBeenCalled() + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'target_changed', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a send the user replaced with a new chat before it was posted', async () => { + const onSendFailed = vi.fn() + const postMessage = vi.fn() + let reset: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ postMessage }), + events: fakeEvents().source, + onSendFailed, + workflow: { + current: () => undefined, + adopted: vi.fn(), + prepare: async () => reset() + } + }) + session.start() + reset = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(postMessage).not.toHaveBeenCalled() + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'not_dispatched', + reason: 'session_reset', + http_status: null, + admission_reason: null + } + ] + ]) + }) + + it('reports a refusal that lands after the session moved on, because the attempt was still counted', async () => { + const onSendFailed = vi.fn() + let abandon: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi.fn(async () => { + abandon() + throw new AgentApiError('server exploded', 500, undefined) + }) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + abandon = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed.mock.calls).toEqual([ + [ + { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + ] + ]) + }) + + it('reports nothing for an acknowledgement dropped after the session moved on, because the turn exists', async () => { + const onSendFailed = vi.fn() + let abandon: () => void = () => {} + const session = useAgentSession({ + rest: fakeRest({ + postMessage: vi.fn(async () => { + abandon() + return { thread_id: 'th-1', message_id: 'msg-1' } + }) + }), + events: fakeEvents().source, + onSendFailed + }) + session.start() + abandon = () => session.newChat() + + expect(await session.sendMessage('make a cat')).toBe(false) + + expect(onSendFailed).not.toHaveBeenCalled() + }) +}) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index 98bc60bcfa4..e951836c1c0 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -4,6 +4,7 @@ import { i18n } from '@/i18n' import { useTelemetry } from '@/platform/telemetry' import { reportError } from '@/platform/telemetry/reportError' import type { + AgentSendFailure, AgentStopClickedMetadata, AgentStopMethod, AgentThreadStartSource, @@ -25,7 +26,6 @@ import { isAgentEvent, parseAgentWsEvent, toTurnId, - zAgentAdmissionError, zDisownedWorkflowError } from '../../schemas/agentApiSchema' import { AgentApiError } from '../../services/agent/agentRestClient' @@ -38,6 +38,11 @@ import type { import { useAgentConversationStore } from '../../stores/agent/agentConversationStore' import { useAgentWorkflowTabBindingStore } from '../../stores/agent/agentWorkflowTabBindingStore' import type { WorkflowReference } from '../../types/workflowReference' +import { + classifyAgentSendError, + parseAdmissionError, + sendNotDispatched +} from '../../utils/agentSendFailure' import { serializeWorkflowReferences } from '../../utils/workflowReferenceText' export interface AgentEventSource { @@ -93,6 +98,13 @@ export interface AgentSessionDeps { events: AgentEventSource onThreadStarted?: (source: AgentSessionThreadStartSource) => void onAskResolved?: (askId: string) => void + /** + * Once per send attempt that produced no turn. A turn the server accepted + * after the session moved on is not reported here: the ack was dropped by the + * client, but the turn exists, and counting it as a failed send would + * overstate the loss the event is measuring. + */ + onSendFailed?: (failure: AgentSendFailure) => void workflow?: { /** Resolve fresh versus restored startup before asynchronous hydration. */ initialize?(hasThread: boolean): void @@ -147,16 +159,6 @@ function consumeStopPendingAck() { return pending } -function parseAdmissionError(error: unknown) { - if (!(error instanceof AgentApiError)) return undefined - const parsed = zAgentAdmissionError.safeParse(error.body) - if (!parsed.success) return undefined - const expectedStatus = - parsed.data.error.type === 'PAYMENT_REQUIRED' ? 402 : 503 - if (error.status !== expectedStatus) return undefined - return { ...parsed.data.error, retryAfterSeconds: error.retryAfterSeconds } -} - function disownsWorkflow(error: unknown): boolean { return ( error instanceof AgentApiError && @@ -166,7 +168,14 @@ function disownsWorkflow(error: unknown): boolean { } export function useAgentSession(deps: AgentSessionDeps) { - const { rest, events, onThreadStarted, onAskResolved, workflow } = deps + const { + rest, + events, + onThreadStarted, + onAskResolved, + onSendFailed, + workflow + } = deps const threadStorageKey = StorageKeys.agentThread(getWorkspaceId()) clearLegacyAgentStorage() @@ -533,6 +542,10 @@ export function useAgentSession(deps: AgentSessionDeps) { if (pendingStop !== null) void stopTurn(pendingStop.method) } + function reportSendFailure(failure: AgentSendFailure): void { + onSendFailed?.(failure) + } + function recordSendError(error: unknown, text: string): void { const admission = parseAdmissionError(error) if (admission?.reason === 'no_funds') { @@ -604,10 +617,14 @@ export function useAgentSession(deps: AgentSessionDeps) { let sentContext: WorkflowTurnContext | undefined try { await prepareWorkflow() - if (generation !== loadGeneration) return false + if (generation !== loadGeneration) { + reportSendFailure(sendNotDispatched('session_reset')) + return false + } const wfContext = workflow?.current(origin) if (workflowTargetChanged(originContext, wfContext)) { recordUnavailableTarget(text) + reportSendFailure(sendNotDispatched('target_changed')) return false } sentContext = wfContext @@ -621,6 +638,7 @@ export function useAgentSession(deps: AgentSessionDeps) { workflowReferences, selectionWorkflowId ) + // A dropped ack is not a failed send - see onSendFailed's contract. if (generation !== loadGeneration) return false acceptTurn(ack, text, wfContext, attachments, tags, workflowReferences) return true @@ -629,6 +647,9 @@ export function useAgentSession(deps: AgentSessionDeps) { // persisted, so a refusal that lands after newChat()/loadThread() has // moved on still has to release, or the dead id survives the reload. releaseDisownedWorkflow(sentContext, error) + // Also before it: the attempt was counted when it was made, so the + // refusal has to be counted even when nothing is left to render it to. + reportSendFailure(classifyAgentSendError(error)) if (generation !== loadGeneration) return false recordSendError(error, text) return false @@ -656,6 +677,7 @@ export function useAgentSession(deps: AgentSessionDeps) { text, i18n.global.t('agent.sendBusy') ) + reportSendFailure(sendNotDispatched('send_in_flight')) return false } promptEditState.value = { phase: 'idle' } diff --git a/src/workbench/extensions/agent/utils/agentSendFailure.test.ts b/src/workbench/extensions/agent/utils/agentSendFailure.test.ts new file mode 100644 index 00000000000..811d32cf914 --- /dev/null +++ b/src/workbench/extensions/agent/utils/agentSendFailure.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +import type { AgentSendFailure } from '@/platform/telemetry/types' + +import { zAgentAdmissionError } from '../schemas/agentApiSchema' +import { AgentApiError } from '../services/agent/agentRestClient' + +import { classifyAgentSendError, sendNotDispatched } from './agentSendFailure' + +function admission( + reason: 'no_funds' | 'manual_block' | 'funds_unavailable', + status: number +): AgentApiError { + const body = zAgentAdmissionError.parse({ + error: { + message: 'declined', + reason, + type: + reason === 'funds_unavailable' + ? 'SERVICE_UNAVAILABLE' + : 'PAYMENT_REQUIRED' + } + }) + return new AgentApiError('declined', status, body) +} + +/** What the turn schema actually throws when the ack body is the wrong shape. */ +function schemaError(): unknown { + const result = z.object({ thread_id: z.string() }).safeParse({}) + return result.success ? undefined : result.error +} + +describe('classifyAgentSendError', () => { + it.for([ + { + name: 'a credit refusal', + error: () => admission('no_funds', 402), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'no_funds' + } + }, + { + name: 'a billing outage', + error: () => admission('funds_unavailable', 503), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 503, + admission_reason: 'funds_unavailable' + } + }, + { + name: 'a blocked workspace', + error: () => admission('manual_block', 402), + expected: { + stage: 'refused', + reason: 'admission_denied', + http_status: 402, + admission_reason: 'manual_block' + } + }, + { + // The server-side flag miss gc-7 named: ingest hides the feature behind a + // 404, so the status is the only evidence the send was refused at all. + name: 'a server-side flag miss', + error: () => new AgentApiError('not found', 404, { error: 'not found' }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 404, + admission_reason: null + } + }, + { + name: 'a single-active-turn conflict', + error: () => new AgentApiError('conflict', 409, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 409, + admission_reason: null + } + }, + { + name: 'an agent service outage', + error: () => new AgentApiError('unavailable', 502, undefined), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 502, + admission_reason: null + } + }, + { + // A 402 whose body does not parse is still a refusal; it just cannot be + // attributed to the admission gate's reason. + name: 'a 402 with no admission body', + error: () => + new AgentApiError('payment required', 402, { + error: { message: 'payment required', type: 'PAYMENT_REQUIRED' } + }), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 402, + admission_reason: null + } + }, + { + name: 'an admission body on the wrong status', + error: () => admission('no_funds', 500), + expected: { + stage: 'refused', + reason: 'http_error', + http_status: 500, + admission_reason: null + } + }, + { + name: 'a request the browser could not complete', + error: () => new TypeError('Failed to fetch'), + expected: { + stage: 'no_response', + reason: 'network_error', + http_status: null, + admission_reason: null + } + }, + { + name: "the api layer's own response deadline", + error: () => new DOMException('Fetch timeout', 'TimeoutError'), + expected: { + stage: 'no_response', + reason: 'timeout', + http_status: null, + admission_reason: null + } + }, + { + name: 'a cancelled request', + error: () => new DOMException('Aborted', 'AbortError'), + expected: { + stage: 'no_response', + reason: 'aborted', + http_status: null, + admission_reason: null + } + }, + { + // Kept out of network_error on purpose: a bug on the send path must not + // read as connectivity. + name: 'an unexpected throw on the send path', + error: () => new RangeError('bad index'), + expected: { + stage: 'no_response', + reason: 'unknown', + http_status: null, + admission_reason: null + } + }, + { + name: 'a non-error rejection', + error: () => 'nope', + expected: { + stage: 'no_response', + reason: 'unknown', + http_status: null, + admission_reason: null + } + }, + { + name: 'an acknowledgement that does not match the schema', + error: schemaError, + expected: { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + }, + { + name: 'an acknowledgement that is not json', + error: () => new SyntaxError('Unexpected token < in JSON'), + expected: { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + } + ])('classifies $name', ({ error, expected }) => { + expect(classifyAgentSendError(error())).toEqual(expected) + }) +}) + +describe('sendNotDispatched', () => { + it.for(['send_in_flight', 'target_changed', 'session_reset'] as const)( + 'reports %s with no status, because no request was issued', + (reason) => { + const expected: AgentSendFailure = { + stage: 'not_dispatched', + reason, + http_status: null, + admission_reason: null + } + expect(sendNotDispatched(reason)).toEqual(expected) + } + ) +}) diff --git a/src/workbench/extensions/agent/utils/agentSendFailure.ts b/src/workbench/extensions/agent/utils/agentSendFailure.ts new file mode 100644 index 00000000000..1fd0e2fdc57 --- /dev/null +++ b/src/workbench/extensions/agent/utils/agentSendFailure.ts @@ -0,0 +1,86 @@ +import type { + AgentSendFailure, + AgentSendNoResponseReason, + AgentSendNotDispatchedReason +} from '@/platform/telemetry/types' + +import { zAgentAdmissionError } from '../schemas/agentApiSchema' +import { AgentApiError } from '../services/agent/agentRestClient' + +export function parseAdmissionError(error: unknown) { + if (!(error instanceof AgentApiError)) return undefined + const parsed = zAgentAdmissionError.safeParse(error.body) + if (!parsed.success) return undefined + const expectedStatus = + parsed.data.error.type === 'PAYMENT_REQUIRED' ? 402 : 503 + if (error.status !== expectedStatus) return undefined + return { ...parsed.data.error, retryAfterSeconds: error.retryAfterSeconds } +} + +export function sendNotDispatched( + reason: AgentSendNotDispatchedReason +): AgentSendFailure { + return { + stage: 'not_dispatched', + reason, + http_status: null, + admission_reason: null + } +} + +/** + * Errors are matched on `name`, never on class identity: the response schema + * comes from `@comfyorg/ingest-types`, which declares its own zod dependency, so + * a second copy of zod in the tree would make `instanceof ZodError` silently + * false and reclassify a parse failure as a network one. + */ +function errorName(error: unknown): string | undefined { + return error instanceof Error ? error.name : undefined +} + +/** + * Zod and `Response.json()` both reject only after the status line was read and + * found ok, so these are the two throws that mean "the turn may well have + * started and the client cannot tell". + */ +function isUnreadableResponse(error: unknown): boolean { + const name = errorName(error) + return name === 'ZodError' || name === 'SyntaxError' +} + +function noResponseReason(error: unknown): AgentSendNoResponseReason { + const name = errorName(error) + if (name === 'TimeoutError') return 'timeout' + if (name === 'AbortError') return 'aborted' + return name === 'TypeError' ? 'network_error' : 'unknown' +} + +/** + * The turn POST rejected. Only an `AgentApiError` proves a status line came + * back, so every other throw is classified from what it says about itself + * rather than assumed to be the network. + */ +export function classifyAgentSendError(error: unknown): AgentSendFailure { + if (error instanceof AgentApiError) { + const admission = parseAdmissionError(error) + return { + stage: 'refused', + reason: admission === undefined ? 'http_error' : 'admission_denied', + http_status: error.status, + admission_reason: admission?.reason ?? null + } + } + if (isUnreadableResponse(error)) + return { + stage: 'unreadable_response', + reason: 'malformed_body', + http_status: null, + admission_reason: null + } + return { + stage: 'no_response', + reason: noResponseReason(error), + http_status: null, + admission_reason: null + } +}