Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import type {
AgentPanelOpenedMetadata,
AgentPaywallCtaMetadata,
AgentPaywallShownMetadata,
AgentSendFailedMetadata,
AgentWorkflowAppliedMetadata,
BillingTelemetryEvent,
CheckoutJourneyTelemetryEvent,
Expand Down Expand Up @@ -300,6 +301,14 @@ describe('TelemetryRegistry', () => {
client_message_id: 'cm-1',
input_method: 'typed'
} satisfies AgentMessageSentMetadata
const sendFailedMetadata = {
stage: 'refused',
reason: 'admission_denied',
http_status: 402,
admission_reason: 'no_funds',
thread_id: 'th-1',
client_message_id: 'cm-1'
} satisfies AgentSendFailedMetadata
const consentShownMetadata = {
trigger: 'button_click'
} satisfies AgentConsentShownMetadata
Expand Down Expand Up @@ -364,6 +373,11 @@ describe('TelemetryRegistry', () => {
invoke: (registry) =>
registry.trackAgentMessageSent(messageSentMetadata)
},
{
method: 'trackAgentSendFailed',
expected: { ...sendFailedMetadata },
invoke: (registry) => registry.trackAgentSendFailed(sendFailedMetadata)
},
{
method: 'trackAgentConsentShown',
expected: { ...consentShownMetadata },
Expand Down
5 changes: 5 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import type {
AgentRunApprovalResolvedMetadata,
AgentRunApprovalShownMetadata,
AgentRunModeChangedMetadata,
AgentSendFailedMetadata,
AgentStopClickedMetadata,
AgentThreadStartedMetadata,
AgentWorkflowBoundMetadata,
Expand Down Expand Up @@ -419,6 +420,10 @@ export class TelemetryRegistry implements TelemetryDispatcher {
this.dispatch((provider) => provider.trackAgentMessageSent?.(metadata))
}

trackAgentSendFailed(metadata: AgentSendFailedMetadata): void {
this.dispatch((provider) => provider.trackAgentSendFailed?.(metadata))
}

trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void {
this.dispatch((provider) => provider.trackAgentNodeTagged?.(metadata))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,29 @@ describe('DatadogRumTelemetryProvider', () => {
)
})

it('records failed agent sends as RUM actions so they can be monitored', () => {
new DatadogRumTelemetryProvider().trackAgentSendFailed({
stage: 'refused',
reason: 'http_error',
http_status: 404,
admission_reason: null,
thread_id: 'thread-1',
client_message_id: 'client-message-1'
})

expect(addAction).toHaveBeenCalledExactlyOnceWith(
TelemetryEvents.AGENT_SEND_FAILED,
{
stage: 'refused',
reason: 'http_error',
http_status: 404,
admission_reason: null,
thread_id: 'thread-1',
client_message_id: 'client-message-1'
}
)
})

it('records terminal unified auth retry outcomes without request data', () => {
new DatadogRumTelemetryProvider().trackUnifiedAuthRetry({
transport: 'axios',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { datadogRum } from '@datadog/browser-rum'
import { useCurrentUser } from '@/composables/auth/useCurrentUser'

import type {
AgentSendFailedMetadata,
AuthMetadata,
BillingTelemetryEvent,
CheckoutJourneyTelemetryEvent,
Expand Down Expand Up @@ -66,6 +67,14 @@ export class DatadogRumTelemetryProvider implements TelemetryProvider {
)
}

/**
* A funnel drop-off worth a monitor, so ADR-TELEMETRY-ROUTING-0013 puts it
* here as well as in PostHog rather than only in the growth layer.
*/
trackAgentSendFailed(metadata: AgentSendFailedMetadata): void {
datadogRum.addAction(TelemetryEvents.AGENT_SEND_FAILED, metadata)
}

trackImageLoadFailed(metadata: ImageLoadFailureMetadata): void {
datadogRum.addAction(TelemetryEvents.IMAGE_LOAD_FAILED, metadata)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,32 @@ describe('PostHogTelemetryProvider', () => {
)
})

it('captures a failed send against the attempt it belongs to', async () => {
const provider = createProvider()
await vi.dynamicImportSettled()

provider.trackAgentSendFailed({
stage: 'refused',
reason: 'admission_denied',
http_status: 402,
admission_reason: 'no_funds',
thread_id: null,
client_message_id: 'client-message-1'
})

expect(hoisted.mockCapture).toHaveBeenCalledWith(
TelemetryEvents.AGENT_SEND_FAILED,
{
stage: 'refused',
reason: 'admission_denied',
http_status: 402,
admission_reason: 'no_funds',
thread_id: null,
client_message_id: 'client-message-1'
}
)
})

it('captures link dedup drop events with metadata', async () => {
const provider = createProvider()
await vi.dynamicImportSettled()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import type {
AgentRunApprovalResolvedMetadata,
AgentRunApprovalShownMetadata,
AgentRunModeChangedMetadata,
AgentSendFailedMetadata,
AgentStopClickedMetadata,
AgentThreadStartedMetadata,
AgentWorkflowBoundMetadata,
Expand Down Expand Up @@ -786,6 +787,10 @@ export class PostHogTelemetryProvider implements TelemetryProvider {
this.trackEvent(TelemetryEvents.AGENT_MESSAGE_SENT, metadata)
}

trackAgentSendFailed(metadata: AgentSendFailedMetadata): void {
this.trackEvent(TelemetryEvents.AGENT_SEND_FAILED, metadata)
}

trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void {
this.trackEvent(TelemetryEvents.AGENT_NODE_TAGGED, metadata)
}
Expand Down
20 changes: 20 additions & 0 deletions src/platform/telemetry/providers/host/HostTelemetrySink.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,26 @@ describe('HostTelemetrySink', () => {
input_method: 'typed'
}
},
{
name: TelemetryEvents.AGENT_SEND_FAILED,
track: (sink: HostTelemetrySink) =>
sink.trackAgentSendFailed({
stage: 'no_response',
reason: 'timeout',
http_status: null,
admission_reason: null,
thread_id: 'thread-1',
client_message_id: 'client-message-3'
}),
properties: {
stage: 'no_response',
reason: 'timeout',
http_status: null,
admission_reason: null,
thread_id: 'thread-1',
client_message_id: 'client-message-3'
}
},
{
name: TelemetryEvents.AGENT_CONSENT_SHOWN,
track: (sink: HostTelemetrySink) =>
Expand Down
5 changes: 5 additions & 0 deletions src/platform/telemetry/providers/host/HostTelemetrySink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type {
AgentRunApprovalResolvedMetadata,
AgentRunApprovalShownMetadata,
AgentRunModeChangedMetadata,
AgentSendFailedMetadata,
AgentStopClickedMetadata,
AgentThreadStartedMetadata,
AgentWorkflowBoundMetadata,
Expand Down Expand Up @@ -364,6 +365,10 @@ export class HostTelemetrySink implements TelemetryProvider {
this.capture(TelemetryEvents.AGENT_MESSAGE_SENT, metadata)
}

trackAgentSendFailed(metadata: AgentSendFailedMetadata): void {
this.capture(TelemetryEvents.AGENT_SEND_FAILED, metadata)
}

trackAgentNodeTagged(metadata: AgentNodeTaggedMetadata): void {
this.capture(TelemetryEvents.AGENT_NODE_TAGGED, metadata)
}
Expand Down
101 changes: 101 additions & 0 deletions src/platform/telemetry/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,105 @@ export interface AgentMessageSentMetadata extends Record<string, unknown> {
client_message_id: string
input_method: AgentInputMethod
}
/**
* How far a send got before it failed. `agent_message_sent` counts attempts, so
* it fires for every send in this list too; the pair is what separates an
* attempt the server never saw from one it refused.
*
* - `not_dispatched` — the client stopped the send itself; no request was issued.
* - `no_response` — the request was issued and no HTTP status came back. It does
* **not** prove the server ignored it: a lost response and a refused request
* look the same from here, so a turn may exist.
* - `refused` — an HTTP error status came back, carried in `http_status`.
* - `unreadable_response` — a success status came back whose body the client
* could not parse, so the turn probably did start.
*/
type AgentSendFailureStage =
| 'not_dispatched'
| 'no_response'
| 'refused'
| 'unreadable_response'
/**
* `send_in_flight`: an earlier send of this session had not settled.
* `target_changed`: the workflow the send was composed against stopped being
* the target while the client resolved it. `session_reset`: a new chat or a
* thread load replaced the session mid-send.
*/
export type AgentSendNotDispatchedReason =
| 'send_in_flight'
| 'target_changed'
| 'session_reset'
/**
* `timeout` is the api layer's own response-header deadline, `aborted` any other
* cancellation, `network_error` a request the browser could not complete, and
* `unknown` anything else thrown on the send path — kept separate so a client
* bug cannot be read as a network failure.
*/
export type AgentSendNoResponseReason =
| 'timeout'
| 'aborted'
| 'network_error'
| 'unknown'
/** `admission_denied` is the pre-turn spend gate; `http_error` every other status. */
type AgentSendRefusedReason = 'admission_denied' | 'http_error'
type AgentSendFailureReason =
| AgentSendNotDispatchedReason
| AgentSendNoResponseReason
| AgentSendRefusedReason
| 'malformed_body'
/**
* The pre-turn admission gate's own reason for declining, mirroring
* `AgentAdmissionError['error']['reason']`. Narrowed on purpose: a new value on
* that generated enum should fail to compile here rather than reach a dashboard
* unnoticed.
*/
type AgentAdmissionReason = 'no_funds' | 'manual_block' | 'funds_unavailable'
/**
* A failed send, classified by where it stopped. The fields that cannot apply to
* a stage are `null` rather than absent, so a query never has to distinguish a
* missing property from a real one.
*/
export type AgentSendFailure =
| {
stage: 'not_dispatched'
reason: AgentSendNotDispatchedReason
http_status: null
admission_reason: null
}
| {
stage: 'no_response'
reason: AgentSendNoResponseReason
http_status: null
admission_reason: null
}
| {
stage: 'refused'
reason: AgentSendRefusedReason
http_status: number
admission_reason: AgentAdmissionReason | null
}
| {
stage: 'unreadable_response'
reason: 'malformed_body'
http_status: null
admission_reason: null
}
/**
* Carries no server message and no request body: the reason enum is what a
* funnel can group by, and free text on a refusal is the one field that could
* carry user content (ADR-AGENT-OBSERVABILITY-0034 forbids raw errors in
* correlated fields).
*/
export interface AgentSendFailedMetadata extends Record<string, unknown> {
stage: AgentSendFailureStage
reason: AgentSendFailureReason
http_status: number | null
admission_reason: AgentAdmissionReason | null
/** The thread the attempt was posted into, `null` when it opened a new one. */
thread_id: string | null
/** The `agent_message_sent` attempt this failure belongs to. */
client_message_id: string
}
export interface AgentNodeTaggedMetadata extends Record<string, unknown> {
source: 'mention_picker'
}
Expand Down Expand Up @@ -1439,6 +1538,7 @@ export interface TelemetryProvider {
trackAgentOnboardingShown?(): void
trackAgentOnboardingStep?(metadata: AgentOnboardingStepMetadata): void
trackAgentMessageSent?(metadata: AgentMessageSentMetadata): void
trackAgentSendFailed?(metadata: AgentSendFailedMetadata): void
trackAgentNodeTagged?(metadata: AgentNodeTaggedMetadata): void
trackAgentAttachButtonClicked?(
metadata: AgentAttachButtonClickedMetadata
Expand Down Expand Up @@ -1628,6 +1728,7 @@ export const TelemetryEvents = {
AGENT_ONBOARDING_SHOWN: 'app:agent_onboarding_shown',
AGENT_ONBOARDING_STEP: 'app:agent_onboarding_step',
AGENT_MESSAGE_SENT: 'app:agent_message_sent',
AGENT_SEND_FAILED: 'app:agent_send_failed',
AGENT_NODE_TAGGED: 'app:agent_node_tagged',
AGENT_ATTACH_BUTTON_CLICKED: 'app:agent_attach_button_clicked',
AGENT_WORKFLOW_APPLIED: 'app:agent_workflow_applied',
Expand Down
Loading
Loading