Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions browser_tests/fixtures/agentTurnLockFixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,12 @@ async function routeTurnLock(
*/
await page.route('**/api/agent/threads/*/asks/*/answer', (route) => {
const url = new URL(route.request().url())
const segments = url.pathname.split('/').map(decodeURIComponent)
let segments: string[]
try {
segments = url.pathname.split('/').map(decodeURIComponent)
} catch {
return route.fulfill({ status: 404, body: 'malformed thread or ask' })
}
const threadId = segments[segments.indexOf('threads') + 1]
const askId = segments[segments.indexOf('asks') + 1]
if (threadId !== THREAD_ID || askId !== RUN_APPROVAL_ASK_ID)
Expand Down Expand Up @@ -376,6 +381,7 @@ export class AgentTurnLockHarness {
}

push(ws: WebSocketRoute, event: AgentWsEvent): void {
this.server.recordAsk(event)
ws.send(JSON.stringify(event))
}

Expand All @@ -384,7 +390,7 @@ export class AgentTurnLockHarness {
this.server.recordAsk(event)
}

/** True only when a test explicitly primed transcript-based recovery. */
/** True when the fake server has parked the turn on an ask. */
pendingAskIsPrimed(): boolean {
return this.server.askIsPending
}
Expand Down
24 changes: 14 additions & 10 deletions browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ test.describe(
// answer. `ws.send()` throws on a dead route, so pushing stays above
// test.fail() too.
turnLock.push(reconnected, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)
expect(turnLock.pendingAskIsPrimed()).toBe(true)

test.fail()
await expect(
Expand All @@ -74,17 +74,21 @@ test.describe(
const reconnected = await turnLock.dropSocket()
await expect(turnLock.userBubbles).toHaveText([PROMPT])
turnLock.push(reconnected, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)
expect(turnLock.pendingAskIsPrimed()).toBe(true)

test.fail()
// Recovery must restore both the card and its active turn identity;
// `answerAsk` deliberately refuses to POST without `activeTurnId`.
await turnLock.panel
.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
.click({ timeout: 10_000 })
const runButton = turnLock.panel.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
try {
await expect(runButton).toBeVisible({ timeout: 10_000 })
} catch (error) {
test.fail()
throw error
}
await runButton.click()
await expect.poll(() => turnLock.answeredAsks()).toEqual(['run'])
})

Expand All @@ -98,7 +102,7 @@ test.describe(
const live = await turnLock.liveSocket()

turnLock.push(live, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)
expect(turnLock.pendingAskIsPrimed()).toBe(true)

await expect(
turnLock.panel.getByText(enMessages.agent.runApproval.question)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -495,18 +495,10 @@ describe('agentEventTransport run approval', () => {
expect(message.streaming).toBe(true)
})

it('reports ask_user as a generated kind without a client renderer', () => {
it('does not report the supported ask_user kind as an error', () => {
drive([runApproval('ask-user-1', 'ask_user')])

expect(reportError).toHaveBeenCalledWith(
expect.any(Error),
expect.objectContaining({
tags: expect.objectContaining({
reason: 'unrendered-kind',
ask_kind: 'ask_user'
})
})
)
expect(reportError).not.toHaveBeenCalled()
})

it('reports an unknown ask kind once with a bounded tag', () => {
Expand All @@ -522,19 +514,20 @@ describe('agentEventTransport run approval', () => {
expect.objectContaining({
tags: {
reason: 'unknown-kind',
ask_kind: 'x'.repeat(64)
}
ask_kind: 'unknown'
},
context: expect.objectContaining({ rawAskKind: 'x'.repeat(100) })
})
)
})

it('bounds reported ask identities and evicts the oldest', () => {
it('caps report volume for the lifetime of the reporter', () => {
const asks = Array.from({ length: 33 }, (_, index) =>
runApproval(`unknown-${index}`, 'unsupported')
)
drive([...asks, asks[0]])

expect(reportError).toHaveBeenCalledTimes(34)
expect(reportError).toHaveBeenCalledTimes(32)
})
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ export function createAgentEventTransport(
data: AgentAskEvent['data'],
reason: UndeliverableAskReason,
context?: UndeliverableAskContext
) => void = createUndeliverableAskReporter().report
) => void = createUndeliverableAskReporter().report,
markAskDelivered: (data: AgentAskEvent['data']) => void = () => {}
): AgentEventTransport {
let openText: TextPart | null = null
let openThinking: ThinkingPart | null = null
Expand Down Expand Up @@ -312,15 +313,13 @@ export function createAgentEventTransport(
*
* That `false` reaches a live turn and still shows the user nothing, while
* the server parks waiting for an answer — the same dead-panel outcome as an
* ask dropped in routing, so it is reported the same way. Generated-contract
* kinds without a client renderer are tagged separately from unknown input.
* ask dropped in routing, so unknown input is reported. The supported
* `ask_user` kind is not an approval-card delivery failure.
*/
function handleAskEvent(data: AgentAskEvent['data']): boolean {
if (data.kind === 'ask_user') return false
if (data.kind !== 'run_approval') {
reportUndeliverableAsk(
data,
data.kind === 'ask_user' ? 'unrendered-kind' : 'unknown-kind'
)
reportUndeliverableAsk(data, 'unknown-kind')
return false
}
closeOpenText()
Expand All @@ -334,6 +333,7 @@ export function createAgentEventTransport(
workflowName: data.context?.workflow_name || undefined
}
message.parts.push(part)
markAskDelivered(data)
return true
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ type AgentAskData = Extract<AgentWsEvent, { type: 'agent_ask' }>['data']
export type UndeliverableAskReason =
| 'no-live-turn'
| 'settled-turn'
| 'unknown-turn'
| 'unknown-kind'
| 'unrendered-kind'

export interface UndeliverableAskContext {
hasActiveTurn?: boolean
Expand All @@ -16,24 +16,60 @@ export interface UndeliverableAskContext {
activeTurnId?: string | null
}

const MAX_REPORTED_ASKS = 32
const MAX_REPORTS_PER_SESSION = 32
const MAX_IDENTIFIER_LENGTH = 128

function bounded(value: string | null | undefined): string | null | undefined {
return value?.slice(0, MAX_IDENTIFIER_LENGTH)
}

function identityHash(value: string): string {
let hash = 0x811c9dc5
for (let index = 0; index < value.length; index++) {
hash ^= value.charCodeAt(index)
hash = Math.imul(hash, 0x01000193)
}
return (hash >>> 0).toString(16).padStart(8, '0')
}

function askIdentity(data: AgentAskData): string {
return identityHash(`${data.thread_id}\u0000${data.ask_id}`)
}

/** One bounded, identity-aware telemetry path shared by store and transport. */
export function createUndeliverableAskReporter() {
const reportedAskIds = new Set<string>()
const reported = new Set<string>()
const delivered = new Set<string>()
let reportCount = 0

function retain(set: Set<string>, key: string): void {
if (set.has(key)) set.delete(key)
set.add(key)
if (set.size <= MAX_REPORTS_PER_SESSION) return
const oldest = set.values().next().value
if (oldest !== undefined) set.delete(oldest)
}

return {
report(
data: AgentAskData,
reason: UndeliverableAskReason,
context: UndeliverableAskContext = {}
): void {
if (reportedAskIds.has(data.ask_id)) return
reportedAskIds.add(data.ask_id)
if (reportedAskIds.size > MAX_REPORTED_ASKS) {
const oldest = reportedAskIds.values().next().value
if (oldest !== undefined) reportedAskIds.delete(oldest)
}
const identity = askIdentity(data)
if (delivered.has(identity) || reportCount >= MAX_REPORTS_PER_SESSION)
return
const reportKey = `${identity}\u0000${reason}`
if (reported.has(reportKey)) return
retain(reported, reportKey)
reportCount++

const knownKind =
data.kind === 'run_approval' || data.kind === 'ask_user'
? data.kind
: data.kind
? 'unknown'
: 'missing'

reportError(
new Error(`agent approval ask could not be delivered (${reason})`),
Expand All @@ -42,21 +78,27 @@ export function createUndeliverableAskReporter() {
level: 'warning',
tags: {
reason,
ask_kind: data.kind?.slice(0, 64) || 'missing',
ask_kind: knownKind,
has_active_turn: context.hasActiveTurn,
background_turn_count: context.backgroundTurnCount
},
context: {
threadId: data.thread_id,
messageId: data.message_id,
activeThreadId: context.activeThreadId,
activeTurnId: context.activeTurnId
threadId: bounded(data.thread_id),
messageId: bounded(data.message_id),
rawAskKind: bounded(data.kind),
activeThreadId: bounded(context.activeThreadId),
activeTurnId: bounded(context.activeTurnId)
}
}
)
},
markDelivered(data: AgentAskData): void {
retain(delivered, askIdentity(data))
},
reset(): void {
reportedAskIds.clear()
reported.clear()
delivered.clear()
reportCount = 0
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,20 @@ describe('useAgentConversationStore', () => {
expect(reportError).not.toHaveBeenCalled()
})

it('reports an ask for a never-registered turn on the active thread', () => {
const store = useAgentConversationStore()
store.setThreadId('th')

store.ingest(runApproval('never-registered', 'turn-1:call-1'))

expect(reportError).toHaveBeenCalledWith(
expect.any(Error),
expect.objectContaining({
tags: expect.objectContaining({ reason: 'unknown-turn' })
})
)
})

it('reports the same dropped ask only once', () => {
const store = useAgentConversationStore()
store.setThreadId('th')
Expand Down Expand Up @@ -960,6 +974,19 @@ describe('useAgentConversationStore', () => {
expect(reportError).toHaveBeenCalledTimes(1)
})

it('does not report a redelivery after the ask was rendered', () => {
const store = useAgentConversationStore()
store.setThreadId('th')
store.startTurn(T1)
const ask = runApproval('t1', 'turn-1:call-1')

store.ingest(ask)
store.abortActiveTurn()
store.ingest(ask)

expect(reportError).not.toHaveBeenCalled()
})

it('reports a late ask after hydrate retires an active turn', () => {
const store = useAgentConversationStore()
store.setThreadId('th')
Expand Down Expand Up @@ -1122,8 +1149,9 @@ describe('useAgentConversationStore', () => {
errorType: 'failure_delivering_agent_approval_ask',
tags: expect.objectContaining({
reason: 'unknown-kind',
ask_kind: 'pick_a_model'
})
ask_kind: 'unknown'
}),
context: expect.objectContaining({ rawAskKind: 'pick_a_model' })
})
)
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ export const useAgentConversationStore = defineStore(
replaceActive,
() => canvasSyncGate(),
() => canvasSyncOutcomeCount(),
reportUndeliverableAskData
reportUndeliverableAskData,
undeliverableAskReporter.markDelivered
)
}

Expand Down Expand Up @@ -212,19 +213,15 @@ export const useAgentConversationStore = defineStore(
*/
function reportUndeliverableAsk(
event: AgentChatEvent,
reason: 'no-live-turn' | 'settled-turn'
reason: 'no-live-turn' | 'settled-turn' | 'unknown-turn'
): void {
if (event.type !== 'agent_ask') return
reportUndeliverableAskData(event.data, reason)
}

function reportUndeliverableAskData(
data: Extract<AgentChatEvent, { type: 'agent_ask' }>['data'],
reason:
| 'no-live-turn'
| 'settled-turn'
| 'unknown-kind'
| 'unrendered-kind'
reason: 'no-live-turn' | 'settled-turn' | 'unknown-turn' | 'unknown-kind'
): void {
undeliverableAskReporter.report(data, reason, {
hasActiveTurn: activeTurnId.value !== null,
Expand Down Expand Up @@ -280,6 +277,8 @@ export const useAgentConversationStore = defineStore(
? undefined
: departedTurns.get(departedTurnKey(eventThreadId, eventMessageId))
if (reason) reportUndeliverableAsk(event, reason)
else if (eventThreadId === threadId.value)
reportUndeliverableAsk(event, 'unknown-turn')
return
}
if (event.type === 'agent_message_done') {
Expand Down Expand Up @@ -533,7 +532,8 @@ export const useAgentConversationStore = defineStore(
replaceActive,
() => canvasSyncGate(),
() => canvasSyncOutcomeCount(),
reportUndeliverableAskData
reportUndeliverableAskData,
undeliverableAskReporter.markDelivered
)
}
}
Expand Down
Loading