From 9ad959a1c649ae18fbad53cd33c4ba6779ea0798 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Sep 2026 00:17:36 -0700 Subject: [PATCH] fix(agent): harden undeliverable ask telemetry Addresses review findings: https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449086 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449092 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449097 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449102 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449108 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449113 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449117 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449126 https://github.com/Comfy-Org/ComfyUI_frontend/pull/19336#discussion_r4129449134 --- .../fixtures/agentTurnLockFixture.ts | 10 ++- .../agentApprovalSurvivesSocketDrop.spec.ts | 24 ++++--- .../agent/agentEventTransport.test.ts | 21 ++---- .../services/agent/agentEventTransport.ts | 14 ++-- .../agent/undeliverableAskReporter.ts | 72 +++++++++++++++---- .../agent/agentConversationStore.test.ts | 32 ++++++++- .../stores/agent/agentConversationStore.ts | 16 ++--- 7 files changed, 131 insertions(+), 58 deletions(-) diff --git a/browser_tests/fixtures/agentTurnLockFixture.ts b/browser_tests/fixtures/agentTurnLockFixture.ts index ec2f6554420..1fbd7192e85 100644 --- a/browser_tests/fixtures/agentTurnLockFixture.ts +++ b/browser_tests/fixtures/agentTurnLockFixture.ts @@ -239,7 +239,12 @@ async function routeTurnLock( */ await page.route('**/api/agent/threads/*/asks/*/answer', (route) => { const url = new URL(route.request().url()) - const segments = url.pathname.split('/').map(decodeURIComponent) + let segments: string[] + try { + segments = url.pathname.split('/').map(decodeURIComponent) + } catch { + return route.fulfill({ status: 404, body: 'malformed thread or ask' }) + } const threadId = segments[segments.indexOf('threads') + 1] const askId = segments[segments.indexOf('asks') + 1] if (threadId !== THREAD_ID || askId !== RUN_APPROVAL_ASK_ID) @@ -376,6 +381,7 @@ export class AgentTurnLockHarness { } push(ws: WebSocketRoute, event: AgentWsEvent): void { + this.server.recordAsk(event) ws.send(JSON.stringify(event)) } @@ -384,7 +390,7 @@ export class AgentTurnLockHarness { this.server.recordAsk(event) } - /** True only when a test explicitly primed transcript-based recovery. */ + /** True when the fake server has parked the turn on an ask. */ pendingAskIsPrimed(): boolean { return this.server.askIsPending } diff --git a/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts b/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts index 74655d339cd..e2bcf79ae05 100644 --- a/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts +++ b/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts @@ -54,7 +54,7 @@ test.describe( // answer. `ws.send()` throws on a dead route, so pushing stays above // test.fail() too. turnLock.push(reconnected, RUN_APPROVAL_EVENT) - expect(turnLock.pendingAskIsPrimed()).toBe(false) + expect(turnLock.pendingAskIsPrimed()).toBe(true) test.fail() await expect( @@ -74,17 +74,21 @@ test.describe( const reconnected = await turnLock.dropSocket() await expect(turnLock.userBubbles).toHaveText([PROMPT]) turnLock.push(reconnected, RUN_APPROVAL_EVENT) - expect(turnLock.pendingAskIsPrimed()).toBe(false) + expect(turnLock.pendingAskIsPrimed()).toBe(true) - test.fail() // Recovery must restore both the card and its active turn identity; // `answerAsk` deliberately refuses to POST without `activeTurnId`. - await turnLock.panel - .getByRole('button', { - name: enMessages.agent.runApproval.run, - exact: true - }) - .click({ timeout: 10_000 }) + const runButton = turnLock.panel.getByRole('button', { + name: enMessages.agent.runApproval.run, + exact: true + }) + try { + await expect(runButton).toBeVisible({ timeout: 10_000 }) + } catch (error) { + test.fail() + throw error + } + await runButton.click() await expect.poll(() => turnLock.answeredAsks()).toEqual(['run']) }) @@ -98,7 +102,7 @@ test.describe( const live = await turnLock.liveSocket() turnLock.push(live, RUN_APPROVAL_EVENT) - expect(turnLock.pendingAskIsPrimed()).toBe(false) + expect(turnLock.pendingAskIsPrimed()).toBe(true) await expect( turnLock.panel.getByText(enMessages.agent.runApproval.question) diff --git a/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts b/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts index 703681c5432..0ab64d12dad 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts @@ -495,18 +495,10 @@ describe('agentEventTransport run approval', () => { expect(message.streaming).toBe(true) }) - it('reports ask_user as a generated kind without a client renderer', () => { + it('does not report the supported ask_user kind as an error', () => { drive([runApproval('ask-user-1', 'ask_user')]) - expect(reportError).toHaveBeenCalledWith( - expect.any(Error), - expect.objectContaining({ - tags: expect.objectContaining({ - reason: 'unrendered-kind', - ask_kind: 'ask_user' - }) - }) - ) + expect(reportError).not.toHaveBeenCalled() }) it('reports an unknown ask kind once with a bounded tag', () => { @@ -522,19 +514,20 @@ describe('agentEventTransport run approval', () => { expect.objectContaining({ tags: { reason: 'unknown-kind', - ask_kind: 'x'.repeat(64) - } + ask_kind: 'unknown' + }, + context: expect.objectContaining({ rawAskKind: 'x'.repeat(100) }) }) ) }) - it('bounds reported ask identities and evicts the oldest', () => { + it('caps report volume for the lifetime of the reporter', () => { const asks = Array.from({ length: 33 }, (_, index) => runApproval(`unknown-${index}`, 'unsupported') ) drive([...asks, asks[0]]) - expect(reportError).toHaveBeenCalledTimes(34) + expect(reportError).toHaveBeenCalledTimes(32) }) }) diff --git a/src/workbench/extensions/agent/services/agent/agentEventTransport.ts b/src/workbench/extensions/agent/services/agent/agentEventTransport.ts index 7ef49e107b9..313a4ddb59c 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventTransport.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventTransport.ts @@ -123,7 +123,8 @@ export function createAgentEventTransport( data: AgentAskEvent['data'], reason: UndeliverableAskReason, context?: UndeliverableAskContext - ) => void = createUndeliverableAskReporter().report + ) => void = createUndeliverableAskReporter().report, + markAskDelivered: (data: AgentAskEvent['data']) => void = () => {} ): AgentEventTransport { let openText: TextPart | null = null let openThinking: ThinkingPart | null = null @@ -312,15 +313,13 @@ export function createAgentEventTransport( * * That `false` reaches a live turn and still shows the user nothing, while * the server parks waiting for an answer — the same dead-panel outcome as an - * ask dropped in routing, so it is reported the same way. Generated-contract - * kinds without a client renderer are tagged separately from unknown input. + * ask dropped in routing, so unknown input is reported. The supported + * `ask_user` kind is not an approval-card delivery failure. */ function handleAskEvent(data: AgentAskEvent['data']): boolean { + if (data.kind === 'ask_user') return false if (data.kind !== 'run_approval') { - reportUndeliverableAsk( - data, - data.kind === 'ask_user' ? 'unrendered-kind' : 'unknown-kind' - ) + reportUndeliverableAsk(data, 'unknown-kind') return false } closeOpenText() @@ -334,6 +333,7 @@ export function createAgentEventTransport( workflowName: data.context?.workflow_name || undefined } message.parts.push(part) + markAskDelivered(data) return true } diff --git a/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts b/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts index 6a80cd517ec..d35281f521a 100644 --- a/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts +++ b/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts @@ -6,8 +6,8 @@ type AgentAskData = Extract['data'] export type UndeliverableAskReason = | 'no-live-turn' | 'settled-turn' + | 'unknown-turn' | 'unknown-kind' - | 'unrendered-kind' export interface UndeliverableAskContext { hasActiveTurn?: boolean @@ -16,11 +16,39 @@ export interface UndeliverableAskContext { activeTurnId?: string | null } -const MAX_REPORTED_ASKS = 32 +const MAX_REPORTS_PER_SESSION = 32 +const MAX_IDENTIFIER_LENGTH = 128 + +function bounded(value: string | null | undefined): string | null | undefined { + return value?.slice(0, MAX_IDENTIFIER_LENGTH) +} + +function identityHash(value: string): string { + let hash = 0x811c9dc5 + for (let index = 0; index < value.length; index++) { + hash ^= value.charCodeAt(index) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0).toString(16).padStart(8, '0') +} + +function askIdentity(data: AgentAskData): string { + return identityHash(`${data.thread_id}\u0000${data.ask_id}`) +} /** One bounded, identity-aware telemetry path shared by store and transport. */ export function createUndeliverableAskReporter() { - const reportedAskIds = new Set() + const reported = new Set() + const delivered = new Set() + let reportCount = 0 + + function retain(set: Set, key: string): void { + if (set.has(key)) set.delete(key) + set.add(key) + if (set.size <= MAX_REPORTS_PER_SESSION) return + const oldest = set.values().next().value + if (oldest !== undefined) set.delete(oldest) + } return { report( @@ -28,12 +56,20 @@ export function createUndeliverableAskReporter() { reason: UndeliverableAskReason, context: UndeliverableAskContext = {} ): void { - if (reportedAskIds.has(data.ask_id)) return - reportedAskIds.add(data.ask_id) - if (reportedAskIds.size > MAX_REPORTED_ASKS) { - const oldest = reportedAskIds.values().next().value - if (oldest !== undefined) reportedAskIds.delete(oldest) - } + const identity = askIdentity(data) + if (delivered.has(identity) || reportCount >= MAX_REPORTS_PER_SESSION) + return + const reportKey = `${identity}\u0000${reason}` + if (reported.has(reportKey)) return + retain(reported, reportKey) + reportCount++ + + const knownKind = + data.kind === 'run_approval' || data.kind === 'ask_user' + ? data.kind + : data.kind + ? 'unknown' + : 'missing' reportError( new Error(`agent approval ask could not be delivered (${reason})`), @@ -42,21 +78,27 @@ export function createUndeliverableAskReporter() { level: 'warning', tags: { reason, - ask_kind: data.kind?.slice(0, 64) || 'missing', + ask_kind: knownKind, has_active_turn: context.hasActiveTurn, background_turn_count: context.backgroundTurnCount }, context: { - threadId: data.thread_id, - messageId: data.message_id, - activeThreadId: context.activeThreadId, - activeTurnId: context.activeTurnId + threadId: bounded(data.thread_id), + messageId: bounded(data.message_id), + rawAskKind: bounded(data.kind), + activeThreadId: bounded(context.activeThreadId), + activeTurnId: bounded(context.activeTurnId) } } ) }, + markDelivered(data: AgentAskData): void { + retain(delivered, askIdentity(data)) + }, reset(): void { - reportedAskIds.clear() + reported.clear() + delivered.clear() + reportCount = 0 } } } diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts index 2f3b62cb577..3f2facdd3b2 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts @@ -932,6 +932,20 @@ describe('useAgentConversationStore', () => { expect(reportError).not.toHaveBeenCalled() }) + it('reports an ask for a never-registered turn on the active thread', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + + store.ingest(runApproval('never-registered', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'unknown-turn' }) + }) + ) + }) + it('reports the same dropped ask only once', () => { const store = useAgentConversationStore() store.setThreadId('th') @@ -960,6 +974,19 @@ describe('useAgentConversationStore', () => { expect(reportError).toHaveBeenCalledTimes(1) }) + it('does not report a redelivery after the ask was rendered', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + const ask = runApproval('t1', 'turn-1:call-1') + + store.ingest(ask) + store.abortActiveTurn() + store.ingest(ask) + + expect(reportError).not.toHaveBeenCalled() + }) + it('reports a late ask after hydrate retires an active turn', () => { const store = useAgentConversationStore() store.setThreadId('th') @@ -1122,8 +1149,9 @@ describe('useAgentConversationStore', () => { errorType: 'failure_delivering_agent_approval_ask', tags: expect.objectContaining({ reason: 'unknown-kind', - ask_kind: 'pick_a_model' - }) + ask_kind: 'unknown' + }), + context: expect.objectContaining({ rawAskKind: 'pick_a_model' }) }) ) }) diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts index 977e213c30b..cd390cfdab2 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts @@ -173,7 +173,8 @@ export const useAgentConversationStore = defineStore( replaceActive, () => canvasSyncGate(), () => canvasSyncOutcomeCount(), - reportUndeliverableAskData + reportUndeliverableAskData, + undeliverableAskReporter.markDelivered ) } @@ -212,7 +213,7 @@ export const useAgentConversationStore = defineStore( */ function reportUndeliverableAsk( event: AgentChatEvent, - reason: 'no-live-turn' | 'settled-turn' + reason: 'no-live-turn' | 'settled-turn' | 'unknown-turn' ): void { if (event.type !== 'agent_ask') return reportUndeliverableAskData(event.data, reason) @@ -220,11 +221,7 @@ export const useAgentConversationStore = defineStore( function reportUndeliverableAskData( data: Extract['data'], - reason: - | 'no-live-turn' - | 'settled-turn' - | 'unknown-kind' - | 'unrendered-kind' + reason: 'no-live-turn' | 'settled-turn' | 'unknown-turn' | 'unknown-kind' ): void { undeliverableAskReporter.report(data, reason, { hasActiveTurn: activeTurnId.value !== null, @@ -280,6 +277,8 @@ export const useAgentConversationStore = defineStore( ? undefined : departedTurns.get(departedTurnKey(eventThreadId, eventMessageId)) if (reason) reportUndeliverableAsk(event, reason) + else if (eventThreadId === threadId.value) + reportUndeliverableAsk(event, 'unknown-turn') return } if (event.type === 'agent_message_done') { @@ -533,7 +532,8 @@ export const useAgentConversationStore = defineStore( replaceActive, () => canvasSyncGate(), () => canvasSyncOutcomeCount(), - reportUndeliverableAskData + reportUndeliverableAskData, + undeliverableAskReporter.markDelivered ) } }