diff --git a/browser_tests/fixtures/agentTurnLockFixture.ts b/browser_tests/fixtures/agentTurnLockFixture.ts index e11f9f66f27..ec2f6554420 100644 --- a/browser_tests/fixtures/agentTurnLockFixture.ts +++ b/browser_tests/fixtures/agentTurnLockFixture.ts @@ -2,12 +2,18 @@ import { expect, mergeTests } from '@playwright/test' import type { Locator, Page, WebSocketRoute } from '@playwright/test' import type { + AgentAnswerAccepted, AgentCancelAccepted, AgentError, AgentMessage, + AgentPendingAsk, AgentTurnAccepted } from '@comfyorg/ingest-types' -import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' +import { + zAgentPendingAsk, + zAgentPostMessageRequest +} from '@comfyorg/ingest-types/zod' +import { z } from 'zod' import enMessages from '@/locales/en/main.json' with { type: 'json' } import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' @@ -19,6 +25,8 @@ import { TestIds } from '@e2e/fixtures/selectors' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' import { webSocketFixture } from '@e2e/fixtures/ws' +const zAnswerRequest = z.object({ selected: z.array(z.string()) }) + const THREAD_ID = 'b9d0a2a1-0f2c-4f1a-9a5e-6b0f4f2c1d77' const TURN_ID = '2dd4f367-3399-4cb4-8127-547f531c289a' const WORKFLOW_ID = 'a81718a4-02ae-41e6-ae85-000000000001' @@ -67,6 +75,46 @@ export const TURN_DONE_EVENT: AgentWsEvent = { data: { message_id: TURN_ID, thread_id: THREAD_ID } } +const RUN_APPROVAL_ASK_ID = `${TURN_ID}:call-run-workflow` + +/** + * The option ids this ask offers. Declared separately because `AgentWsEvent` + * types `data` loosely, so reading them back off the event would be an + * `unknown` the route would have to cast. + */ +const RUN_APPROVAL_OPTION_IDS = ['run', 'cancel'] as const + +/** + * The frame the server sends when a turn parks waiting for the user to approve + * a run. Shaped after `pendingRunApproval`'s reader and cloud's `asks` writer: + * the turn does not proceed until an answer posts back, so a client that drops + * this frame strands the user with no way to answer and no error. + */ +export const RUN_APPROVAL_EVENT: AgentWsEvent = { + type: 'agent_ask', + data: { + message_id: TURN_ID, + thread_id: THREAD_ID, + ask_id: RUN_APPROVAL_ASK_ID, + kind: 'run_approval', + prompt: 'Run workflow “Unsaved Workflow”?', + context: { workflow_id: WORKFLOW_ID, workflow_name: 'Unsaved Workflow' }, + options: [ + { + id: RUN_APPROVAL_OPTION_IDS[0], + label: enMessages.agent.runApproval.run + }, + { + id: RUN_APPROVAL_OPTION_IDS[1], + label: enMessages.agent.runApproval.cancel + } + ], + min_selections: 1, + max_selections: 1, + allow_other: false + } +} + /** * The server's half of a turn, modelled on the real single-active-turn guard: * an assistant row goes `streaming` when a turn starts and only leaves that @@ -83,6 +131,8 @@ class TurnLockServer { private prompt = '' private rejected = 0 private posts = 0 + private readonly answered: string[][] = [] + private pendingAsk: AgentPendingAsk | undefined get turnIsStreaming(): boolean { return this.streaming @@ -101,6 +151,25 @@ class TurnLockServer { this.posts++ } + /** Every ask answer the server accepted, in order, as the selected ids. */ + get answers(): string[][] { + return this.answered + } + + get askIsPending(): boolean { + return this.pendingAsk !== undefined + } + + recordAnswer(selected: string[]): void { + this.answered.push(selected) + this.pendingAsk = undefined + } + + recordAsk(event: AgentWsEvent): void { + if (event.type === 'agent_ask') + this.pendingAsk = zAgentPendingAsk.parse(event.data) + } + completeTurn(): void { this.streaming = false } @@ -124,7 +193,8 @@ class TurnLockServer { seq: 2, role: 'assistant', status: this.streaming ? 'streaming' : 'complete', - workflow_id: WORKFLOW_ID + workflow_id: WORKFLOW_ID, + pending_ask: this.pendingAsk } ] } @@ -160,6 +230,48 @@ async function routeTurnLock( }) }) + /** + * The glob accepts any thread and any ask id, so without these checks an + * answer posted against the wrong ask -- or an option this ask never offered + * -- would still be recorded and `answeredAsks()` would report success. The + * assertion this fixture exists to support is "the answer left the client for + * *this* ask", so the route has to be the thing that enforces it. + */ + await page.route('**/api/agent/threads/*/asks/*/answer', (route) => { + const url = new URL(route.request().url()) + const segments = url.pathname.split('/').map(decodeURIComponent) + const threadId = segments[segments.indexOf('threads') + 1] + const askId = segments[segments.indexOf('asks') + 1] + if (threadId !== THREAD_ID || askId !== RUN_APPROVAL_ASK_ID) + return route.fulfill({ + status: 404, + contentType: 'application/json', + body: JSON.stringify({ error: 'unknown thread or ask' }) + }) + + let body: unknown + try { + body = route.request().postDataJSON() + } catch { + return route.fulfill({ status: 400, body: 'invalid JSON' }) + } + const parsed = zAnswerRequest.safeParse(body) + if (!parsed.success) + return route.fulfill({ status: 400, body: 'invalid answer' }) + const selected = parsed.data.selected + const offered: readonly string[] = RUN_APPROVAL_OPTION_IDS + if (selected.length !== 1 || !offered.includes(selected[0])) + return route.fulfill({ + status: 422, + contentType: 'application/json', + body: JSON.stringify({ error: 'selection not offered by this ask' }) + }) + + server.recordAnswer(selected) + const accepted: AgentAnswerAccepted = { status: 'answered' } + return route.fulfill(jsonRoute(accepted)) + }) + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => { server.completeTurn() const accepted: AgentCancelAccepted = { status: 'cancelling' } @@ -267,6 +379,16 @@ export class AgentTurnLockHarness { ws.send(JSON.stringify(event)) } + /** Makes an ask available through transcript hydration, independently of WS delivery. */ + primePendingAsk(event: AgentWsEvent): void { + this.server.recordAsk(event) + } + + /** True only when a test explicitly primed transcript-based recovery. */ + pendingAskIsPrimed(): boolean { + return this.server.askIsPending + } + /** * Replays the Web Audio work an audio preview does while a turn is live: * open an AudioContext, `decodeAudioData` real WAV bytes, close it. This is @@ -303,6 +425,42 @@ export class AgentTurnLockHarness { }) } + /** + * Every ask the user answered through the panel, flattened to the selected + * option ids. Reads the fake server rather than the DOM, so it proves the + * answer actually left the client. + */ + answeredAsks(): string[] { + return this.server.answers.flat() + } + + /** Posts an answer through the browser so the real route validator sees it. */ + async postAnswer( + threadId: string, + askId: string, + selected: string[] + ): Promise { + return await this.page.evaluate( + async ({ threadId, askId, selected }) => { + const response = await fetch( + `/api/agent/threads/${encodeURIComponent(threadId)}/asks/${encodeURIComponent(askId)}/answer`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ selected }) + } + ) + return response.status + }, + { threadId, askId, selected } + ) + } + + /** The socket the client is currently on, with no drop. */ + async liveSocket(): Promise { + return this.getWebSocket() + } + /** Drops the live socket and resolves with the one the client reconnects on. */ async dropSocket(): Promise { // Resolve the live route first: with no socket open yet both calls would diff --git a/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts b/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts new file mode 100644 index 00000000000..74655d339cd --- /dev/null +++ b/browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts @@ -0,0 +1,115 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { + RUN_APPROVAL_EVENT, + agentTurnLockTest as test +} from '@e2e/fixtures/agentTurnLockFixture' + +// The worst shape a stall can take, from a real user report: they asked for an +// img2img workflow, then for img2video, and the panel went dead. They restarted +// the agent tab to get out of it. +// +// A session-id trace said the backend never hung. The turn was alive the whole +// time, parked waiting for the user to approve a tool call — and the approval +// prompt never made it into the panel. The product was working and waiting on +// the user; the user had no way to answer. +// +// The mechanism is the socket blip. `useAgentSession.onStatus(false)` answers a +// transient close with `abortActiveTurn()` + `dropBackgroundTurns()`, so the +// transport slot is null and the background map is empty. The server, which +// never learned the socket went away, then emits `agent_ask` for the approval +// it is parked on, and `agentConversationStore.ingest` has nothing to route it +// to: `ingestBackgroundTurnEvent` returns at `!entry` and the frame is gone. +// Nothing errors, the panel looks idle, the server looks healthy. +// +// Deliberately not a refresh: on a refresh the ask comes back, because the +// persisted row carries `pending_ask` and `normalizeAgentTranscript` restores +// the card. That asymmetry is why restarting the tab "fixed" it for the user +// and why this only ever showed up through a feedback form. +test.describe.configure({ timeout: 120_000 }) +test.use({ connectWebSocketToServer: false }) + +test.describe( + 'an approval ask survives a socket drop', + { tag: ['@cloud', '@ui'] }, + () => { + const PROMPT = 'turn this image into a video' + + test.beforeEach(async ({ turnLock }) => { + await turnLock.openOnBlankWorkflow() + await turnLock.startTurn(PROMPT) + }) + + test('the approval card reaches the panel after a reconnect', async ({ + turnLock + }) => { + const reconnected = await turnLock.dropSocket() + + // Preconditions stay above the marker so a wiped thread or a lost prompt + // reads as a real failure rather than the expected one. + await expect(turnLock.userBubbles).toHaveText([PROMPT]) + + // The server is parked on this ask and will not proceed without an + // answer. `ws.send()` throws on a dead route, so pushing stays above + // test.fail() too. + turnLock.push(reconnected, RUN_APPROVAL_EVENT) + expect(turnLock.pendingAskIsPrimed()).toBe(false) + + test.fail() + await expect( + turnLock.panel.getByText(enMessages.agent.runApproval.question) + ).toBeVisible() + await expect( + turnLock.panel.getByRole('button', { + name: enMessages.agent.runApproval.run, + exact: true + }) + ).toBeVisible() + }) + + test('the user can answer the ask that arrived after the reconnect', async ({ + turnLock + }) => { + const reconnected = await turnLock.dropSocket() + await expect(turnLock.userBubbles).toHaveText([PROMPT]) + turnLock.push(reconnected, RUN_APPROVAL_EVENT) + expect(turnLock.pendingAskIsPrimed()).toBe(false) + + test.fail() + // Recovery must restore both the card and its active turn identity; + // `answerAsk` deliberately refuses to POST without `activeTurnId`. + await turnLock.panel + .getByRole('button', { + name: enMessages.agent.runApproval.run, + exact: true + }) + .click({ timeout: 10_000 }) + await expect.poll(() => turnLock.answeredAsks()).toEqual(['run']) + }) + + // Keeps the locators above honest. If the approval card never renders under + // this fixture at all — a changed testid, a gate, reworded copy — this case + // reddens and the two expected failures above stop being evidence of the + // defect they name. + test('the same ask renders when the socket never drops', async ({ + turnLock + }) => { + const live = await turnLock.liveSocket() + + turnLock.push(live, RUN_APPROVAL_EVENT) + expect(turnLock.pendingAskIsPrimed()).toBe(false) + + await expect( + turnLock.panel.getByText(enMessages.agent.runApproval.question) + ).toBeVisible() + await turnLock.panel + .getByRole('button', { + name: enMessages.agent.runApproval.run, + exact: true + }) + .click() + await expect.poll(() => turnLock.answeredAsks()).toEqual(['run']) + }) + } +) diff --git a/src/composables/billing/useBillingContext.test.ts b/src/composables/billing/useBillingContext.test.ts index ba80b84d6e4..2bd3a5ef1ed 100644 --- a/src/composables/billing/useBillingContext.test.ts +++ b/src/composables/billing/useBillingContext.test.ts @@ -8,6 +8,7 @@ import type { Plan } from '@/platform/workspace/api/workspaceApi' import { + authenticatedRemoteConfigState, remoteConfig, remoteConfigState } from '@/platform/remoteConfig/remoteConfig' @@ -182,6 +183,7 @@ describe('useBillingContext', () => { beforeEach(() => { remoteConfig.value = {} remoteConfigState.value = 'unloaded' + authenticatedRemoteConfigState.value = 'unloaded' mockIsPersonal.value = true mockBillingRail.value = undefined mockSetWorkspaceBillingRail.mockImplementation( @@ -358,6 +360,7 @@ describe('useBillingContext', () => { it('routes migrated legacy Stripe topups through workspace billing', async () => { remoteConfig.value = { legacy_billing_migration_enabled: true } remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' mockBillingRail.value = 'legacy_stripe' const context = useBillingContext() diff --git a/src/composables/useFeatureFlags.test.ts b/src/composables/useFeatureFlags.test.ts index 954e2b948a2..750fd9a0400 100644 --- a/src/composables/useFeatureFlags.test.ts +++ b/src/composables/useFeatureFlags.test.ts @@ -16,12 +16,14 @@ import { } from '@/composables/useFeatureFlags' import * as distributionTypes from '@/platform/distribution/types' import { + authenticatedRemoteConfigState, cachedBillingControlEnabled, cachedLegacyBillingMigrationEnabled, cachedV1PaymentRecovery, remoteConfig, remoteConfigState, - sessionAgentGrant + sessionAgentGrant, + sessionAgentGrantValidUntil } from '@/platform/remoteConfig/remoteConfig' import { api } from '@/scripts/api' import { getSessionOverride } from '@/utils/sessionFeatureFlagOverride' @@ -339,11 +341,13 @@ describe('useFeatureFlags', () => { beforeEach(() => { vi.mocked(distributionTypes).isCloud = true remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' }) afterEach(() => { vi.mocked(distributionTypes).isCloud = false remoteConfigState.value = 'unloaded' + authenticatedRemoteConfigState.value = 'unloaded' remoteConfig.value = {} cachedLegacyBillingMigrationEnabled.value = undefined }) @@ -515,6 +519,7 @@ describe('useFeatureFlags', () => { it('prefers authenticated remoteConfig over the server feature fallback', () => { remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = { billing_control_enabled: false, v1_payment_recovery: true @@ -528,6 +533,7 @@ describe('useFeatureFlags', () => { it('falls back to api.getServerFeature when authenticated config omits the flag', () => { remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = {} vi.mocked(api.getServerFeature).mockImplementation( (path, defaultValue) => { @@ -824,20 +830,25 @@ describe('useFeatureFlags', () => { beforeEach(() => { vi.mocked(distributionTypes).isCloud = true remoteConfigState.value = 'unloaded' + authenticatedRemoteConfigState.value = 'unloaded' remoteConfig.value = {} sessionAgentGrant.value = undefined + sessionAgentGrantValidUntil.value = undefined }) afterEach(() => { vi.mocked(distributionTypes).isCloud = false remoteConfigState.value = 'unloaded' + authenticatedRemoteConfigState.value = 'unloaded' remoteConfig.value = {} sessionAgentGrant.value = undefined + sessionAgentGrantValidUntil.value = undefined }) it('is false off-cloud even when the authenticated config grants it', () => { vi.mocked(distributionTypes).isCloud = false remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = { 'agent-in-app-experience': true } const { flags } = useFeatureFlags() @@ -846,6 +857,7 @@ describe('useFeatureFlags', () => { it('is false during the anonymous window even when the config already grants it', () => { remoteConfigState.value = 'anonymous' + authenticatedRemoteConfigState.value = 'unloaded' remoteConfig.value = { 'agent-in-app-experience': true } const { flags } = useFeatureFlags() @@ -854,6 +866,7 @@ describe('useFeatureFlags', () => { it('is true once the authenticated config grants it', () => { remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = { 'agent-in-app-experience': true } const { flags } = useFeatureFlags() @@ -862,6 +875,7 @@ describe('useFeatureFlags', () => { it('is false when the authenticated config omits the key', () => { remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = {} const { flags } = useFeatureFlags() @@ -870,12 +884,15 @@ describe('useFeatureFlags', () => { it('keeps this session granted when a later refresh fails transiently', () => { remoteConfigState.value = 'authenticated' + authenticatedRemoteConfigState.value = 'authenticated' remoteConfig.value = { 'agent-in-app-experience': true } sessionAgentGrant.value = true + sessionAgentGrantValidUntil.value = Date.now() + 60_000 const { flags } = useFeatureFlags() expect(flags.agentInAppExperienceEnabled).toBe(true) remoteConfigState.value = 'error' + authenticatedRemoteConfigState.value = 'error' expect(flags.agentInAppExperienceEnabled).toBe(true) }) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index f753adc95d0..e0c135aed9f 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -47,6 +47,7 @@ let consentStore: ReturnType let workspaceStore: ReturnType const currentUser = ref<{ id: string } | null>({ id: 'account-a' }) +const isAuthInitialized = ref(true) const firstRunTookScreen = ref(false) const activeTour = ref(null) let startupDecision: Promise = Promise.resolve(true) @@ -54,6 +55,7 @@ let startupDecision: Promise = Promise.resolve(true) vi.mock(import('@/composables/auth/useCurrentUser'), () => ({ useCurrentUser: () => ({ resolvedUserInfo: currentUser, + isAuthInitialized, isLoggedIn: computed(() => currentUser.value !== null) }) })) diff --git a/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts b/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts index a9b7e6392a3..ea37f15a51a 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventTransport.test.ts @@ -2,6 +2,8 @@ import fs from 'fs' import { describe, expect, it, vi } from 'vitest' +import { reportError } from '@/platform/telemetry/reportError' + import type { AgentMessages } from '../../schemas/agentApiSchema' import { toTurnId, zAgentWsEvent } from '../../schemas/agentApiSchema' import { normalizeAgentTranscript } from './agentTranscript' @@ -17,6 +19,10 @@ import type { } from './agentMessageParts' import { createAssistantMessage } from './agentMessageParts' +vi.mock(import('@/platform/telemetry/reportError'), () => ({ + reportError: vi.fn() +})) + const fixtureText = import.meta.glob( '../../schemas/__fixtures__/agent/*.jsonl', { query: '?raw', import: 'default', eager: true } @@ -104,14 +110,17 @@ function activeTab( } } -function runApproval(askId = 'turn-1:call-1'): AgentChatEvent { +function runApproval( + askId = 'turn-1:call-1', + kind = 'run_approval' +): AgentChatEvent { return zAgentWsEvent.parse({ type: 'agent_ask', data: { thread_id: 't', message_id: 'm', ask_id: askId, - kind: 'run_approval', + kind, context: { workflow_id: 'workflow-1', workflow_name: 'Portrait workflow' @@ -485,6 +494,63 @@ describe('agentEventTransport run approval', () => { ).toEqual(['ask-2']) expect(message.streaming).toBe(true) }) + + it('reports ask_user as a generated kind without a client renderer', () => { + drive([runApproval('ask-user-1', 'ask_user')]) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ + reason: 'unrendered-kind', + ask_kind: 'ask_user' + }) + }) + ) + }) + + it('reports an unknown ask kind once with a bounded-cardinality tag', () => { + const unknownKind = 'x'.repeat(100) + drive([ + runApproval('unknown-1', unknownKind), + runApproval('unknown-1', unknownKind) + ]) + + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: { + reason: 'unknown-kind', + ask_kind: 'unknown' + }, + context: expect.objectContaining({ askKind: unknownKind }) + }) + ) + }) + + it('bounds report volume for the session', () => { + const asks = Array.from({ length: 33 }, (_, index) => + runApproval(`unknown-${index}`, 'unsupported') + ) + drive([...asks, asks[0]]) + + expect(reportError).toHaveBeenCalledTimes(32) + }) + + it('does not duplicate an approval restored by hydration', () => { + const message = createAssistantMessage(T) + message.parts.push({ type: 'runApproval', askId: 'restored-ask' }) + const emit = vi.fn<(m: AssistantMessage) => void>() + const transport = createAgentEventTransport(message, emit) + + transport.ingest(runApproval('restored-ask')) + + expect( + message.parts.filter((part) => part.type === 'runApproval') + ).toHaveLength(1) + expect(emit).not.toHaveBeenCalled() + }) }) describe('agentEventTransport settle lifecycle', () => { @@ -532,6 +598,34 @@ describe('agentEventTransport settle lifecycle', () => { expect(toolParts(message)).toHaveLength(0) }) + it('reports a late approval ask once after settle', () => { + const message = createAssistantMessage(T) + const transport = createAgentEventTransport(message, vi.fn()) + transport.settle() + + transport.ingest(runApproval('late-ask')) + transport.ingest(runApproval('late-ask')) + + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'settled-turn' }) + }) + ) + }) + + it('does not report redelivery of an approval rendered before settle', () => { + const message = createAssistantMessage(T) + const transport = createAgentEventTransport(message, vi.fn()) + transport.ingest(runApproval('visible-ask')) + transport.settle() + + transport.ingest(runApproval('visible-ask')) + + expect(reportError).not.toHaveBeenCalled() + }) + it('records an explicitly targeted node link when the agent switches workflow tabs', () => { const message = drive([ delta('opening it now'), diff --git a/src/workbench/extensions/agent/services/agent/agentEventTransport.ts b/src/workbench/extensions/agent/services/agent/agentEventTransport.ts index b7ab242aa6f..66c59d4d96d 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventTransport.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventTransport.ts @@ -9,6 +9,11 @@ import type { ToolPart } from './agentMessageParts' import { snapshotMessage } from './agentMessageParts' +import type { + UndeliverableAskContext, + UndeliverableAskReason +} from './undeliverableAskReporter' +import { createUndeliverableAskReporter } from './undeliverableAskReporter' export type AgentChatEvent = Extract< AgentWsEvent, @@ -113,7 +118,12 @@ export function createAgentEventTransport( * `shouldAwaitCanvasSync` at its default) never spuriously looks "caught * up". */ - getCanvasSyncOutcomeCount: () => number = () => 0 + getCanvasSyncOutcomeCount: () => number = () => 0, + reportUndeliverableAsk: ( + data: AgentAskEvent['data'], + reason: UndeliverableAskReason, + context?: UndeliverableAskContext + ) => void = createUndeliverableAskReporter().report ): AgentEventTransport { let openText: TextPart | null = null let openThinking: ThinkingPart | null = null @@ -127,6 +137,15 @@ export function createAgentEventTransport( .filter((part): part is ToolPart => part.type === 'tool') .map((part) => [part.callId, part]) ) + // Hydration can restore a pending approval before its websocket frame is + // replayed. Keep the rendered identities beside the tool-call identity map + // so replay neither duplicates a card nor misreports a visible card after + // the turn settles. + const renderedAskIds = new Set( + message.parts.flatMap((part) => + part.type === 'runApproval' ? [part.askId] : [] + ) + ) let settled = false let lastTabTargetKey: string | undefined // Tool parts whose frame reported done but whose displayed state is held at @@ -299,9 +318,21 @@ export function createAgentEventTransport( * Applies one `agent_ask` frame. Only the `run_approval` kind renders a * part; returns `false` for any other kind, mirroring `ingest`'s early * `return` for that case. + * + * That `false` reaches a live turn and still shows the user nothing, while + * the server parks waiting for an answer — the same dead-panel outcome as an + * ask dropped in routing, so it is reported the same way. Generated-contract + * kinds without a client renderer are tagged separately from unknown input. */ function handleAskEvent(data: AgentAskEvent['data']): boolean { - if (data.kind !== 'run_approval') return false + if (data.kind !== 'run_approval') { + reportUndeliverableAsk( + data, + data.kind === 'ask_user' ? 'unrendered-kind' : 'unknown-kind' + ) + return false + } + if (renderedAskIds.has(data.ask_id)) return false closeOpenText() closeOpenThinking() message.thinking = false @@ -313,6 +344,7 @@ export function createAgentEventTransport( workflowName: data.context?.workflow_name || undefined } message.parts.push(part) + renderedAskIds.add(data.ask_id) return true } @@ -414,7 +446,11 @@ export function createAgentEventTransport( } function ingest(event: AgentChatEvent): void { - if (settled) return + if (settled) { + if (event.type === 'agent_ask' && !renderedAskIds.has(event.data.ask_id)) + reportUndeliverableAsk(event.data, 'settled-turn') + return + } if (applyChatEvent(event)) emit(snapshotMessage(message)) } diff --git a/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts b/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts new file mode 100644 index 00000000000..0a91ad0312d --- /dev/null +++ b/src/workbench/extensions/agent/services/agent/undeliverableAskReporter.ts @@ -0,0 +1,70 @@ +import { reportError } from '@/platform/telemetry/reportError' +import type { AgentWsEvent } from '../../schemas/agentApiSchema' + +type AgentAskData = Extract['data'] + +export type UndeliverableAskReason = + | 'no-live-turn' + | 'settled-turn' + | 'unknown-kind' + | 'unrendered-kind' + +export interface UndeliverableAskContext { + hasActiveTurn?: boolean + backgroundTurnCount?: number + activeThreadId?: string | null + activeTurnId?: string | null +} + +const MAX_REPORTED_ASKS = 32 + +/** One bounded, identity-aware telemetry path shared by store and transport. */ +export function createUndeliverableAskReporter() { + const reportedAskIds = new Set() + + return { + report( + data: AgentAskData, + reason: UndeliverableAskReason, + context: UndeliverableAskContext = {} + ): void { + if ( + reportedAskIds.has(data.ask_id) || + reportedAskIds.size >= MAX_REPORTED_ASKS + ) + return + reportedAskIds.add(data.ask_id) + + const askKind = + data.kind === undefined + ? 'missing' + : data.kind === 'run_approval' || data.kind === 'ask_user' + ? data.kind + : 'unknown' + + reportError( + new Error(`agent approval ask could not be delivered (${reason})`), + { + errorType: 'failure_delivering_agent_approval_ask', + level: 'warning', + tags: { + reason, + ask_kind: askKind, + has_active_turn: context.hasActiveTurn, + background_turn_count: context.backgroundTurnCount + }, + context: { + threadId: data.thread_id, + messageId: data.message_id, + askKind: data.kind, + activeThreadId: context.activeThreadId, + activeTurnId: context.activeTurnId + } + } + ) + }, + reset(): void { + reportedAskIds.clear() + } + } +} diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts index 26c61288310..1358422cf9c 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts @@ -2,12 +2,18 @@ import { createPinia, setActivePinia } from 'pinia' import { beforeEach, describe, expect, it, vi } from 'vitest' import { nextTick, ref, watch } from 'vue' +import { reportError } from '@/platform/telemetry/reportError' + import type { AgentMessages, TurnId } from '../../schemas/agentApiSchema' import { zAgentMessages, zAgentWsEvent } from '../../schemas/agentApiSchema' import type { AgentChatEvent } from '../../services/agent/agentEventTransport' import { useAgentConversationStore } from './agentConversationStore' +vi.mock(import('@/platform/telemetry/reportError'), () => ({ + reportError: vi.fn() +})) + const chat = (raw: unknown): AgentChatEvent => zAgentWsEvent.parse(raw) const thinking = (id: string, delta: string): AgentChatEvent => chat({ @@ -35,6 +41,29 @@ const done = (id: string): AgentChatEvent => type: 'agent_message_done', data: { message_id: id, thread_id: 'th', usage: null } }) +const runApproval = ( + id: string, + askId: string, + overrides: Record = {} +): AgentChatEvent => + chat({ + type: 'agent_ask', + data: { + message_id: id, + thread_id: 'th', + ask_id: askId, + kind: 'run_approval', + prompt: 'Run workflow?', + options: [ + { id: 'run', label: 'Run' }, + { id: 'cancel', label: 'Cancel' } + ], + min_selections: 1, + max_selections: 1, + allow_other: false, + ...overrides + } + }) const askResolved = (id: string, askId: string): AgentChatEvent => chat({ type: 'agent_ask_resolved', @@ -465,9 +494,7 @@ describe('useAgentConversationStore', () => { store.ingest(delta('assistant-message-1', 'Running now.')) expect( - store.messages[0].parts.some( - (part) => (part as { type: string }).type === 'runApproval' - ) + store.messages[0].parts.some((part) => part.type === 'runApproval') ).toBe(false) expect(partTexts(store)).toContain('Running now.') expect(store.isStreaming).toBe(true) @@ -810,4 +837,303 @@ describe('useAgentConversationStore', () => { 'assistant' ]) }) + describe('a dropped approval ask is reported', () => { + it('reports when the socket-drop teardown left nothing to route the ask to', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.ingest(delta('t1', 'building your workflow')) + + // What onStatus(false) does today: settle the active turn locally and + // empty the background map, while the server keeps running the turn. + store.abortActiveTurn() + store.dropBackgroundTurns() + + // The server, still running, now asks the user to approve the run. + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect( + store.messages.some((message) => + message.parts.some((part) => part.type === 'runApproval') + ) + ).toBe(false) + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'failure_delivering_agent_approval_ask', + level: 'warning', + tags: expect.objectContaining({ + reason: 'no-live-turn', + ask_kind: 'run_approval' + }) + }) + ) + }) + + it('stays quiet when the ask reaches its turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.ingest(delta('t1', 'building your workflow')) + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(store.messages[0].parts).toContainEqual( + expect.objectContaining({ + type: 'runApproval', + askId: 'turn-1:call-1' + }) + ) + expect(reportError).not.toHaveBeenCalled() + }) + + it('stays quiet for every non-ask frame the same routing drops', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.abortActiveTurn() + store.dropBackgroundTurns() + + store.ingest(delta('t1', 'more text')) + store.ingest(thinking('t1', 'still thinking')) + store.ingest(done('t1')) + + expect(reportError).not.toHaveBeenCalled() + }) + + it('stays quiet for an ask belonging to an untracked thread', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.abortActiveTurn() + + store.ingest( + chat({ + ...runApproval('t1', 'turn-1:call-1'), + data: { + ...runApproval('t1', 'turn-1:call-1').data, + thread_id: 'another-thread' + } + }) + ) + + expect(reportError).not.toHaveBeenCalled() + }) + + it('reports the same dropped ask only once', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.abortActiveTurn() + + const ask = runApproval('t1', 'turn-1:call-1') + store.ingest(ask) + store.ingest(ask) + + expect(reportError).toHaveBeenCalledTimes(1) + }) + + it('dedupes one ask across the transport and store routing paths', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.ingest(done('t1')) + + const ask = runApproval('t1', 'turn-1:call-1') + store.ingest(ask) + store.dropBackgroundTurns() + store.ingest(ask) + + expect(reportError).toHaveBeenCalledTimes(1) + }) + + it('reports a late ask after hydrate retires an active turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + + store.hydrate([]) + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'no-live-turn' }) + }) + ) + }) + + it('reports a late ask after resume discards a settled turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.ingest(done('t1')) + + store.hydrate([]) + store.resumeBackgroundTurn() + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'settled-turn' }) + }) + ) + }) + + it('reports after a dropped background turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.dropBackgroundTurns() + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'no-live-turn' }) + }) + ) + }) + + it('records a background turn replaced by a newer turn in the same thread', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.startTurn(T2) + store.stashActiveTurn() + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'no-live-turn' }) + }) + ) + }) + + it('preserves the settled reason when dropping background turns', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.ingest(done('t1')) + store.dropBackgroundTurns() + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'settled-turn' }) + }) + ) + }) + + it('records malformed background settlement before deleting the turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.stashActiveTurn() + store.settleBackgroundTurn(T1) + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'settled-turn' }) + }) + ) + }) + + it('reports after a settled active turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.ingest(done('t1')) + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'settled-turn' }) + }) + ) + }) + + it('keeps an aborted identity after an unrelated turn starts', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.abortActiveTurn() + store.startTurn(T2) + + store.ingest(runApproval('t1', 'turn-1:call-1')) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'no-live-turn' }) + }) + ) + }) + + it('records an aborted turn under the thread that owns its transport', () => { + const store = useAgentConversationStore() + store.setThreadId('original-thread') + store.startTurn(T1) + store.setThreadId('replacement-thread') + store.abortActiveTurn() + + store.ingest( + chat({ + ...runApproval('t1', 'turn-1:call-1'), + data: { + ...runApproval('t1', 'turn-1:call-1').data, + thread_id: 'original-thread' + } + }) + ) + + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + tags: expect.objectContaining({ reason: 'no-live-turn' }) + }) + ) + }) + + it('reports an ask kind the panel has no card for', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.startTurn(T1) + store.ingest(delta('t1', 'building your workflow')) + + store.ingest(runApproval('t1', 'turn-1:call-1', { kind: 'pick_a_model' })) + + expect( + store.messages[0].parts.some((part) => part.type === 'runApproval') + ).toBe(false) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'failure_delivering_agent_approval_ask', + tags: expect.objectContaining({ + reason: 'unknown-kind', + ask_kind: 'unknown' + }), + context: expect.objectContaining({ askKind: 'pick_a_model' }) + }) + ) + }) + }) }) diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts index 5291b18c05b..5aa5e9b4dc7 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts @@ -10,6 +10,7 @@ import { createAgentEventTransport } from '../../services/agent/agentEventTransp import type { AssistantMessage } from '../../services/agent/agentMessageParts' import { createAssistantMessage } from '../../services/agent/agentMessageParts' import { normalizeAgentTranscript } from '../../services/agent/agentTranscript' +import { createUndeliverableAskReporter } from '../../services/agent/undeliverableAskReporter' import type { UserAttachment } from '../../services/agent/agentTranscript' import type { WorkflowReference } from '../../types/workflowReference' @@ -36,6 +37,8 @@ interface BackgroundTurn { settled: boolean } +const MAX_DEPARTED_TURNS = 32 + export const useAgentConversationStore = defineStore( 'agentConversation', () => { @@ -49,6 +52,7 @@ export const useAgentConversationStore = defineStore( const latestWorkflowId = ref() const resolvedPaywallIds = ref(new Set()) let transport: AgentEventTransport | null = null + let activeTransportThreadId: string | null = null let liveMessage: AssistantMessage | null = null // PM-1575: whether a newly-created transport should hold a tool-call's // chat "done" state back until canvas catch-up is confirmed (see @@ -81,6 +85,8 @@ export const useAgentConversationStore = defineStore( const backgroundTurns = new Map() let hydratedMessageIds = new Set() let hydratedAssistantTurnIds = new Set() + const undeliverableAskReporter = createUndeliverableAskReporter() + const departedTurns = new Map() const activeIndex = ref(-1) function replaceActive(message: AssistantMessage): void { @@ -157,11 +163,13 @@ export const useAgentConversationStore = defineStore( liveMessage = message activeIndex.value = messages.value.push(message) - 1 activeTurnId.value = turnId + activeTransportThreadId = threadId.value transport = createAgentEventTransport( message, replaceActive, () => canvasSyncGate(), - () => canvasSyncOutcomeCount() + () => canvasSyncOutcomeCount(), + reportUndeliverableAskData ) } @@ -181,10 +189,47 @@ export const useAgentConversationStore = defineStore( ingestActiveTabEvent(event, eventThreadId) return } + // Unreachable for `agent_ask`: the schema makes `thread_id` required on + // every event except `agent_active_tab`, handled above. if (eventThreadId === undefined) return ingestBackgroundTurnEvent(event, eventThreadId) } + /** + * A dropped `agent_ask` is the one lost frame with no user-visible symptom: + * the server parks the turn waiting for an answer, the panel keeps showing + * whatever it last had, and nothing errors. The only report we have of this + * class reached us through a feedback form that happened to include a + * session id, so every `return` that can swallow an ask says so here. + * + * Deliberately not a `pushError`: the user cannot act on it, and a notice + * would replace a silent stall with a stall plus a scary message. This is + * for the dashboard. + */ + function reportUndeliverableAsk( + event: AgentChatEvent, + reason: 'no-live-turn' | 'settled-turn' + ): void { + if (event.type !== 'agent_ask') return + reportUndeliverableAskData(event.data, reason) + } + + function reportUndeliverableAskData( + data: Extract['data'], + reason: + | 'no-live-turn' + | 'settled-turn' + | 'unknown-kind' + | 'unrendered-kind' + ): void { + undeliverableAskReporter.report(data, reason, { + hasActiveTurn: activeTurnId.value !== null, + backgroundTurnCount: backgroundTurns.size, + activeThreadId: threadId.value, + activeTurnId: activeTurnId.value + }) + } + function ingestActiveTurnEvent( event: AgentChatEvent, activeTransport: AgentEventTransport @@ -203,6 +248,7 @@ export const useAgentConversationStore = defineStore( // something pending -- a settled transport with nothing held has no // reason to stay reachable. function settleActiveTurn(activeTransport: AgentEventTransport): void { + rememberDepartedActiveTurn('settled-turn') activeTransport.settle() if (activeTransport.hasPendingCanvasSync()) settledActiveTransports.add(activeTransport) @@ -223,7 +269,15 @@ export const useAgentConversationStore = defineStore( eventThreadId: string ): void { const entry = backgroundTurns.get(eventThreadId) - if (!entry || entry.messageId !== event.data.message_id) return + if (!entry || entry.messageId !== event.data.message_id) { + const eventMessageId = event.data.message_id + const reason = + eventMessageId === undefined + ? undefined + : departedTurns.get(departedTurnKey(eventThreadId, eventMessageId)) + if (reason) reportUndeliverableAsk(event, reason) + return + } if (event.type === 'agent_message_done') { entry.transport.settle() entry.settled = true @@ -260,6 +314,7 @@ export const useAgentConversationStore = defineStore( function abortActiveTurn(): void { if (!transport) return + rememberDepartedActiveTurn('no-live-turn') transport.settle() // Not `settledActiveTransports`: an abort is not a natural completion // whose held parts might still catch up, so flush them to `done` and @@ -275,6 +330,15 @@ export const useAgentConversationStore = defineStore( abortActiveTurn() return } + const replaced = backgroundTurns.get(threadId.value) + if (replaced) { + rememberDepartedTurn( + threadId.value, + replaced.messageId, + replaced.settled ? 'settled-turn' : 'no-live-turn' + ) + replaced.transport.dispose() + } backgroundTurns.set(threadId.value, { messageId: activeTurnId.value, message: liveMessage, @@ -313,6 +377,7 @@ export const useAgentConversationStore = defineStore( !poppedHydratedCopy && hydratedMessageIds.has(entry.messageId) ) { + rememberDepartedTurn(threadId.value, entry.messageId, 'settled-turn') // The persisted, authoritative copy is already on screen (kept, via // the filter above) -- this entry's transport is now discarded for // good, so flush anything it is still holding rather than leaving it @@ -328,6 +393,7 @@ export const useAgentConversationStore = defineStore( const index = kept.push(entry.message) - 1 messages.value = kept if (entry.settled) { + rememberDepartedTurn(threadId.value, entry.messageId, 'settled-turn') // PM-1575: this settled turn is kept on screen but not reactivated -- // its transport is discarded for good right after this, same as the // hydrated-copy-dropped branch above, so flush anything it is still @@ -338,6 +404,7 @@ export const useAgentConversationStore = defineStore( } activeIndex.value = index activeTurnId.value = entry.messageId + activeTransportThreadId = threadId.value transport = entry.transport liveMessage = entry.message } @@ -345,6 +412,7 @@ export const useAgentConversationStore = defineStore( function settleBackgroundTurn(turnId: string): void { for (const [key, entry] of backgroundTurns) { if (entry.messageId !== turnId) continue + rememberDepartedTurn(key, entry.messageId, 'settled-turn') entry.transport.settle() // This turn is being dropped from the map here, unlike the // agent_message_done path in ingestBackgroundTurnEvent -- nothing @@ -356,15 +424,46 @@ export const useAgentConversationStore = defineStore( } function dropBackgroundTurns(): void { - for (const entry of backgroundTurns.values()) { + for (const [backgroundThreadId, entry] of backgroundTurns) { + rememberDepartedTurn( + backgroundThreadId, + entry.messageId, + entry.settled ? 'settled-turn' : 'no-live-turn' + ) entry.transport.settle() entry.transport.dispose() } backgroundTurns.clear() } + function departedTurnKey(threadId: string, messageId: string): string { + return `${threadId}\u0000${messageId}` + } + + function rememberDepartedTurn( + departedThreadId: string, + messageId: TurnId, + reason: 'no-live-turn' | 'settled-turn' + ): void { + const key = departedTurnKey(departedThreadId, messageId) + departedTurns.delete(key) + departedTurns.set(key, reason) + if (departedTurns.size <= MAX_DEPARTED_TURNS) return + const oldestKey = departedTurns.keys().next().value + if (oldestKey !== undefined) departedTurns.delete(oldestKey) + } + + function rememberDepartedActiveTurn( + reason: 'no-live-turn' | 'settled-turn' + ): void { + if (activeTransportThreadId === null || activeTurnId.value === null) + return + rememberDepartedTurn(activeTransportThreadId, activeTurnId.value, reason) + } + function clearActive(): void { transport = null + activeTransportThreadId = null liveMessage = null activeIndex.value = -1 activeTurnId.value = null @@ -404,10 +503,13 @@ export const useAgentConversationStore = defineStore( threadId.value = null hydratedMessageIds = new Set() hydratedAssistantTurnIds = new Set() + undeliverableAskReporter.reset() + departedTurns.clear() clearActive() } function hydrate(history: AgentMessages): void { + if (transport) rememberDepartedActiveTurn('no-live-turn') disposeActiveAndSettledTransports() clearActive() const transcript = normalizeAgentTranscript(history) @@ -425,11 +527,13 @@ export const useAgentConversationStore = defineStore( liveMessage = transcript.pending.message activeIndex.value = messages.value.indexOf(transcript.pending.message) activeTurnId.value = transcript.pending.messageId + activeTransportThreadId = threadId.value transport = createAgentEventTransport( transcript.pending.message, replaceActive, () => canvasSyncGate(), - () => canvasSyncOutcomeCount() + () => canvasSyncOutcomeCount(), + reportUndeliverableAskData ) } }