Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions browser_tests/tests/agent/agentPanel.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,38 @@ test.describe('In-App Agent panel', { tag: '@cloud' }, () => {
})
})

test('keeps a failed prompt available to retry', async ({
agentPanel,
comfyPage
}) => {
// Regression: https://github.com/Comfy-Org/ComfyUI_frontend/pull/16628
await comfyPage.page.route(
'**/api/agent/threads/*/messages',
async (route) => {
if (route.request().method() !== 'POST') return route.fallback()
await route.fulfill({
status: 500,
body: 'Agent temporarily unavailable'
})
}
)
await agentPanel.open()
await agentPanel.selectWorkflow()

const prompt = 'Build a product photo workflow'
await agentPanel.composer.fill(prompt)
await agentPanel.sendButton.click()

await expect(agentPanel.root.getByTestId('user-message-bubble')).toHaveText(
prompt
)
await expect(
agentPanel.root.getByText(enMessages.agent.sendFailed)
).toBeVisible()
await expect(agentPanel.composer).toHaveText(prompt)
await expect(agentPanel.sendButton).toBeEnabled()
})

test.describe('diagnostic report', () => {
test.use({
permissions: ['clipboard-read', 'clipboard-write'],
Expand Down
6 changes: 6 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import type {
AgentPaywallCtaMetadata,
AgentPaywallShownMetadata,
AgentEntryButtonClickedMetadata,
AgentErrorMetadata,
AgentMessageSentMetadata,
AgentMessageFeedbackMetadata,
AgentNodeTaggedMetadata,
Expand Down Expand Up @@ -436,6 +437,11 @@ export class TelemetryRegistry implements TelemetryDispatcher {
this.dispatch((provider) => provider.trackAgentWorkflowApplied?.(metadata))
}

// fallow-ignore-next-line unused-class-member
trackAgentError(metadata: AgentErrorMetadata): void {
this.dispatch((provider) => provider.trackAgentError?.(metadata))
}

trackAgentStopClicked(metadata: AgentStopClickedMetadata): void {
this.dispatch((provider) => provider.trackAgentStopClicked?.(metadata))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import type {
AgentConsentResolvedMetadata,
AgentConsentShownMetadata,
AgentEntryButtonClickedMetadata,
AgentErrorMetadata,
AgentPaywallCtaMetadata,
AgentPaywallShownMetadata,
AgentMessageSentMetadata,
Expand Down Expand Up @@ -801,6 +802,11 @@ export class PostHogTelemetryProvider implements TelemetryProvider {
this.trackEvent(TelemetryEvents.AGENT_WORKFLOW_APPLIED, metadata)
}

// fallow-ignore-next-line unused-class-member
trackAgentError(metadata: AgentErrorMetadata): void {
this.trackEvent(TelemetryEvents.AGENT_ERROR, metadata)
}

trackAgentStopClicked(metadata: AgentStopClickedMetadata): void {
this.trackEvent(TelemetryEvents.AGENT_STOP_CLICKED, metadata)
}
Expand Down
19 changes: 19 additions & 0 deletions src/platform/telemetry/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,23 @@ export interface AgentThreadStartedMetadata extends Record<string, unknown> {
source: AgentThreadStartSource
}

export type AgentErrorClass =
| 'request_failed'
| 'malformed_stream_event'
| 'cancel_failed'
| 'history_load_failed'
| 'ask_answer_failed'
| 'thread_list_load_failed'
| 'workflow_open_failed'
export interface AgentErrorMetadata extends Record<string, unknown> {
error_class: AgentErrorClass
failure_stage: 'pre_acceptance' | 'post_acceptance'
retryable: boolean
turn_accepted: boolean
/** `none` is a failure the user was never shown. */
ui_treatment: 'inline_notice' | 'error_overlay' | 'toast' | 'none'
}

/**
* Widget (input/parameter) favorite toggle tracking metadata.
* Used to measure discoverability of the right side panel favoriting feature.
Expand Down Expand Up @@ -1604,6 +1621,7 @@ export interface TelemetryProvider {
metadata: AgentAttachButtonClickedMetadata
): void
trackAgentWorkflowApplied?(metadata: AgentWorkflowAppliedMetadata): void
trackAgentError?(metadata: AgentErrorMetadata): void
trackAgentStopClicked?(metadata: AgentStopClickedMetadata): void
trackAgentWorkflowBound?(metadata: AgentWorkflowBoundMetadata): void
trackAgentRunApprovalShown?(metadata: AgentRunApprovalShownMetadata): void
Expand Down Expand Up @@ -1802,6 +1820,7 @@ export const TelemetryEvents = {
AGENT_NODE_TAGGED: 'app:agent_node_tagged',
AGENT_ATTACH_BUTTON_CLICKED: 'app:agent_attach_button_clicked',
AGENT_WORKFLOW_APPLIED: 'app:agent_workflow_applied',
AGENT_ERROR: 'app:agent_error',
AGENT_STOP_CLICKED: 'app:agent_stop_clicked',
AGENT_WORKFLOW_BOUND: 'app:agent_workflow_bound',
AGENT_RUN_APPROVAL_SHOWN: 'app:agent_run_approval_shown',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1563,7 +1563,8 @@ describe('useWorkflowService', () => {
await vi.waitFor(() => expect(app.loadGraphData).toHaveBeenCalledTimes(2))
const staleOpen = service.openWorkflow(closing)
resolveReplacement?.()
await Promise.all([secondClose, staleOpen])
await expect(staleOpen).resolves.toBe(false)
await secondClose

expect(app.loadGraphData).toHaveBeenCalledTimes(2)
expect(workflowStore.openWorkflows).not.toContain(closing)
Expand Down
6 changes: 3 additions & 3 deletions src/platform/workflow/core/services/workflowService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,9 +378,9 @@ export const useWorkflowService = () => {
* Open a workflow in the current workspace
* @param workflow The workflow to open
* @param options The options for opening the workflow
* @returns false when the graph load reported failure (the error
* dialog was shown and the workflow never painted) or when the open
* was skipped because the workflow is mid-close; true otherwise
* @returns false when the graph load reported failure (the error dialog was
* shown and the workflow never painted) or when the open was skipped because
* the workflow is mid-close; true otherwise
*/
/**
* A failed replacement load leaves the shared root graph cleaned or
Expand Down
70 changes: 63 additions & 7 deletions src/workbench/extensions/agent/AgentPanelRoot.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1498,6 +1498,15 @@ describe('AgentPanelRoot session notices', () => {
type: 'agent_api_failed',
details: i18n.global.t('agent.malformedEvent')
})
expect(telemetry.trackAgentError).toHaveBeenCalledWith(
expect.objectContaining({
error_class: 'malformed_stream_event',
failure_stage: 'pre_acceptance',
retryable: false,
turn_accepted: false,
ui_treatment: 'error_overlay'
})
)
})
})

Expand Down Expand Up @@ -2980,7 +2989,7 @@ describe('AgentPanelRoot attach flow', () => {
detail: 'cat.png could not be uploaded'
})
)
expect(reportError).toHaveBeenCalledExactlyOnceWith(expect.any(Error), {
expect(reportError).toHaveBeenCalledWith(expect.any(Error), {
errorType: 'agent_attachment_upload_failed',
tags: {
failure_kind: 'caught_unexpected',
Expand Down Expand Up @@ -3394,6 +3403,7 @@ describe('AgentPanelRoot history', () => {

it('surfaces a thread-list failure via the host error modal', async () => {
executionErrors.showErrorOverlay.mockClear()
telemetry.trackAgentError.mockClear()
vi.stubGlobal(
'fetch',
vi.fn(async () => new Response('{}', { status: 500 }))
Expand All @@ -3407,6 +3417,15 @@ describe('AgentPanelRoot history', () => {
expect(executionErrors.lastPromptError).toMatchObject({
type: 'agent_api_failed'
})
expect(telemetry.trackAgentError).toHaveBeenCalledWith(
expect.objectContaining({
error_class: 'thread_list_load_failed',
failure_stage: 'pre_acceptance',
retryable: true,
turn_accepted: false,
ui_treatment: 'error_overlay'
})
)
expect(useAgentChatHistoryStore().sessions).toHaveLength(0)
})

Expand Down Expand Up @@ -5963,6 +5982,26 @@ describe('AgentPanelRoot workflow binding', () => {
)
})

it('agent_active_tab reports when a bound tab cannot be opened', async () => {
makeTab('wf-42')
mockMessagesEndpoint('wf-42')
await renderAndSend('work here')
telemetry.trackAgentError.mockClear()
vi.mocked(useWorkflowService()).openWorkflow.mockResolvedValueOnce(false)

ws.emit('agent_active_tab', { workflow_id: 'wf-42', thread_id: 'th-1' })

await vi.waitFor(() =>
expect(telemetry.trackAgentError).toHaveBeenCalledWith(
expect.objectContaining({
error_class: 'workflow_open_failed',
ui_treatment: 'toast'
})
)
)
expect(useTelemetry()!.trackAgentWorkflowApplied).not.toHaveBeenCalled()
})

// A browser tab closed without the SPA's own unbind() left 'wf-abandoned'
// bound to the default unsaved path in localStorage, and the thread pointer
// survived beside it. On the next boot the thread hydrates and names that
Expand Down Expand Up @@ -6415,14 +6454,13 @@ describe('AgentPanelRoot workflow binding', () => {
)
})

it('agent_active_tab closes the minted tab when opening it fails', async () => {
it('agent_active_tab reports when a minted tab cannot be opened', async () => {
makeTab('wf-42')
mockMessagesEndpoint('wf-42')

await renderAndSend('work here')
vi.mocked(useWorkflowService()).openWorkflow.mockRejectedValueOnce(
new Error('disk full')
)
telemetry.trackAgentError.mockClear()
vi.mocked(useWorkflowService()).openWorkflow.mockResolvedValueOnce(false)
vi.useFakeTimers()

ws.emit('agent_active_tab', {
Expand All @@ -6442,14 +6480,23 @@ describe('AgentPanelRoot workflow binding', () => {
workflowStore.getWorkflowByPath('workflows/Video test.json')
).toBeNull()
)
expect(telemetry.trackAgentError).toHaveBeenCalledWith(
expect.objectContaining({
error_class: 'workflow_open_failed',
ui_treatment: 'toast'
})
)
})

it('agent_active_tab closes the minted tab when opening it reports failure', async () => {
it('agent_active_tab closes the minted tab when opening it throws', async () => {
makeTab('wf-42')
mockMessagesEndpoint('wf-42')

await renderAndSend('work here')
vi.mocked(useWorkflowService()).openWorkflow.mockResolvedValueOnce(false)
telemetry.trackAgentError.mockClear()
vi.mocked(useWorkflowService()).openWorkflow.mockRejectedValueOnce(
new Error('open boom')
)
vi.useFakeTimers()

ws.emit('agent_active_tab', {
Expand All @@ -6473,6 +6520,15 @@ describe('AgentPanelRoot workflow binding', () => {
useAgentWorkflowTabBindingStore().tabPathFor('wf-77')
).toBeUndefined()
expect(useTelemetry()!.trackAgentWorkflowApplied).not.toHaveBeenCalled()
expect(telemetry.trackAgentError).toHaveBeenCalledWith(
expect.objectContaining({
error_class: 'workflow_open_failed',
failure_stage: 'post_acceptance',
retryable: false,
turn_accepted: true,
ui_treatment: 'error_overlay'
})
)
})

it('agent_active_tab strips dotfile prefixes hidden behind whitespace', async () => {
Expand Down
Loading
Loading