diff --git a/browser_tests/fixtures/agentTurnLockFixture.ts b/browser_tests/fixtures/agentTurnLockFixture.ts new file mode 100644 index 00000000000..e11f9f66f27 --- /dev/null +++ b/browser_tests/fixtures/agentTurnLockFixture.ts @@ -0,0 +1,340 @@ +import { expect, mergeTests } from '@playwright/test' +import type { Locator, Page, WebSocketRoute } from '@playwright/test' + +import type { + AgentCancelAccepted, + AgentError, + AgentMessage, + AgentTurnAccepted +} from '@comfyorg/ingest-types' +import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { agentTest } from '@e2e/fixtures/agentPanelFixture' +import { workflowSelectionTest } from '@e2e/fixtures/agentWorkflowSelectionFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { TestIds } from '@e2e/fixtures/selectors' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const THREAD_ID = 'b9d0a2a1-0f2c-4f1a-9a5e-6b0f4f2c1d77' +const TURN_ID = '2dd4f367-3399-4cb4-8127-547f531c289a' +const WORKFLOW_ID = 'a81718a4-02ae-41e6-ae85-000000000001' + +/** + * Verbatim from `services/agent/server/agent_handler.go`, which answers a post + * to a thread whose assistant row is still `streaming` with HTTP 409 and this + * body. The client renders it as `agent.sendFailed` + ': ' + this text. + */ +export const TURN_IN_PROGRESS_MESSAGE = + 'a turn is already in progress for this thread' + +const TURN_IN_PROGRESS: AgentError = { error: TURN_IN_PROGRESS_MESSAGE } + +const TURN_THINKING_TEXT = 'Wiring the audio output node.' +export const POST_RECONNECT_TEXT = 'Reconnected, and the graph is ready.' + +const TURN_THINKING_EVENT: AgentWsEvent = { + type: 'agent_thinking', + data: { delta: TURN_THINKING_TEXT, message_id: TURN_ID, thread_id: THREAD_ID } +} + +const TURN_TOOL_EVENT: AgentWsEvent = { + type: 'agent_tool_call', + data: { + tool_call_id: 'call-add-node', + tool_name: 'add_node', + status: 'success', + duration_ms: 1300, + message_id: TURN_ID, + thread_id: THREAD_ID + } +} + +export const POST_RECONNECT_EVENT: AgentWsEvent = { + type: 'agent_message_delta', + data: { + delta: POST_RECONNECT_TEXT, + message_id: TURN_ID, + thread_id: THREAD_ID + } +} + +export const TURN_DONE_EVENT: AgentWsEvent = { + type: 'agent_message_done', + data: { message_id: TURN_ID, thread_id: THREAD_ID } +} + +/** + * The server's half of a turn, modelled on the real single-active-turn guard: + * an assistant row goes `streaming` when a turn starts and only leaves that + * state when the turn completes, fails, or is cancelled. Dropping the client's + * socket does not touch it — that asymmetry is what these specs exercise. + * + * `transcript()` and the cancel route are not reached by the current specs. + * They are here so the fake stays a faithful server: a repair that re-hydrates + * on reconnect, or a spec that clicks Stop, needs both, and a half-modelled + * server would make such a fix look broken. + */ +class TurnLockServer { + private streaming = false + private prompt = '' + private rejected = 0 + private posts = 0 + + get turnIsStreaming(): boolean { + return this.streaming + } + + get rejectedPosts(): number { + return this.rejected + } + + /** Every post the server answered, accepted or rejected. */ + get postAttempts(): number { + return this.posts + } + + countPost(): void { + this.posts++ + } + + completeTurn(): void { + this.streaming = false + } + + transcript(): AgentMessage[] { + return [ + { + id: 'user-1', + thread_id: THREAD_ID, + turn_id: TURN_ID, + seq: 1, + role: 'user', + status: 'complete', + workflow_id: WORKFLOW_ID, + content: { text: this.prompt } + }, + { + id: TURN_ID, + thread_id: THREAD_ID, + turn_id: TURN_ID, + seq: 2, + role: 'assistant', + status: this.streaming ? 'streaming' : 'complete', + workflow_id: WORKFLOW_ID + } + ] + } + + startTurn(prompt: string): AgentTurnAccepted { + this.prompt = prompt + this.streaming = true + return { message_id: TURN_ID, thread_id: THREAD_ID } + } + + rejectPost(): AgentError { + this.rejected++ + return TURN_IN_PROGRESS + } +} + +async function routeTurnLock( + page: Page, + server: TurnLockServer +): Promise { + await page.route('**/api/agent/threads/*/messages', (route) => { + if (route.request().method() === 'GET') + return route.fulfill(jsonRoute(server.transcript())) + server.countPost() + if (server.turnIsStreaming) + return route.fulfill({ ...jsonRoute(server.rejectPost()), status: 409 }) + const request = zAgentPostMessageRequest.parse( + route.request().postDataJSON() + ) + return route.fulfill({ + ...jsonRoute(server.startTurn(request.content)), + status: 202 + }) + }) + + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => { + server.completeTurn() + const accepted: AgentCancelAccepted = { status: 'cancelling' } + return route.fulfill(jsonRoute(accepted)) + }) +} + +export class AgentTurnLockHarness { + public readonly panel: Locator + public readonly composer: Locator + public readonly sendButton: Locator + public readonly stopButton: Locator + public readonly workSummary: Locator + public readonly workingRow: Locator + public readonly userBubbles: Locator + private readonly agentPanel: AgentPanel + + constructor( + private readonly page: Page, + private readonly server: TurnLockServer, + private readonly finishSave: (success: boolean) => void, + private readonly savedPaths: () => number, + private readonly getWebSocket: () => Promise, + private readonly nextWebSocket: () => Promise + ) { + this.agentPanel = new AgentPanel(page) + this.panel = this.agentPanel.root + this.composer = this.panel.getByRole('textbox') + this.sendButton = this.panel.getByRole('button', { + name: enMessages.agent.send, + exact: true + }) + this.stopButton = this.panel.getByRole('button', { + name: enMessages.agent.stop, + exact: true + }) + // WorkSummary.vue renders three labels off the elapsed total: `worked` + // alone, `workedForSeconds`, or `workedForMinutes`. Anchoring on the + // shared `worked` stem matches all three, so the negative assertions on + // this locator cannot go vacuous when a turn is shorter or longer than the + // fixture's tool duration, or when the copy is reworded. + this.workSummary = this.panel.getByRole('button', { + name: new RegExp( + `^${enMessages.agent.worked.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}` + ) + }) + this.workingRow = this.panel.getByText(enMessages.agent.working, { + exact: true + }) + this.userBubbles = this.panel.getByTestId('user-message-bubble') + } + + rejectedPosts(): number { + return this.server.rejectedPosts + } + + postAttempts(): number { + return this.server.postAttempts + } + + /** Opens the panel on a blank workflow and points the composer at that tab. */ + async openOnBlankWorkflow(): Promise { + await expect( + this.page.getByTestId(TestIds.topbar.integratedTabBarActions) + ).toHaveAttribute('data-agent-gate-settled', 'true', { timeout: 15_000 }) + await this.agentPanel.open() + await this.page + .getByRole('button', { + name: enMessages.sideToolbar.newBlankWorkflow, + exact: true + }) + .click() + await expect(this.panel).toBeVisible() + // Not AgentPanel.selectWorkflow(): it asserts the picker label straight + // after the menu click, but under this fixture the label only settles once + // the pending workflow save is released below. + await this.agentPanel.workflowPicker.click() + await this.page + .getByRole('menuitemradio', { name: 'Unsaved Workflow', exact: true }) + .click() + await expect.poll(() => this.savedPaths()).toBe(1) + this.finishSave(true) + await expect(this.agentPanel.workflowPicker).toHaveText('Unsaved Workflow') + } + + /** Sends a prompt and streams it to the point where a user sees work happening. */ + async startTurn(prompt: string): Promise { + const live = await this.getWebSocket() + await this.composer.fill(prompt) + await this.sendButton.click() + // Stop is NOT an ack: Composer renders it from `isSending`, which + // useAgentSession sets before the POST (and before a prepare() race worth + // up to PREPARE_TIMEOUT_MS). The user bubble comes from recordUser(), one + // line above startTurn(), so it is the first signal that activeTurnId is + // set. Pushing a frame before that point gets it silently dropped by + // agentConversationStore.ingest, with no retry. + await expect(this.userBubbles).toHaveText([prompt]) + this.push(live, TURN_THINKING_EVENT) + await expect(this.panel.getByText(TURN_THINKING_TEXT)).toBeVisible() + this.push(live, TURN_TOOL_EVENT) + await expect(this.workingRow).toBeVisible() + } + + push(ws: WebSocketRoute, event: AgentWsEvent): void { + ws.send(JSON.stringify(event)) + } + + /** + * Replays the Web Audio work an audio preview does while a turn is live: + * open an AudioContext, `decodeAudioData` real WAV bytes, close it. This is + * the shape of `useWaveAudioPlayer.decodeAudioSource`, not a call into it — + * the product's `api.fetchApi` step and its waveform pass are not exercised, + * so this shows Web Audio alone is harmless rather than clearing the whole + * player. + */ + async decodeAudioLikeAPreview(): Promise { + await this.page.evaluate(async () => { + const frames = 800 + const bytes = new ArrayBuffer(44 + frames * 2) + const view = new DataView(bytes) + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) + view.setUint8(offset + i, text.charCodeAt(i)) + } + ascii(0, 'RIFF') + view.setUint32(4, 36 + frames * 2, true) + ascii(8, 'WAVEfmt ') + view.setUint32(16, 16, true) + view.setUint16(20, 1, true) + view.setUint16(22, 1, true) + view.setUint32(24, 8000, true) + view.setUint32(28, 16000, true) + view.setUint16(32, 2, true) + view.setUint16(34, 16, true) + ascii(36, 'data') + view.setUint32(40, frames * 2, true) + + const context = new AudioContext() + await context.decodeAudioData(bytes) + await context.close() + }) + } + + /** Drops the live socket and resolves with the one the client reconnects on. */ + async dropSocket(): Promise { + // Resolve the live route first: with no socket open yet both calls would + // queue on the same waiter and hand back the same route, so the close + // below would kill the one returned as the reconnect. + const live = await this.getWebSocket() + const reconnected = this.nextWebSocket() + await live.close() + return reconnected + } +} + +export const agentTurnLockTest = mergeTests( + agentTest, + workflowSelectionTest, + webSocketFixture +).extend<{ turnLock: AgentTurnLockHarness }>({ + turnLock: async ( + { page, workflowSelection, getWebSocket, nextWebSocket }, + use + ) => { + const server = new TurnLockServer() + await routeTurnLock(page, server) + await use( + new AgentTurnLockHarness( + page, + server, + workflowSelection.finishSave, + () => workflowSelection.savedPaths.length, + getWebSocket, + nextWebSocket + ) + ) + } +}) diff --git a/browser_tests/fixtures/ws.ts b/browser_tests/fixtures/ws.ts index b54525c8bd9..716e9272615 100644 --- a/browser_tests/fixtures/ws.ts +++ b/browser_tests/fixtures/ws.ts @@ -1,6 +1,16 @@ import { networkIsolationFixture as base } from '@e2e/fixtures/networkIsolationFixture' import type { WebSocketRoute } from '@playwright/test' +/** + * The `/ws` connections the page opens, in order. A reconnect routes a second + * time, so a test that drops the socket needs the route the client opened + * *after* the drop, not the dead one it already holds. + */ +interface WebSocketTracker { + current: () => WebSocketRoute | undefined + waitForNext: () => Promise +} + function createWebSocketRouteHandler( connectWebSocketToServer: boolean, onRouted: (ws: WebSocketRoute) => void @@ -20,28 +30,44 @@ function createWebSocketRouteHandler( export const webSocketFixture = base.extend<{ connectWebSocketToServer: boolean getWebSocket: () => Promise + nextWebSocket: () => Promise + webSocketTracker: WebSocketTracker }>({ connectWebSocketToServer: [true, { option: true }], - getWebSocket: [ + webSocketTracker: [ async ({ context, connectWebSocketToServer }, use) => { - let latest: WebSocketRoute | undefined - let resolve: ((ws: WebSocketRoute) => void) | undefined + let current: WebSocketRoute | undefined + const waiters = new Set<(ws: WebSocketRoute) => void>() await context.routeWebSocket( /\/ws/, createWebSocketRouteHandler(connectWebSocketToServer, (ws) => { - latest = ws - resolve?.(ws) + current = ws + const pending = [...waiters] + waiters.clear() + for (const resolve of pending) resolve(ws) }) ) - await use(() => { - if (latest) return Promise.resolve(latest) - return new Promise((r) => { - resolve = r - }) + await use({ + current: () => current, + waitForNext: () => + new Promise((resolve) => waiters.add(resolve)) }) }, { auto: true } - ] + ], + getWebSocket: async ({ webSocketTracker }, use) => { + await use(() => { + const open = webSocketTracker.current() + return open ? Promise.resolve(open) : webSocketTracker.waitForNext() + }) + }, + /** + * Resolves with the NEXT socket the page opens. Call it before closing the + * current one so the waiter is armed before the client reconnects. + */ + nextWebSocket: async ({ webSocketTracker }, use) => { + await use(() => webSocketTracker.waitForNext()) + } }) diff --git a/browser_tests/tests/agent/agentTurnSurvivesSocketDrop.spec.ts b/browser_tests/tests/agent/agentTurnSurvivesSocketDrop.spec.ts new file mode 100644 index 00000000000..981d75575a6 --- /dev/null +++ b/browser_tests/tests/agent/agentTurnSurvivesSocketDrop.spec.ts @@ -0,0 +1,156 @@ +import { expect } from '@playwright/test' + +import { + POST_RECONNECT_EVENT, + POST_RECONNECT_TEXT, + TURN_DONE_EVENT, + TURN_IN_PROGRESS_MESSAGE, + agentTurnLockTest as test +} from '@e2e/fixtures/agentTurnLockFixture' + +// PM-1199 / PM-1200. Reported after a long auto-mode run: the action stream +// stopped updating, the "Working..." row was replaced by a "Worked for 3m" +// summary as if the turn had finished, and the next message came back +// "Message failed to send: a turn is already in progress for this thread" — +// with no refresh and no tab close anywhere in the journey. +// +// The agent chat stream rides the shared ComfyUI websocket, so any transient +// close dispatches `reconnecting`. `useAgentSession.onStatus(false)` answers it +// with `abortActiveTurn()`, which settles the assistant message locally, while +// `onStatus(true)` only sets a flag — nothing re-attaches. The server never saw +// the socket go away, so its assistant row is still `streaming` and it keeps +// answering posts with 409. These specs close one socket and hold the client to +// the three things the user watched break. +// +// Deliberately not a refresh or a tab reopen: PM-1154 and PM-1043 cover those +// triggers. Here the page is never reloaded — only the socket blips. +test.describe.configure({ timeout: 120_000 }) +test.use({ connectWebSocketToServer: false }) + +test.describe( + 'Agent turn across a transient websocket reconnect', + { tag: ['@cloud', '@ui'] }, + () => { + const PROMPT = 'add an audio output node' + + test.beforeEach(async ({ turnLock }) => { + await turnLock.openOnBlankWorkflow() + await turnLock.startTurn(PROMPT) + }) + + test('keeps rendering the turn after the socket reconnects', async ({ + turnLock + }) => { + const reconnected = await turnLock.dropSocket() + + // The thread itself must survive the blip. A wipe here would be a + // different defect, and test.fail() below would swallow it. (The agent's + // narration is deliberately not asserted: settling the turn collapses it + // into the closed "Worked for ..." summary, which is the very thing the + // next test pins.) + await expect(turnLock.userBubbles).toHaveText([PROMPT]) + + // The server never stopped running this turn, so it keeps broadcasting the + // same message_id down the new socket. Sending it stays above the marker + // with the rest of the arrange: `ws.send()` throws on a dead route, and + // below test.fail() that throw would read as the expected failure without + // the assertion ever running. + turnLock.push(reconnected, POST_RECONNECT_EVENT) + + // `agentConversationStore.ingest` drops every one of those frames: + // `transport` is null after the abort, and `dropBackgroundTurns()` emptied + // the map it would otherwise fall back to. + test.fail() + await expect(turnLock.panel.getByText(POST_RECONNECT_TEXT)).toBeVisible() + }) + + test('keeps the turn marked as running after the socket reconnects', async ({ + turnLock + }) => { + await expect(turnLock.stopButton).toBeVisible() + await expect(turnLock.workSummary).toHaveCount(0) + + await turnLock.dropSocket() + + // `abortActiveTurn()` calls `transport.settle()`, which flips + // `message.streaming` to false. AgentMessage.vue swaps ActivityTrace for + // WorkSummary on exactly that flag, so the live rows collapse into the + // "Worked for ..." button the user saw, the Working... row disappears and + // Stop reverts to Send — while the server still owns the turn. + test.fail() + await expect(turnLock.stopButton).toBeVisible() + await expect(turnLock.workSummary).toHaveCount(0) + await expect(turnLock.workingRow).toBeVisible() + }) + + test('does not reject the next message after the socket reconnects', async ({ + turnLock + }) => { + await turnLock.dropSocket() + + // Preconditions stay above the marker so a broken composer or a lost + // prompt reads as a real failure rather than the expected one. Send + // visibility is deliberately not asserted here: Send replacing Stop is + // itself part of the defect. + await expect(turnLock.composer).toBeVisible() + await expect(turnLock.userBubbles).toHaveText([PROMPT]) + + // The abandoned turn puts Send back in front of the user, so the nudge + // reaches a thread the server still has locked and comes back 409 + // TURN_IN_PROGRESS. + // + // The nudge stays ABOVE test.fail(): body-level test.fail() only sets the + // expected status when it executes, so a throw up here is still an + // unexpected failure. Once the client re-attaches there is no Send + // button, and this click reports that by name in seconds instead of + // running out the file timeout four times over under CI retries. + await turnLock.composer.fill('are you still there?') + await turnLock.sendButton.click({ timeout: 10_000 }) + // Inequality, so a future client-side retry cannot fail this line in + // place of the alert assertion below. + await expect.poll(() => turnLock.postAttempts()).toBeGreaterThanOrEqual(2) + + test.fail() + await expect( + turnLock.panel + .getByRole('alert') + .filter({ hasText: TURN_IN_PROGRESS_MESSAGE }) + ).toHaveCount(0) + expect(turnLock.rejectedPosts()).toBe(0) + }) + + // Keeps the `workSummary` locator honest. Every other use of it above is a + // toHaveCount(0), which a locator that matched nothing would satisfy for + // free; this shows it does resolve once a turn ends. + test('summarises a turn that ends normally', async ({ + turnLock, + getWebSocket + }) => { + await expect(turnLock.workSummary).toHaveCount(0) + + turnLock.push(await getWebSocket(), TURN_DONE_EVENT) + + await expect(turnLock.workSummary).toBeVisible() + await expect(turnLock.sendButton).toBeVisible() + }) + + // The report mentions audio playback right before the disconnect. What + // shows it was incidental is really the three specs above, which reproduce + // every symptom with no audio anywhere; this only adds that Web Audio + // decoding beside a live turn leaves it alone. It does not call the + // product's player, so it does not clear the whole audio path. + test('keeps the turn running while audio decoding runs alongside it', async ({ + turnLock, + getWebSocket + }) => { + const ws = await getWebSocket() + await turnLock.decodeAudioLikeAPreview() + + await expect(turnLock.stopButton).toBeVisible() + await expect(turnLock.workSummary).toHaveCount(0) + + turnLock.push(ws, POST_RECONNECT_EVENT) + await expect(turnLock.panel.getByText(POST_RECONNECT_TEXT)).toBeVisible() + }) + } +) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index 0c9a3ce53cb..bb3660b096d 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -1,5 +1,5 @@ import type { AgentAdmissionError } from '@comfyorg/ingest-types' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { assert, beforeEach, describe, expect, it, vi } from 'vitest' import { nextTick } from 'vue' import { reportError } from '@/platform/telemetry/reportError' @@ -872,6 +872,92 @@ describe('useAgentSession (v1 composition root)', () => { expect(session.isStreaming.value).toBe(true) }) + // PM-1199 / PM-1200. (g) above pins today's behaviour: a live->down->live + // socket blip settles the turn and the session does nothing on the way back + // up. The server never learned the socket went away, so it keeps running the + // turn and keeps broadcasting the same message_id. These two say what the + // user needs instead, and are the lowest level that proves it — the browser + // spec `agentTurnSurvivesSocketDrop.spec.ts` covers the same defect through + // the real socket and the rendered panel. + // + // `it.fails` accepts a throw from anywhere in the body, so the arrange these + // two share cannot guard itself. (g) and (g2) directly above are unmarked and + // drive exactly that arrange — start, sendMessage, emit, the isStreaming + // precondition — so a fixture or setup regression reddens there instead of + // being absorbed here. Keep them unmarked for as long as these two exist. + // Backend invariant (cloud `newAssistantMessage` + the complete/fail writes): + // an assistant row carries content only once it goes terminal, so a live turn + // has nothing to re-hydrate from. Keep this row contentless — giving it text + // would let a repair that re-hydrates mid-turn satisfy these pins, when that + // repair blanks the reply on screen. + const streamingTurnRest = () => + fakeRest({ + getMessages: vi.fn( + async (): Promise => [ + historyRow(1, 'user', 'msg-1', 'go'), + { + ...historyRow(2, 'assistant', 'msg-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ] + ) + }) + + it.fails('(g3) KNOWN BUG: a reconnect leaves the turn running instead of settling it', async () => { + const rest = streamingTurnRest() + const { source, emit, status } = fakeEvents() + const session = useAgentSession({ rest, events: source }) + session.start() + status(true) + + await session.sendMessage('go') + emit(delta('msg-1', 'partial')) + expect(session.isStreaming.value).toBe(true) + + status(false) + status(true) + + // Generous on purpose, and matched in (g4). Under it.fails a waitFor that + // runs out its budget THROWS, and a throw is what marks the case green — so + // a tight budget here would quietly disarm the tripwire against any repair + // that debounces recovery after the socket flaps. + await vi.waitFor(() => expect(session.isStreaming.value).toBe(true), { + timeout: 2000 + }) + }) + + it.fails('(g4) KNOWN BUG: deltas that arrive after a reconnect still reach the turn', async () => { + const rest = streamingTurnRest() + const { source, emit, status } = fakeEvents() + const session = useAgentSession({ rest, events: source }) + session.start() + status(true) + + await session.sendMessage('go') + emit(delta('msg-1', 'partial')) + + status(false) + status(true) + emit(delta('msg-1', ' and the rest')) + + // Retried, because a re-hydrate repair reaches the reply through an async + // getMessages. Joined rather than part-by-part: a re-attach that opens a + // fresh text part on reconnect still shows the user the whole reply, and + // must count as fixed. + await vi.waitFor( + () => { + const assistant = session.entries.value.at(-1) + assert(assistant !== undefined && 'parts' in assistant) + const replyText = assistant.parts + .flatMap((part) => (part.type === 'text' ? [part.text] : [])) + .join('') + expect(replyText).toBe('partial and the rest') + }, + { timeout: 2000 } + ) + }) + it('(h) attachments pass through to the postMessage wire body', async () => { const rest = fakeRest() const { source } = fakeEvents()