diff --git a/browser_tests/fixtures/agentTurnLockFixture.ts b/browser_tests/fixtures/agentTurnLockFixture.ts index 982b32ad4b5..ca8a09f61e6 100644 --- a/browser_tests/fixtures/agentTurnLockFixture.ts +++ b/browser_tests/fixtures/agentTurnLockFixture.ts @@ -17,6 +17,7 @@ import { escapeRegExp } from 'es-toolkit' import { z } from 'zod' import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { TURN_RECOVERY_DELAYS_MS } from '@/workbench/extensions/agent/services/agent/turnRecoverySchedule' import type { AgentWsEvent, zTurnInProgressError @@ -51,7 +52,8 @@ const TURN_IN_PROGRESS: TurnInProgressBody = { turn_id: TURN_ID } -const TURN_THINKING_TEXT = 'Wiring the audio output node.' +export const TURN_THINKING_TEXT = 'Wiring the audio output node.' +export const LATE_THINKING_TEXT = 'Back online, so I can finish the wiring.' export const POST_RECONNECT_TEXT = 'Reconnected, and the graph is ready.' export const PERSISTED_BEFORE_TOOL_TEXT = 'Prepared the graph. ' export const PERSISTED_AFTER_TOOL_TEXT = 'Audio output connected.' @@ -73,6 +75,11 @@ const TURN_TOOL_EVENT: AgentWsEvent = { } } +export const LATE_THINKING_EVENT: AgentWsEvent = { + type: 'agent_thinking', + data: { delta: LATE_THINKING_TEXT, message_id: TURN_ID, thread_id: THREAD_ID } +} + export const POST_RECONNECT_EVENT: AgentWsEvent = { type: 'agent_message_delta', data: { @@ -178,6 +185,7 @@ class TurnLockServer { private foreignPrompt: string | undefined private rejected = 0 private posts = 0 + private transcriptReads = 0 private readonly answered: string[][] = [] private pendingAsk: AgentPendingAsk | undefined private heldTranscript: @@ -203,6 +211,14 @@ class TurnLockServer { return this.posts } + get transcriptReadCount(): number { + return this.transcriptReads + } + + countTranscriptRead(): void { + this.transcriptReads++ + } + countPost(): void { this.posts++ } @@ -324,6 +340,10 @@ class TurnLockServer { tool_name: 'add_node', status: 'success' } + ], + thinking: [ + { text: TURN_THINKING_TEXT, tool_call_id: 'call-add-node' }, + { text: LATE_THINKING_TEXT } ] } }, @@ -438,6 +458,7 @@ async function routeTurnLock( const path = new URL(route.request().url()).pathname.split('/') const threadId = decodeURIComponent(path.at(-2) ?? '') await server.waitForTranscriptRelease(threadId) + server.countTranscriptRead() return route.fulfill(jsonRoute(server.transcript(threadId))) } server.countPost() @@ -512,9 +533,9 @@ export class AgentTurnLockHarness { public readonly liveProgressRow: Locator public readonly userBubbles: Locator public readonly turnInProgressNotice: Locator + public readonly agentPanel: AgentPanel private readonly entryButton: Locator private readonly dock: Locator - private readonly agentPanel: AgentPanel constructor( private readonly page: Page, @@ -580,6 +601,13 @@ export class AgentTurnLockHarness { return this.server.postAttempts } + async waitForRecoveryExhausted(): Promise { + const hydrationRead = 1 + await expect + .poll(() => this.server.transcriptReadCount, { timeout: 90_000 }) + .toBeGreaterThanOrEqual(TURN_RECOVERY_DELAYS_MS.length + hydrationRead) + } + /** Opens the panel on a blank workflow and points the composer at that tab. */ async openOnBlankWorkflow(): Promise { await expect( diff --git a/browser_tests/fixtures/components/AgentPanel.ts b/browser_tests/fixtures/components/AgentPanel.ts index 1f1348cdbb6..c3d04d11560 100644 --- a/browser_tests/fixtures/components/AgentPanel.ts +++ b/browser_tests/fixtures/components/AgentPanel.ts @@ -124,7 +124,14 @@ export class AgentPanel { name: 'Workflow JSON' }) this.workSummary = this.root.getByRole('button', { - name: new RegExp(`^${escapeRegExp(enMessages.agent.worked)}`) + name: new RegExp( + `^(${[ + enMessages.agent.worked, + enMessages.agent.workedNotConfirmedOnCanvas + ] + .map(escapeRegExp) + .join('|')})` + ) }) } diff --git a/browser_tests/tests/agent/agentThinkingAcrossMidTurnReload.spec.ts b/browser_tests/tests/agent/agentThinkingAcrossMidTurnReload.spec.ts new file mode 100644 index 00000000000..feda5605858 --- /dev/null +++ b/browser_tests/tests/agent/agentThinkingAcrossMidTurnReload.spec.ts @@ -0,0 +1,61 @@ +import { expect } from '@playwright/test' + +import { + LATE_THINKING_EVENT, + LATE_THINKING_TEXT, + PERSISTED_AFTER_TOOL_TEXT, + TURN_THINKING_TEXT, + agentTurnLockTest as test +} from '@e2e/fixtures/agentTurnLockFixture' + +test.describe.configure({ timeout: 180_000 }) +test.use({ connectWebSocketToServer: false }) + +test( + 'restores the thinking from before a mid-turn reload when the turn finishes', + { + tag: ['@cloud', '@ui'], + annotation: { + type: 'source', + description: 'https://linear.app/comfyorg/issue/FRONT-3659' + } + }, + async ({ turnLock, getAgentSocket }) => { + const prompt = 'add an audio output node' + + await test.step('narrate a round, then reload mid-turn', async () => { + await turnLock.openOnBlankWorkflow() + await turnLock.startTurn(prompt) + + await turnLock.agentPanel.reload() + await expect(turnLock.userBubbles).toHaveText([prompt]) + await expect(turnLock.stopButton).toBeVisible() + }) + + const socket = + await test.step('narrate another round after the reload', async () => { + await turnLock.waitForRecoveryExhausted() + + const live = await getAgentSocket() + turnLock.push(live, LATE_THINKING_EVENT) + await expect(turnLock.panel.getByText(LATE_THINKING_TEXT)).toBeVisible() + return live + }) + + await test.step('finishing the turn shows both rounds beside the tool call', async () => { + turnLock.finishTurn(socket) + + await expect(turnLock.sendButton).toBeVisible({ timeout: 30_000 }) + + await turnLock.agentPanel.openWorkSummary() + await expect(turnLock.agentPanel.activityRows).toHaveText([ + TURN_THINKING_TEXT, + /Add node/, + LATE_THINKING_TEXT + ]) + await expect( + turnLock.panel.getByText(PERSISTED_AFTER_TOOL_TEXT) + ).toBeVisible() + }) + } +) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index 7434bf0237b..49484914980 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -33,7 +33,8 @@ import type { import { zAgentAdmissionError, zAgentTurnAccepted, - zAgentWsEvent + zAgentWsEvent, + toTurnId } from '../../schemas/agentApiSchema' import { AgentApiError, @@ -819,6 +820,336 @@ describe('useAgentSession (v1 composition root)', () => { expect(conversation.activeTurnId).toBe('msg-1') }) + it('keeps socket parts and releases the composer when a settled snapshot read fails', async () => { + const readFailure = new AgentApiError('history unavailable', 503, undefined) + let rejectSettledRead: ((error: AgentApiError) => void) | undefined + const getMessages = vi + .fn() + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ]) + .mockImplementationOnce(hangingGetMessages) + .mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectSettledRead = reject + }) + ) + .mockRejectedValue(readFailure) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + + expect(session.isStreaming.value).toBe(false) + expect(session.entries.value.at(-1)).toMatchObject({ + role: 'assistant', + parts: [ + { + type: 'thinking', + text: 'live after reload', + state: 'done' + } + ] + }) + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(3)) + assert.exists(rejectSettledRead) + vi.useFakeTimers() + try { + rejectSettledRead(readFailure) + await vi.advanceTimersByTimeAsync(40_000) + } finally { + vi.useRealTimers() + } + + expect(getMessages).toHaveBeenCalledTimes(8) + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith(readFailure, { + surface: 'agent', + errorType: 'failure_recovering_agent_turn' + }) + expect(session.entries.value.at(-1)).toMatchObject({ + role: 'assistant', + parts: [ + { + type: 'thinking', + text: 'live after reload', + state: 'done' + } + ] + }) + + conversation.reconcileSettledTurn( + { threadId: 'th-1', messageId: toTurnId('msg-1') }, + [{ type: 'text', text: 'a later read', state: 'done' }] + ) + expect(session.entries.value.at(-1)).toMatchObject({ + parts: [{ type: 'thinking', text: 'live after reload' }] + }) + }) + + it('does not retry a settled snapshot read the server refused', async () => { + const refused = new AgentApiError('unauthorized', 401, undefined) + const getMessages = vi + .fn() + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ]) + .mockImplementationOnce(hangingGetMessages) + .mockRejectedValue(refused) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + vi.useFakeTimers() + try { + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + await vi.advanceTimersByTimeAsync(40_000) + } finally { + vi.useRealTimers() + } + + expect(getMessages).toHaveBeenCalledTimes(3) + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith(refused, { + surface: 'agent', + errorType: 'failure_recovering_agent_turn' + }) + }) + + it('reports a settled snapshot read that is still failing when its deadline passes', async () => { + const slowFailure = new AgentApiError('gateway timeout', 504, undefined) + const getMessages = vi + .fn() + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ]) + .mockImplementationOnce(hangingGetMessages) + .mockImplementation( + () => + new Promise((_resolve, reject) => { + setTimeout(() => reject(slowFailure), 25_000) + }) + ) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + vi.useFakeTimers() + try { + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + await vi.advanceTimersByTimeAsync(90_000) + } finally { + vi.useRealTimers() + } + + expect(reportError).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledWith(slowFailure, { + surface: 'agent', + errorType: 'failure_recovering_agent_turn' + }) + }) + + it('reads a settled snapshot turn again after a transient read failure', async () => { + const getMessages = vi + .fn() + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ]) + .mockImplementationOnce(hangingGetMessages) + .mockRejectedValueOnce( + new AgentApiError('history unavailable', 503, undefined) + ) + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', 'Added it.', 'msg-1'), + content: { + text: 'Added it.', + thinking: [{ text: 'before the reload' }] + } + } + ]) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + + await vi.waitFor( + () => + expect(session.entries.value.at(-1)).toMatchObject({ + role: 'assistant', + parts: [{ type: 'thinking', text: 'before the reload' }, {}] + }), + { timeout: 5000 } + ) + expect(reportError).not.toHaveBeenCalled() + }) + + it('reads a settled snapshot turn again while its row still reads as streaming', async () => { + const streamingRow = { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' as const + } + const getMessages = vi + .fn() + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + streamingRow + ]) + .mockImplementationOnce(hangingGetMessages) + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + streamingRow + ]) + .mockResolvedValueOnce([ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', 'Added it.', 'msg-1'), + content: { + text: 'Added it.', + thinking: [{ text: 'before the reload' }] + } + } + ]) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + + await vi.waitFor( + () => { + const last = session.entries.value.at(-1) + expect( + last?.role === 'assistant' + ? last.parts.flatMap((part) => + part.type === 'thinking' ? [part.text] : [] + ) + : [] + ).toContain('before the reload') + }, + { timeout: 5000 } + ) + }) + + it('honors a hydrate deferred behind an in-flight settled snapshot read', async () => { + const restored: AgentMessages = [ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', '', 'msg-1'), + content: {}, + status: 'streaming' + } + ] + const finished: AgentMessages = [ + historyRow(1, 'user', 'turn-1', 'add an audio output node'), + { + ...historyRow(2, 'assistant', 'turn-1', 'Added it.', 'msg-1'), + content: { + text: 'Added it.', + thinking: [{ text: 'before the reload' }] + } + } + ] + let settleRead: ((history: AgentMessages) => void) | undefined + const getMessages = vi + .fn() + .mockResolvedValueOnce(restored) + .mockImplementationOnce(hangingGetMessages) + .mockImplementationOnce( + () => + new Promise((resolve) => { + settleRead = resolve + }) + ) + .mockResolvedValueOnce(restored) + .mockResolvedValue(finished) + const conversation = useAgentConversationStore() + conversation.setThreadId('th-1') + const { source, emit } = fakeEvents() + const session = useAgentSession({ + rest: fakeRest({ getMessages }), + events: source + }) + session.start() + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(2)) + + emit(thinking('msg-1', 'live after reload')) + emit(done('msg-1')) + await vi.waitFor(() => expect(getMessages).toHaveBeenCalledTimes(3)) + + session.newChat('new_chat_button') + void session.loadThread('th-1') + await vi.waitFor(() => expect(session.isStreaming.value).toBe(true)) + + assert.exists(settleRead) + settleRead(finished) + + await vi.waitFor(() => expect(session.isStreaming.value).toBe(false), { + timeout: 2000 + }) + expect(conversation.activeTurnId).toBeNull() + }) + it('re-attaches to the running turn when a send is refused with TURN_IN_PROGRESS', async () => { const conversation = useAgentConversationStore() const postMessage = vi diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index da1a88ed9eb..70ee094920e 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -63,6 +63,7 @@ import { useAgentWorkflowTabBindingStore } from '../../stores/agent/agentWorkflo import type { ComposerAttachment } from '../../types/composerAttachment' import type { WorkflowReference } from '../../types/workflowReference' import { serializeWorkflowReferences } from '../../utils/workflowReferenceText' +import { TURN_RECOVERY_DELAYS_MS } from '../../services/agent/turnRecoverySchedule' export interface AgentEventSource { subscribe(listener: (raw: unknown) => void): () => void @@ -185,12 +186,6 @@ const HYDRATION_MAILBOX_MS = 5 * 60_000 const MAX_HYDRATION_EVENTS = 256 const MAX_HYDRATION_MAILBOXES = 32 -/** - * After a reconnect the server may still be finishing the turn, and its - * terminal event may or may not reach the new socket. Poll the persisted row - * with backoff; once the schedule is exhausted the socket alone is trusted. - */ -const TURN_RECOVERY_DELAYS_MS = [0, 1000, 2000, 4000, 8000, 16000] /** Upper bound on one recovery job, including any history fetch still in flight. */ const TURN_RECOVERY_DEADLINE_MS = 60_000 @@ -198,7 +193,7 @@ type TurnOutcome = | { kind: 'terminal'; parts: AssistantMessage['parts'] | undefined } | { kind: 'thread-missing' } | { kind: 'streaming' } - | { kind: 'error'; message: string } + | { kind: 'error'; cause: unknown } function isTerminalTurnStatus( status: AgentMessages[number]['status'] @@ -259,10 +254,7 @@ function recoveryKey(turn: LiveTurn): string { function turnOutcomeFromError(error: unknown): TurnOutcome { if (error instanceof AgentApiError && error.status === 404) return { kind: 'thread-missing' } - return { - kind: 'error', - message: error instanceof Error ? error.message : String(error) - } + return { kind: 'error', cause: error } } const NON_RETRYABLE_REQUEST_STATUSES = new Set([ @@ -580,6 +572,7 @@ export function useAgentSession(deps: AgentSessionDeps) { let unsubscribe: (() => void) | null = null let unsubscribeStatus: (() => void) | null = null + let unsubscribeSnapshotSettled: (() => void) | null = null let ownedGeneration = 0 let connection: SocketConnection = 'initial' const recoveringTurns = new Map() @@ -641,6 +634,9 @@ export function useAgentSession(deps: AgentSessionDeps) { initializeWorkflowContext(initialThreadId !== null) unsubscribe = events.subscribe(onRaw) if (events.onStatus) unsubscribeStatus = events.onStatus(onStatus) + unsubscribeSnapshotSettled = conversationStore.onSnapshotTurnSettled( + (turn) => void reconcileSettledSnapshotTurn(turn) + ).off if (!restore) return if (initialThreadId === null) { onThreadActivated?.(null) @@ -924,8 +920,10 @@ export function useAgentSession(deps: AgentSessionDeps) { readyThreadId.value = null unsubscribe?.() unsubscribeStatus?.() + unsubscribeSnapshotSettled?.() unsubscribe = null unsubscribeStatus = null + unsubscribeSnapshotSettled = null for (const recovery of recoveringTurns.values()) recovery.abort() recoveringTurns.clear() const stoppedGeneration = ownedGeneration @@ -1889,6 +1887,97 @@ export function useAgentSession(deps: AgentSessionDeps) { } } + function isRetryableSettledRead(outcome: TurnOutcome): boolean { + if (outcome.kind === 'streaming') return true + return ( + outcome.kind === 'error' && + isRetryableRequestFailure(outcome.cause, false) + ) + } + + async function fetchSettledTurnOutcome( + turn: LiveTurn, + generation: number, + signal: AbortSignal, + deadlineReason: Error + ): Promise { + let outcome: TurnOutcome | undefined + try { + outcome = await fetchTurnOutcome(turn, signal) + for (const ms of TURN_RECOVERY_DELAYS_MS.slice(1)) { + if ( + !isRetryableSettledRead(outcome) || + generation !== sessionGeneration + ) + return outcome + await delay(ms, { signal }) + outcome = await fetchTurnOutcome(turn, signal) + } + return outcome + } catch (error) { + if (signal.reason === deadlineReason && outcome?.kind === 'error') + return outcome + throw error + } + } + + function reportSettledReadFailure(cause: unknown): void { + reportError(cause, { + surface: 'agent', + errorType: 'failure_recovering_agent_turn' + }) + } + + function applySettledRead(turn: LiveTurn, outcome: TurnOutcome): void { + if (outcome.kind === 'error') reportSettledReadFailure(outcome.cause) + else if (outcome.kind === 'terminal') + conversationStore.reconcileSettledTurn(turn, outcome.parts) + } + + function releaseSettledRead( + key: string, + recovery: AbortController, + turn: LiveTurn, + generation: number + ): void { + if (recoveringTurns.get(key) !== recovery) return + recoveringTurns.delete(key) + if (generation === sessionGeneration) + conversationStore.reconcileSettledTurn(turn, undefined) + if (isTurnLive(turn, ownedGeneration)) void reconcileTurn(turn) + } + + async function reconcileSettledSnapshotTurn(turn: LiveTurn): Promise { + const generation = ownedGeneration + const key = recoveryKey(turn) + recoveringTurns.get(key)?.abort() + const recovery = new AbortController() + recoveringTurns.set(key, recovery) + const deadlineReason = new Error('settled read deadline') + const deadline = setTimeout( + () => recovery.abort(deadlineReason), + TURN_RECOVERY_DEADLINE_MS + ) + try { + const outcome = await fetchSettledTurnOutcome( + turn, + generation, + recovery.signal, + deadlineReason + ) + if ( + generation === sessionGeneration && + recoveringTurns.get(key) === recovery + ) + applySettledRead(turn, outcome) + } catch (error) { + if (!recovery.signal.aborted) reportSettledReadFailure(error) + } finally { + clearTimeout(deadline) + releaseSettledRead(key, recovery, turn, generation) + } + } + async function recoverTurn( turn: LiveTurn, generation: number, diff --git a/src/workbench/extensions/agent/services/agent/turnRecoverySchedule.ts b/src/workbench/extensions/agent/services/agent/turnRecoverySchedule.ts new file mode 100644 index 00000000000..574d6e1d0f4 --- /dev/null +++ b/src/workbench/extensions/agent/services/agent/turnRecoverySchedule.ts @@ -0,0 +1 @@ +export const TURN_RECOVERY_DELAYS_MS = [0, 1000, 2000, 4000, 8000, 16000] diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts index 5aea3138bfb..e661aa983a9 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.test.ts @@ -8,6 +8,7 @@ import type { TurnId } from '@/workbench/extensions/agent/schemas/agentApiSchema' import { + toTurnId, zAgentMessages, zAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' @@ -155,6 +156,8 @@ const activeTab = ( data: { workflow_id: workflowId, message_id: id, thread_id: threadId } }) +type ConversationStore = ReturnType + const tabLinkIds = (store: ReturnType) => store.messages.flatMap((m) => m.parts.flatMap((p) => (p.type === 'tabLink' ? [p.workflowId] : [])) @@ -1955,6 +1958,406 @@ describe('useAgentConversationStore', () => { } ) + describe('reconciling a snapshot-restored turn settled on the socket', () => { + const SNAPSHOT_ROW_ID = toTurnId('assistant-row-7') + const SNAPSHOT_TURN_ID = toTurnId('turn-7') + const liveSnapshot = (): AgentMessages => + zAgentMessages.parse([ + historyRow(1, 'user', SNAPSHOT_TURN_ID, 'add a loader', 'user-row-1'), + { + id: SNAPSHOT_ROW_ID, + thread_id: 'th', + seq: 2, + role: 'assistant', + status: 'streaming', + turn_id: SNAPSHOT_TURN_ID + } + ]) + const storedTool = (): AssistantMessage['parts'][number] => ({ + type: 'tool', + callId: 'call-add_node', + name: 'add_node', + state: 'done', + ok: true + }) + const storedParts = (): AssistantMessage['parts'] => [ + { type: 'thinking', text: 'before the reload', state: 'done' }, + storedTool(), + { type: 'thinking', text: 'after the reload', state: 'done' }, + { type: 'text', text: 'Added it.', state: 'done' } + ] + const reconciledParts = ( + store: ConversationStore + ): AssistantMessage['parts'] | undefined => + store.messages.find((message) => message.id === SNAPSHOT_TURN_ID)?.parts + const restoredAndSettled = () => { + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest(thinking(SNAPSHOT_ROW_ID, 'after the reload')) + store.ingest(done(SNAPSHOT_ROW_ID)) + return { store, settled } + } + + it('announces the turn and folds the stored row in', () => { + const { store, settled } = restoredAndSettled() + expect(settled).toEqual([SNAPSHOT_ROW_ID]) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + const message = store.messages.find((m) => m.id === SNAPSHOT_TURN_ID) + expect(message?.streaming).toBe(false) + expect(message?.parts).toEqual(storedParts()) + }) + + it('reconciles once, so a second read cannot re-apply the row', () => { + const { store } = restoredAndSettled() + const turn = { threadId: 'th', messageId: SNAPSHOT_ROW_ID } + store.reconcileSettledTurn(turn, storedParts()) + + store.reconcileSettledTurn(turn, [ + { type: 'text', text: 'a later read', state: 'done' } + ]) + + expect( + store.messages.find((m) => m.id === SNAPSHOT_TURN_ID)?.parts + ).toEqual(storedParts()) + }) + + it.for([ + { + reason: 'is longer than the stored copy', + live: 'after the reload continues' + }, + { reason: 'is a suffix of the stored copy', live: ' reload' }, + { + reason: 'is missing from the stored row', + live: 'a round the row dropped' + }, + { reason: 'repeats an earlier stored round', live: 'before the reload' } + ])( + 'renders the stored thinking when the live round $reason', + ({ live }) => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest(thinking(SNAPSHOT_ROW_ID, live)) + store.ingest(done(SNAPSHOT_ROW_ID)) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + expect(reconciledParts(store)).toEqual(storedParts()) + } + ) + + it('renders the stored thinking when the row keys the live call differently', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest(thinking(SNAPSHOT_ROW_ID, 'after the reload, partial')) + store.ingest(toolCall(SNAPSHOT_ROW_ID, 'save', 'success')) + store.ingest(done(SNAPSHOT_ROW_ID)) + const persisted: AssistantMessage['parts'] = [ + { type: 'thinking', text: 'before the reload', state: 'done' }, + storedTool(), + { type: 'thinking', text: 'after the reload', state: 'done' }, + { + type: 'tool', + callId: 'row-fallback-id', + name: 'save', + state: 'done', + ok: true + }, + { type: 'text', text: 'Saved it.', state: 'done' } + ] + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + persisted + ) + + expect(reconciledParts(store)).toEqual(persisted) + }) + + it('keeps live thinking when the stored row holds no thinking at all', () => { + const { store } = restoredAndSettled() + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + [storedTool(), { type: 'text', text: 'Added it.', state: 'done' }] + ) + + expect(reconciledParts(store)).toEqual([ + storedTool(), + { type: 'thinking', text: 'after the reload', state: 'done' }, + { type: 'text', text: 'Added it.', state: 'done' } + ]) + }) + + it('places a post-reload tab link after the stored pre-reload tool', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest(activeTab('wf-1', SNAPSHOT_ROW_ID)) + store.ingest(thinking(SNAPSHOT_ROW_ID, 'after the reload')) + store.ingest(done(SNAPSHOT_ROW_ID)) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + expect(reconciledParts(store)?.map((part) => part.type)).toEqual([ + 'thinking', + 'tool', + 'thinking', + 'tabLink', + 'text' + ]) + }) + + it('keeps a live canvas confirmation for a call the stored row keys by row id', () => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const store = useAgentConversationStore() + store.setCanvasSyncGate(() => 'awaiting') + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(done(SNAPSHOT_ROW_ID)) + store.notifyCanvasCaughtUp(liveCanvasEvidence(workflowId, ['op-add'])) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + [ + { + type: 'tool', + callId: 'row-fallback-id', + name: 'add_node', + state: 'done', + ok: true, + canvasConfirmed: false + }, + { type: 'text', text: 'Added it.', state: 'done' } + ] + ) + + expect( + reconciledParts(store)?.find((part) => part.type === 'tool') + ).toMatchObject({ callId: 'row-fallback-id', canvasConfirmed: true }) + }) + + it('lets stored thinking win when the recovery read settles the turn', () => { + const store = useAgentConversationStore() + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest(thinking(SNAPSHOT_ROW_ID, 'a round the row dropped')) + + store.settleTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + expect(reconciledParts(store)).toEqual(storedParts()) + }) + + it('waits for a held tool call to clear before announcing the turn', () => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setCanvasSyncGate(() => 'awaiting') + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(done(SNAPSHOT_ROW_ID)) + expect(settled).toEqual([]) + + store.notifyCanvasCaughtUp(liveCanvasEvidence(workflowId, ['op-add'])) + expect(settled).toEqual([SNAPSHOT_ROW_ID]) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts().map((part) => + part.type === 'tool' ? { ...part, canvasConfirmed: false } : part + ) + ) + + expect( + store.messages + .find((message) => message.id === SNAPSHOT_TURN_ID) + ?.parts.find((part) => part.type === 'tool') + ).toMatchObject({ + callId: 'call-add_node', + ok: true, + canvasConfirmed: true + }) + }) + + it.for([ + { through: 'reset', leave: (store: ConversationStore) => store.reset() }, + { + through: 'hydrate', + leave: (store: ConversationStore) => store.hydrate([]) + } + ])( + 'does not announce a held turn when the thread is left through $through', + ({ leave }) => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setCanvasSyncGate(() => 'awaiting') + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(done(SNAPSHOT_ROW_ID)) + + leave(store) + + expect(settled).toEqual([]) + } + ) + + it('keeps the reconciled row when a late ask retirement follows a stale held tool call', () => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setCanvasSyncGate(() => 'awaiting') + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(runApproval(SNAPSHOT_ROW_ID, 'turn-7:call-1')) + store.ingest(done(SNAPSHOT_ROW_ID)) + expect(settled).toEqual([]) + + vi.advanceTimersByTime(30_000) + expect(settled).toEqual([SNAPSHOT_ROW_ID]) + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + store.retireAsk('turn-7:call-1', { status: 'retired' }) + + expect(reconciledParts(store)).toEqual(storedParts()) + }) + + it('keeps a re-registered held turn for its own read when an earlier read lands late', () => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const { store, settled } = restoredAndSettled() + const earlierRead = { threadId: 'th', messageId: SNAPSHOT_ROW_ID } + store.setCanvasSyncGate(() => 'awaiting') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(done(SNAPSHOT_ROW_ID)) + expect(settled).toEqual([SNAPSHOT_ROW_ID]) + + store.reconcileSettledTurn(earlierRead, undefined) + store.notifyCanvasCaughtUp(liveCanvasEvidence(workflowId, ['op-add'])) + expect(settled).toEqual([SNAPSHOT_ROW_ID, SNAPSHOT_ROW_ID]) + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + expect(reconciledParts(store)?.map((part) => part.type)).toEqual([ + 'thinking', + 'tool', + 'thinking', + 'text' + ]) + }) + + it('announces a held turn only once when the canvas reports twice', () => { + const workflowId = 'f2b4c6d8-1a3e-4b5c-9d7e-0f1a2b3c4d5e' + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setCanvasSyncGate(() => 'awaiting') + store.setThreadId('th') + store.hydrate(liveSnapshot()) + store.ingest( + toolCall(SNAPSHOT_ROW_ID, 'add_node', 'success', { + workflow_id: workflowId, + op_ids: ['op-add'] + }) + ) + store.ingest(done(SNAPSHOT_ROW_ID)) + + store.notifyCanvasCaughtUp(liveCanvasEvidence(workflowId, ['op-add'])) + store.notifyCanvasCaughtUp(liveCanvasEvidence(workflowId, ['op-add'])) + + expect(settled).toEqual([SNAPSHOT_ROW_ID]) + }) + + it('leaves a turn this client started alone', () => { + const store = useAgentConversationStore() + const settled: TurnId[] = [] + store.onSnapshotTurnSettled((turn) => settled.push(turn.messageId)) + store.setThreadId('th') + store.startTurn(T1) + store.ingest(delta('t1', 'local reply')) + store.ingest(done('t1')) + expect(settled).toEqual([]) + + store.reconcileSettledTurn({ threadId: 'th', messageId: T1 }, [ + { type: 'text', text: 'persisted reply', state: 'done' } + ]) + + expect(messageTexts(store, T1)).toEqual(['local reply']) + }) + + it('drops the registration when the thread is left', () => { + const { store } = restoredAndSettled() + store.reset() + store.setThreadId('th') + store.startTurn(SNAPSHOT_TURN_ID) + + store.reconcileSettledTurn( + { threadId: 'th', messageId: SNAPSHOT_ROW_ID }, + storedParts() + ) + + expect( + store.messages.find((m) => m.id === SNAPSHOT_TURN_ID)?.parts + ).toEqual([]) + }) + }) + it('keeps a local tab link between persisted text and tool parts', () => { const store = useAgentConversationStore() store.setThreadId('th') diff --git a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts index bd220485023..8820a8dbf3c 100644 --- a/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts +++ b/src/workbench/extensions/agent/stores/agent/agentConversationStore.ts @@ -1,4 +1,5 @@ import type { AgentAnswerRequest } from '@comfyorg/ingest-types' +import { createEventHook } from '@vueuse/core' import { defineStore } from 'pinia' import { computed, ref, shallowRef } from 'vue' @@ -93,6 +94,13 @@ export interface LiveTurn { messageId: TurnId } +interface UnreconciledSnapshotTurn { + turn: LiveTurn + messageId: string + transport: AgentEventTransport + announced: boolean +} + interface AnchoredLocalPart { part: AssistantMessage['parts'][number] toolCount: number @@ -203,9 +211,88 @@ function interleaveLocalParts( return mergedParts } +function toolCallIdsOf(parts: AssistantMessage['parts']): string[] { + return parts.flatMap((part) => (part.type === 'tool' ? [part.callId] : [])) +} + +function liveToolOffset( + liveParts: AssistantMessage['parts'], + persistedParts: AssistantMessage['parts'] +): number { + const live = toolCallIdsOf(liveParts) + const persisted = toolCallIdsOf(persistedParts) + if (live.length === 0) return persisted.length + for (const [index, callId] of live.entries()) { + const at = persisted.indexOf(callId) + if (at >= index) return at - index + } + return Math.max(0, persisted.length - live.length) +} + +function storedCallIdOf( + liveParts: AssistantMessage['parts'], + persistedParts: AssistantMessage['parts'], + offset: number +): (callId: string) => string { + const liveCallIds = toolCallIdsOf(liveParts) + const persistedCallIds = toolCallIdsOf(persistedParts) + const persisted = new Set(persistedCallIds) + const live = new Set(liveCallIds) + return (callId) => { + if (persisted.has(callId)) return callId + const position = liveCallIds.indexOf(callId) + offset + const atPosition = + position < persistedCallIds.length + ? persistedCallIds.at(position) + : undefined + return atPosition !== undefined && !live.has(atPosition) + ? atPosition + : callId + } +} + +function preserveCanvasConfirmation( + liveParts: AssistantMessage['parts'], + persistedParts: AssistantMessage['parts'], + storedCallId: (callId: string) => string +): AssistantMessage['parts'] { + const confirmedCallIds = new Set( + liveParts.flatMap((part) => + part.type === 'tool' && part.canvasConfirmed === true + ? [storedCallId(part.callId)] + : [] + ) + ) + return persistedParts.map((part) => + part.type === 'tool' && + part.ok === true && + confirmedCallIds.has(part.callId) + ? { ...part, canvasConfirmed: true } + : part + ) +} + +function finishSnapshotTurn( + message: AssistantMessage, + persistedParts: AssistantMessage['parts'] +): void { + const liveParts = message.parts + const offset = liveToolOffset(liveParts, persistedParts) + finishWithPersistedParts( + message, + preserveCanvasConfirmation( + liveParts, + persistedParts, + storedCallIdOf(liveParts, persistedParts, offset) + ), + offset + ) +} + function finishWithPersistedParts( message: AssistantMessage, - persistedParts: AssistantMessage['parts'] | undefined + persistedParts: AssistantMessage['parts'] | undefined, + localToolOffset = 0 ): void { if (persistedParts === undefined) { message.parts = message.parts.filter((part) => !isOpenAskPart(part)) @@ -227,9 +314,12 @@ function finishWithPersistedParts( ) message.parts = interleaveLocalParts( persistedParts, - anchorLocalParts(message.parts).filter( - ({ part }) => !persistedHasThinking || part.type !== 'thinking' - ) + anchorLocalParts(message.parts) + .filter(({ part }) => !persistedHasThinking || part.type !== 'thinking') + .map((anchor) => ({ + ...anchor, + toolCount: anchor.toolCount + localToolOffset + })) ) } @@ -276,6 +366,11 @@ export const useAgentConversationStore = defineStore( // has nothing left pending. const settledActiveTransports = new Set() const backgroundTurns = new Map() + const unreconciledSnapshotTurns = new Map< + TurnId, + UnreconciledSnapshotTurn + >() + const snapshotTurnSettled = createEventHook() let hydratedMessageIds = new Set() let hydratedTurnIds = new Map() let hydratedAssistantTurnIds = new Set() @@ -327,6 +422,16 @@ export const useAgentConversationStore = defineStore( if (index >= 0) messages.value[index] = message } + function publishSnapshotMessage( + turnId: TurnId, + message: AssistantMessage + ): void { + replaceActive(message) + const unreconciled = unreconciledSnapshotTurns.get(turnId) + if (unreconciled !== undefined) + announceReconcilableSnapshotTurn(unreconciled) + } + function rememberAttachmentNames(attachments: UserAttachment[]): void { const currentThreadId = threadId.value if (currentThreadId === null) return @@ -612,9 +717,59 @@ export const useAgentConversationStore = defineStore( function settleActiveTurn(activeTransport: AgentEventTransport): void { rememberDepartedActiveTurn('settled-turn') activeTransport.settle() + const restoredFromSnapshot = recordUnreconciledSnapshotTurn() if (activeTransport.hasPendingCanvasSync()) settledActiveTransports.add(activeTransport) clearActive() + if (restoredFromSnapshot) + announceReconcilableSnapshotTurn(restoredFromSnapshot) + } + + function recordUnreconciledSnapshotTurn(): + | UnreconciledSnapshotTurn + | undefined { + const slot = activeSlot.value + if (slot?.origin !== 'snapshot' || slot.threadId === null) + return undefined + const unreconciled: UnreconciledSnapshotTurn = { + turn: { threadId: slot.threadId, messageId: slot.turnId }, + messageId: slot.message.id, + transport: slot.transport, + announced: false + } + unreconciledSnapshotTurns.set(slot.turnId, unreconciled) + return unreconciled + } + + function announceReconcilableSnapshotTurn( + unreconciled: UnreconciledSnapshotTurn + ): void { + if ( + unreconciled.announced || + unreconciled.transport.hasPendingCanvasSync() + ) + return + settledActiveTransports.delete(unreconciled.transport) + unreconciled.announced = true + void snapshotTurnSettled.trigger(unreconciled.turn) + } + + function reconcileSettledTurn( + turn: LiveTurn, + persistedParts: AssistantMessage['parts'] | undefined + ): void { + const unreconciled = unreconciledSnapshotTurns.get(turn.messageId) + if (unreconciled === undefined || !unreconciled.announced) return + unreconciledSnapshotTurns.delete(turn.messageId) + if (persistedParts === undefined || threadId.value !== turn.threadId) + return + const index = messages.value.findIndex( + (message) => message.id === unreconciled.messageId + ) + if (index < 0) return + const reconciled = { ...messages.value[index] } + finishSnapshotTurn(reconciled, persistedParts) + messages.value[index] = reconciled } function ingestActiveTabEvent( @@ -1048,6 +1203,7 @@ export const useAgentConversationStore = defineStore( } function reset(): void { + unreconciledSnapshotTurns.clear() disposeActiveAndSettledTransports() messages.value = [] userTexts.value = new Map() @@ -1091,6 +1247,7 @@ export const useAgentConversationStore = defineStore( function hydrate(history: AgentMessages): void { if (activeSlot.value) rememberDepartedActiveTurn('no-live-turn') + unreconciledSnapshotTurns.clear() disposeActiveAndSettledTransports() clearActive() const transcript = normalizeAgentTranscript(history) @@ -1147,7 +1304,7 @@ export const useAgentConversationStore = defineStore( message: pending.message, transport: createAgentEventTransport( pending.message, - replaceActive, + (message) => publishSnapshotMessage(pending.messageId, message), (workflowId) => canvasSyncGate(workflowId), () => canvasSyncOutcomeCount(), reportUndeliverableAskData @@ -1255,6 +1412,7 @@ export const useAgentConversationStore = defineStore( entries, activeTurnId, activeMessageId, + onSnapshotTurnSettled: snapshotTurnSettled.on, threadId, isStreaming, status, @@ -1288,6 +1446,7 @@ export const useAgentConversationStore = defineStore( turnUsesSkill, historySkillNames, settleTurn, + reconcileSettledTurn, reset, hydrate }