diff --git a/browser_tests/fixtures/agentFollowerHostSocket.ts b/browser_tests/fixtures/agentFollowerHostSocket.ts index b979e21e92a..b605918595a 100644 --- a/browser_tests/fixtures/agentFollowerHostSocket.ts +++ b/browser_tests/fixtures/agentFollowerHostSocket.ts @@ -145,6 +145,8 @@ export class AgentFollowerHostSocket { private refusedSubscribes = 0 private socket: WebSocketRoute | null = null + private agentProcessDown = false + private refusedConnections = 0 private comfySocket: WebSocketRoute | null = null private comfyConnections = 0 private connections = 0 @@ -229,12 +231,35 @@ export class AgentFollowerHostSocket { ) }) await this.page.routeWebSocket(AGENT_SOCKET_URL, (socket) => { + if (this.agentProcessDown) { + this.refusedConnections += 1 + void socket.close() + return + } this.connections += 1 this.socket = socket socket.onMessage((raw) => this.onClientFrame(raw)) }) } + /** Refuses reconnect attempts until the outage ends. */ + async beginAgentOutage(): Promise { + this.agentProcessDown = true + const socket = this.socket + this.socket = null + await socket?.close() + } + + /** Allows the next reconnect attempt. */ + endAgentOutage(): void { + this.agentProcessDown = false + } + + /** Returns the number of refused reconnect attempts. */ + refusedConnectionCount(): number { + return this.refusedConnections + } + send(frame: AgentWsEvent | AgentAskServerFrame | HostFrame): void { if (frame.type.startsWith('doc_') || frame.type === 'awareness') { if (parseServerDocFrame(frame) === null) diff --git a/browser_tests/fixtures/agentOfflineEditFixture.ts b/browser_tests/fixtures/agentOfflineEditFixture.ts new file mode 100644 index 00000000000..62b9f6de208 --- /dev/null +++ b/browser_tests/fixtures/agentOfflineEditFixture.ts @@ -0,0 +1,237 @@ +import type { APIRequestContext, Locator, Page } from '@playwright/test' + +import type { WidgetCatalog } from '@comfyorg/comfy-multi-player' +import type { AgentPostMessageRequest } from '@comfyorg/ingest-types' +import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' + +import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { + agentTest, + bootAgentApp, + mockAgentTurnApi +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { Topbar } from '@e2e/fixtures/components/Topbar' +import { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' +import { + AGENT_NODE_IDS, + emptySeed +} from '@e2e/fixtures/data/agent/agentRemoteApply' +import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { mockSavedWorkflowPersistence } from '@e2e/fixtures/utils/savedWorkflowPersistence' + +const WORKFLOW_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10001' +const THREAD_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10002' +const FIRST_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10003' +const SECOND_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10004' +const NODE_IDS = [AGENT_NODE_IDS[0], AGENT_NODE_IDS[1]] as const +const AGENT_PROMPTS = ['agent prompt one', 'agent prompt two'] as const +const HUMAN_PROMPTS = ['human prompt one', 'human prompt two'] as const +const CLASS_TYPE = 'CLIPTextEncode' +const SAVED_NAME = 'pm-2392-offline-edit' + +const catalog: WidgetCatalog = { + types: { [CLASS_TYPE]: { widget_order: ['text'] } } +} + +type SavedWorkflow = Awaited> + +class AgentOfflineEditScenario { + public readonly agentPrompts = AGENT_PROMPTS + public readonly humanPrompts = HUMAN_PROMPTS + + private readonly comfyPage: ComfyPage + private readonly panel: AgentPanel + private readonly topbar: Topbar + + constructor( + private readonly page: Page, + request: APIRequestContext, + private readonly host: HostDoc, + private readonly hostSocket: AgentFollowerHostSocket, + private readonly saved: SavedWorkflow, + private readonly posts: AgentPostMessageRequest[] + ) { + this.comfyPage = new ComfyPage(page, request) + this.panel = new AgentPanel(page) + this.topbar = new Topbar(page) + } + + addPromptNode(nodeId: number, index: number): RecordedGraphOperation { + const pos = [120 + index * 320, 140] + return { + op: 'add_node', + pos, + node: { + id: nodeId, + pos, + mode: 0, + size: [300, 200], + type: CLASS_TYPE, + flags: {}, + order: index, + inputs: [], + outputs: [], + properties: {}, + widgets_values: [AGENT_PROMPTS[index]] + }, + node_id: nodeId, + class_type: CLASS_TYPE + } + } + + done(messageId: string): AgentWsEvent { + return { + type: 'agent_message_done', + data: { thread_id: THREAD_ID, message_id: messageId, usage: null } + } + } + + promptField(index: number): Locator { + return this.comfyPage.vueNodes + .getNodeLocator(String(NODE_IDS[index])) + .getByLabel('text', { exact: true }) + } + + async canvasPromptValues(): Promise { + return await Promise.all( + NODE_IDS.map((_, index) => this.promptField(index).inputValue()) + ) + } + + documentPromptValues(): unknown[] { + return NODE_IDS.map((nodeId) => { + const widgets = this.host.graph().nodes[String(nodeId)]?.widgets as + | Record + | undefined + return widgets?.text + }) + } + + subscribeCount(): number { + return this.hostSocket.subscribeCount() + } + + refusedConnectionCount(): number { + return this.hostSocket.refusedConnectionCount() + } + + postCount(): number { + return this.posts.length + } + + draftAt(index: number): string { + return JSON.stringify(this.posts[index]?.draft ?? {}) + } + + savedContent(): string { + return this.saved.savedContent() ?? '' + } + + async buildAgentPromptNodes(): Promise { + await this.panel.open() + await this.panel.selectWorkflow() + await this.panel.sendMessage('Add two prompt nodes.') + await this.hostSocket.waitForSubscribe() + this.hostSocket.send( + this.host.apply([ + this.addPromptNode(NODE_IDS[0], 0), + this.addPromptNode(NODE_IDS[1], 1) + ]) + ) + this.hostSocket.send(this.done(FIRST_MESSAGE_ID)) + } + + async beginOutageAndRetype(): Promise { + await this.hostSocket.beginAgentOutage() + for (const [index, value] of HUMAN_PROMPTS.entries()) { + await this.promptField(index).fill(value) + await this.promptField(index).press('Tab') + } + } + + restoreAgent(): void { + this.hostSocket.answerSubscribesWith(() => this.host.initialSync()) + this.hostSocket.endAgentOutage() + } + + async saveWorkflow(): Promise { + await this.topbar.saveWorkflowAs(SAVED_NAME) + } + + async reloadSavedWorkflow(): Promise { + const restored = this.page.waitForResponse( + (response) => + response.request().method() === 'GET' && + decodeURIComponent(new URL(response.url()).pathname) === + `/api/userdata/workflows/${SAVED_NAME}.json`, + { timeout: 90_000 } + ) + await this.page.reload({ waitUntil: 'domcontentloaded' }) + await this.panel.open() + await this.panel.selectWorkflow(SAVED_NAME) + return (await restored).ok() + } + + async askNextTurn(): Promise { + await this.panel.sendMessage('Which prompts am I using?') + } +} + +type AgentOfflineEditFixtures = { + offlineEdit: AgentOfflineEditScenario +} + +export const agentOfflineEditTest = agentTest.extend({ + offlineEdit: async ({ page, request, agentFlagEnabled }, use) => { + const host = new HostDoc(WORKFLOW_ID, emptySeed, catalog) + const hostSocket = new AgentFollowerHostSocket( + page, + WORKFLOW_ID, + host, + 'apply' + ) + await page.route('**/api/object_info', (route) => + route.fulfill(jsonRoute(agentReplayNodeDefs)) + ) + await hostSocket.install() + await mockAgentTurnApi(page, { + thread_id: THREAD_ID, + message_id: FIRST_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + + const posts: AgentPostMessageRequest[] = [] + await page.route('**/api/agent/threads/*/messages', (route) => { + if (route.request().method() !== 'POST') return route.fallback() + const parsed: unknown = JSON.parse(route.request().postData() ?? '{}') + posts.push(zAgentPostMessageRequest.parse(parsed)) + return route.fulfill( + jsonRoute({ + thread_id: THREAD_ID, + message_id: posts.length === 1 ? FIRST_MESSAGE_ID : SECOND_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + ) + }) + await bootAgentApp(page, agentFlagEnabled, { + objectInfo: 'server', + settings: { 'Comfy.Graph.CanvasInfo': false } + }) + const saved = await mockSavedWorkflowPersistence(page, WORKFLOW_ID) + const scenario = new AgentOfflineEditScenario( + page, + request, + host, + hostSocket, + saved, + posts + ) + await use(scenario) + } +}) diff --git a/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts new file mode 100644 index 00000000000..98c5a2eabb7 --- /dev/null +++ b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts @@ -0,0 +1,95 @@ +import { expect } from '@playwright/test' + +import { agentOfflineEditTest as test } from '@e2e/fixtures/agentOfflineEditFixture' + +test.describe( + 'Human edits made while the agent process is down', + { tag: ['@cloud', '@agent', '@vue-nodes', '@widget'] }, + () => { + test.describe.configure({ timeout: 180_000 }) + + test( + 'survive the reconnect, the save, a reload and the next agent turn', + { + annotation: [ + { + type: 'issue', + description: + 'https://linear.app/comfyorg/issue/PM-2392/agent-reconnect-overwrites-load-image-edits-saved-during-the-outage' + } + ] + }, + async ({ offlineEdit }) => { + await test.step('the agent builds two prompt nodes', async () => { + await offlineEdit.buildAgentPromptNodes() + await expect + .poll(() => offlineEdit.canvasPromptValues()) + .toEqual(offlineEdit.agentPrompts) + }) + + const subscribesBeforeOutage = offlineEdit.subscribeCount() + + await test.step('the agent process goes down and the user retypes both prompts', async () => { + await offlineEdit.beginOutageAndRetype() + await expect + .poll(() => offlineEdit.refusedConnectionCount(), { + timeout: 90_000, + message: 'the page keeps trying while the agent is down' + }) + .toBeGreaterThanOrEqual(4) + await expect + .poll(() => offlineEdit.canvasPromptValues()) + .toEqual(offlineEdit.humanPrompts) + }) + + await test.step('the agent comes back and catches the canvas up', async () => { + offlineEdit.restoreAgent() + await expect + .poll(() => offlineEdit.subscribeCount(), { timeout: 90_000 }) + .toBeGreaterThan(subscribesBeforeOutage) + await expect + .poll(() => offlineEdit.canvasPromptValues(), { + timeout: 60_000, + message: 'the reconnect must not overwrite the offline edit' + }) + .toEqual(offlineEdit.humanPrompts) + await expect + .poll(() => offlineEdit.documentPromptValues(), { + timeout: 60_000, + message: 'the held edit reaches the document on reconnect' + }) + .toEqual(offlineEdit.humanPrompts) + }) + + await test.step('the save after the reconnect still holds them', async () => { + await offlineEdit.saveWorkflow() + await expect + .poll(() => offlineEdit.savedContent()) + .toContain(offlineEdit.humanPrompts[0]) + const content = offlineEdit.savedContent() + expect(content).toContain(offlineEdit.humanPrompts[1]) + expect(content).not.toContain(offlineEdit.agentPrompts[0]) + expect(content).not.toContain(offlineEdit.agentPrompts[1]) + }) + + await test.step('a reload reopens them from the server', async () => { + expect(await offlineEdit.reloadSavedWorkflow()).toBe(true) + await expect + .poll(() => offlineEdit.canvasPromptValues(), { timeout: 60_000 }) + .toEqual(offlineEdit.humanPrompts) + }) + + await test.step('the next agent turn reads them too', async () => { + const postsBefore = offlineEdit.postCount() + await offlineEdit.askNextTurn() + await expect.poll(() => offlineEdit.postCount()).toBe(postsBefore + 1) + const draft = offlineEdit.draftAt(postsBefore) + expect(draft).toContain(offlineEdit.humanPrompts[0]) + expect(draft).toContain(offlineEdit.humanPrompts[1]) + expect(draft).not.toContain(offlineEdit.agentPrompts[0]) + expect(draft).not.toContain(offlineEdit.agentPrompts[1]) + }) + } + ) + } +) diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index a1241e9aa84..db426b2a04c 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -210,7 +210,7 @@ describe('createOpSender', () => { }) it.for([ - ['undeliverable', false, WORKFLOW, 3_000, 'transport_exhausted'], + ['undeliverable', false, WORKFLOW, 3_000, 'detached'], ['undeliverable', true, null, 0, 'unbound'], ['unconfirmed', true, WORKFLOW, 0, 'detached'], ['unacknowledged', true, WORKFLOW, 20_000, 'result_timeout'] @@ -592,12 +592,57 @@ describe('createOpSender', () => { expect(settled[0].state).toBe('unconfirmed') }) - it('settles undeliverable after the transport retry budget', () => { + it('holds a batch the transport retry budget could not carry', () => { transportUp = false sender.enqueue([addNode(1)]) vi.advanceTimersByTime(500 * 6) + expect(sent).toHaveLength(0) + expect(settled).toEqual([]) + expect(sender.pending()).toBe(1) + }) + + it('resends the held batch unchanged when the transport returns', () => { + transportUp = false + sender.enqueue([addNode(1)]) + vi.advanceTimersByTime(500 * 6) + sender.enqueue([addNode(2)]) + vi.advanceTimersByTime(500 * 6) + + transportUp = true + sender.retryTransport() + expect(sent.map((frame) => frame.ops.map((op) => op.op))).toEqual([ + ['add_node'] + ]) + const [firstOpId] = sent[0].ops.map((op) => op.op_id) + resultListener?.({ + ok: true, + seq: 42, + applied: [firstOpId], + skipped: [], + workflowId: WORKFLOW + }) + + expect(settled).toEqual([ + { + state: 'acknowledged', + ops: expect.any(Array), + result: expect.any(Object) + } + ]) + expect(sent).toHaveLength(2) + expect(sent[1].ops[0].op_id).not.toBe(firstOpId) + }) + + it('settles a held batch undeliverable when its doc is unbound', () => { + transportUp = false + sender.enqueue([addNode(1)]) + vi.advanceTimersByTime(500 * 6) + + boundWorkflow = null + sender.abortIfUnbound() + expect(settled).toEqual([ { state: 'undeliverable', ops: expect.any(Array) } ]) diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index f1fc97c2f41..21a3fd93b24 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -148,6 +148,8 @@ export interface OpSender { * `undeliverable` exactly as it would have. Idempotent. */ resume(): void + /** Retries a batch held after transport exhaustion. */ + retryTransport(): void /** * Eager abort seam (FE #16637 residual): settle the in-flight batch NOW * ('unconfirmed' once transmitted, 'undeliverable' otherwise) if its @@ -206,6 +208,7 @@ interface InFlight extends WireBatch { reportedThrow: boolean resent: boolean parked: boolean + awaitingTransport: boolean timer: ReturnType | null } @@ -326,10 +329,11 @@ export function createOpSender(deps: OpSenderDeps): OpSender { SEND_RETRY_INTERVAL_MS ) } else { - settleUnbound(batch, 'transport_exhausted') + batch.awaitingTransport = true } return } + batch.awaitingTransport = false batch.sends++ armResultTimeout(batch) } @@ -443,6 +447,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { reportedThrow: false, resent: false, parked: false, + awaitingTransport: false, timer: null } transmit(inFlight, 0) @@ -672,6 +677,11 @@ export function createOpSender(deps: OpSenderDeps): OpSender { transmit(inFlight, 0) } }, + retryTransport() { + if (!inFlight?.awaitingTransport) return + inFlight.awaitingTransport = false + transmit(inFlight, 0) + }, abortIfUnbound() { if (inFlight && deps.workflowId() !== inFlight.workflowId) { settleUnbound(inFlight, 'unbound') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 3af662d612a..5a48b798c1c 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -2981,6 +2981,46 @@ describe('useAgentCrdtFollower', () => { unmount() }) + it('holds a widget edit made while the socket is down and resends it on the reconnect ack (PM-2392)', async () => { + vi.useFakeTimers() + const { recordDevEvent } = await import('./devPanelLog') + const edit: GraphOperation = { + op: 'set_widget', + node_id: '1', + widget: 'image', + value: 'human-choice.png' + } + const { enqueue, unmount } = mountFollower('wf-1') + clientState.sendOps.mockClear() + clientState.sendOps.mockReturnValue(false) + + enqueue([edit]) + await Promise.resolve() + vi.advanceTimersByTime(10_000) + + expect(clientState.sendOps).toHaveBeenCalledTimes(6) + expect(projectionState.noteLocalWrites).toHaveBeenCalledWith([edit]) + expect(projectionState.settleLocalWrites).not.toHaveBeenCalled() + expect( + vi + .mocked(recordDevEvent) + .mock.calls.filter(([event]) => event === 'human_ops_settled') + ).toEqual([]) + + clientState.sendOps.mockReturnValue(true) + agentSocket.open() + dispatchFrame('doc_subscribed', { ok: true, workflowId: 'wf-1', seq: 0 }) + + expect(clientState.sendOps).toHaveBeenCalledTimes(7) + const [, , refused] = clientState.sendOps.mock.calls[0] + const [, , resent] = clientState.sendOps.mock.calls[6] + expect(resent).toHaveLength(1) + expect(resent[0]).toMatchObject({ op: 'set_widget', node_id: '1' }) + expect(resent[0].op_id).toBe(refused[0].op_id) + expect(resent[0].base_version).toBe(refused[0].base_version) + unmount() + }) + it('a refused subscription settles the transmitted in-flight batch unconfirmed immediately, without waiting the resend (residual of #16637)', async () => { vi.useFakeTimers() const { recordDevEvent } = await import('./devPanelLog') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 1dcd2207fe1..46ef510b9dd 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -820,6 +820,7 @@ function startAgentCrdtFollower( lifecycle.onSubscribeConfirmed() forgetUnsyncedAtLoad(subscribedWorkflowId.value) resumeHeldOpsIfSubscribed() + sender.retryTransport() } else { handleRejectedSubscription(detail) }