From 61aecea64a9101d7ae87f38c2a303096fedf0777 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 29 Oct 2026 04:00:03 -0700 Subject: [PATCH 1/2] fix(agent): hold human edits made while the doc socket is down (PM-2392) An edit made while the agent process is down was discarded 2.5 s into the outage. `opSender.transmit` settled the batch `undeliverable` once the five 500 ms transport retries ran out, which did two things: the edit never reached the document, and settling it lifted the edit's `LocalWidgetWrites` hold, so the reconnect's catch-up frame was free to project the agent's older value back over the register the user had just changed. The live widget, the next save and the agent's next answer then all read the pre-outage value. Transport retry exhaustion is now a hold, not a terminal discard: the batch stays at the head of the sender's FIFO and goes out on the subscribe ack after the socket returns, with its original `op_id` and stamp. The hold on the register survives with it, so the catch-up frame cannot overwrite the edit in the window before the resend lands. Driven from `doc_subscribed{ok:true}` rather than the socket-live edge: the host answers a subscribe before it reads the ops frame, so the catch-up `doc_update` is already on the wire ahead of the resend. No reconciliation pass and no second graph mutation path: the retained ops are the human's own intent, minted at the command site and replayed through the one wire path. --- .../fixtures/agentFollowerHostSocket.ts | 35 +++ .../agentOfflineEditSurvivesReconnect.spec.ts | 295 ++++++++++++++++++ .../extensions/agent/crdt/opSender.test.ts | 53 +++- .../extensions/agent/crdt/opSender.ts | 26 +- .../agent/crdt/useAgentCrdtFollower.test.ts | 45 +++ .../agent/crdt/useAgentCrdtFollower.ts | 7 + 6 files changed, 458 insertions(+), 3 deletions(-) create mode 100644 browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts diff --git a/browser_tests/fixtures/agentFollowerHostSocket.ts b/browser_tests/fixtures/agentFollowerHostSocket.ts index b979e21e92a..46ea5261e8e 100644 --- a/browser_tests/fixtures/agentFollowerHostSocket.ts +++ b/browser_tests/fixtures/agentFollowerHostSocket.ts @@ -145,6 +145,8 @@ export class AgentFollowerHostSocket { private refusedSubscribes = 0 private socket: WebSocketRoute | null = null + private agentProcessDown = false + private refusedConnections = 0 private comfySocket: WebSocketRoute | null = null private comfyConnections = 0 private connections = 0 @@ -229,12 +231,45 @@ export class AgentFollowerHostSocket { ) }) await this.page.routeWebSocket(AGENT_SOCKET_URL, (socket) => { + // A down agent process refuses the connection outright, so the page's + // reconnect ladder keeps backing off instead of landing on a host that + // answers. The refusal is not counted as a connection. + if (this.agentProcessDown) { + this.refusedConnections += 1 + void socket.close() + return + } this.connections += 1 this.socket = socket socket.onMessage((raw) => this.onClientFrame(raw)) }) } + /** + * Models the agent process going down: the live socket closes and every + * reconnect is refused until {@link endAgentOutage}. Unlike + * {@link disconnect}, which the page recovers from within its first backoff + * step, this holds the outage open for as long as the test needs — long + * enough to outlast the op sender's transport retry budget, which is what + * PM-2392 turns on. + */ + async beginAgentOutage(): Promise { + this.agentProcessDown = true + const socket = this.socket + this.socket = null + await socket?.close() + } + + /** Lets the page's next reconnect attempt through. */ + endAgentOutage(): void { + this.agentProcessDown = false + } + + /** Reconnect attempts refused during an outage, so a test can prove the page kept trying. */ + refusedConnectionCount(): number { + return this.refusedConnections + } + send(frame: AgentWsEvent | AgentAskServerFrame | HostFrame): void { if (frame.type.startsWith('doc_') || frame.type === 'awareness') { if (parseServerDocFrame(frame) === null) diff --git a/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts new file mode 100644 index 00000000000..6f7bd0e5965 --- /dev/null +++ b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts @@ -0,0 +1,295 @@ +import { expect } from '@playwright/test' +import type { WidgetCatalog } from '@comfyorg/comfy-multi-player' +import type { AgentPostMessageRequest } from '@comfyorg/ingest-types' +import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' + +import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { + agentTest as test, + bootAgentApp, + mockAgentTurnApi +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { Topbar } from '@e2e/fixtures/components/Topbar' +import { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' +import { + AGENT_NODE_IDS, + emptySeed +} from '@e2e/fixtures/data/agent/agentRemoteApply' +import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { mockSavedWorkflowPersistence } from '@e2e/fixtures/utils/savedWorkflowPersistence' + +/** + * PM-2392: the agent process goes down, the user changes a widget by hand + * and saves, and the reconnect projects the agent document's older value + * back over the edit — in the live widget, in the next save, and in what the + * next agent turn reads. + * + * The report is about two Load Image nodes. The register this spec drives is + * a `CLIPTextEncode.text` widget instead, because the defect is a property of + * a last-writer-wins widget register and not of any node class, and because + * the Load Image picker only offers a filename the backend can actually serve + * — mocking one through the missing-media path would test the picker, not the + * reconnect. Everything else is the reported sequence: outage, hand edit, + * save, reconnect, reload, next turn. + */ + +const WORKFLOW_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10001' +const THREAD_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10002' +const FIRST_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10003' +const SECOND_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10004' +const [FIRST_NODE_ID, SECOND_NODE_ID] = AGENT_NODE_IDS + +/** What the agent put in the document before the outage. */ +const AGENT_PROMPTS = ['agent prompt one', 'agent prompt two'] as const +/** What the user types by hand while the agent process is down. */ +const HUMAN_PROMPTS = ['human prompt one', 'human prompt two'] as const + +const CLASS_TYPE = 'CLIPTextEncode' +const SAVED_NAME = 'pm-2392-offline-edit' + +/** The follower only projects widgets for a class the pinned catalog names. */ +const catalog: WidgetCatalog = { + types: { [CLASS_TYPE]: { widget_order: ['text'] } } +} + +function addPromptNode(nodeId: number, index: number): RecordedGraphOperation { + const pos = [120 + index * 320, 140] + return { + op: 'add_node', + pos, + node: { + id: nodeId, + pos, + mode: 0, + size: [300, 200], + type: CLASS_TYPE, + flags: {}, + order: index, + inputs: [], + outputs: [], + properties: {}, + widgets_values: [AGENT_PROMPTS[index]] + }, + node_id: nodeId, + class_type: CLASS_TYPE + } +} + +function done(messageId: string): AgentWsEvent { + return { + type: 'agent_message_done', + data: { thread_id: THREAD_ID, message_id: messageId, usage: null } + } +} + +function parsePost(body: string | null): AgentPostMessageRequest { + const parsed: unknown = JSON.parse(body ?? '{}') + return zAgentPostMessageRequest.parse(parsed) +} + +/** + * How long the outage has to last, counted in refused reconnects rather than + * wall clock, so this never becomes a sleep that is "probably long enough". + * + * The op sender's transport retry budget is five retries 500 ms apart, i.e. + * 2.5 s. Below that the batch is still in its ladder and any reconnect + * carries it with no fix needed; above it the pre-fix sender settled the + * batch `undeliverable`, which is both the lost edit and the lifted + * `LocalWidgetWrites` hold that lets the reconnect frame put the agent's + * older value back on the canvas. The page's own reconnect ladder backs off + * 1 s, 2 s, 4 s, 8 s with 0.5-1.0 jitter, so four refusals cannot have taken + * less than 7.5 s — comfortably past the budget, and observed rather than + * assumed. + */ +const REFUSALS_OUTLASTING_RETRY_BUDGET = 4 + +test.describe( + 'Human edits made while the agent process is down', + { tag: ['@cloud', '@agent', '@vue-nodes', '@widget'] }, + () => { + test( + 'survive the reconnect, the save, a reload and the next agent turn', + { + annotation: [ + { + type: 'issue', + description: + 'https://linear.app/comfyorg/issue/PM-2392/agent-reconnect-overwrites-load-image-edits-saved-during-the-outage' + } + ] + }, + async ({ page, request }) => { + test.setTimeout(180_000) + const comfyPage = new ComfyPage(page, request) + const host = new HostDoc(WORKFLOW_ID, emptySeed, catalog) + const hostSocket = new AgentFollowerHostSocket( + page, + WORKFLOW_ID, + host, + 'apply' + ) + const panel = new AgentPanel(page) + const topbar = new Topbar(page) + const posts: AgentPostMessageRequest[] = [] + const docPrompt = (nodeId: number): unknown => { + const widgets = host.graph().nodes[String(nodeId)]?.widgets as + | Record + | undefined + return widgets?.text + } + + await page.route('**/api/object_info', (route) => + route.fulfill(jsonRoute(agentReplayNodeDefs)) + ) + await hostSocket.install() + await mockAgentTurnApi(page, { + thread_id: THREAD_ID, + message_id: FIRST_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + await page.route('**/api/agent/threads/*/messages', (route) => { + if (route.request().method() !== 'POST') return route.fallback() + posts.push(parsePost(route.request().postData())) + return route.fulfill( + jsonRoute({ + thread_id: THREAD_ID, + message_id: + posts.length === 1 ? FIRST_MESSAGE_ID : SECOND_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + ) + }) + await bootAgentApp(page, true, { + objectInfo: 'server', + settings: { 'Comfy.Graph.CanvasInfo': false } + }) + const saved = await mockSavedWorkflowPersistence(page, WORKFLOW_ID) + + const promptField = (index: number) => + comfyPage.vueNodes + .getNodeLocator(String([FIRST_NODE_ID, SECOND_NODE_ID][index])) + .getByLabel('text', { exact: true }) + + await test.step('the agent builds two prompt nodes', async () => { + await panel.open() + await panel.selectWorkflow() + await panel.sendMessage('Add two prompt nodes.') + await hostSocket.waitForSubscribe() + + hostSocket.send( + host.apply([ + addPromptNode(FIRST_NODE_ID, 0), + addPromptNode(SECOND_NODE_ID, 1) + ]) + ) + hostSocket.send(done(FIRST_MESSAGE_ID)) + + for (const [index, value] of AGENT_PROMPTS.entries()) + await expect(promptField(index)).toHaveValue(value) + }) + + const subscribesBeforeOutage = hostSocket.subscribeCount() + + await test.step('the agent process goes down and the user retypes both prompts', async () => { + await hostSocket.beginAgentOutage() + + for (const [index, value] of HUMAN_PROMPTS.entries()) { + await promptField(index).fill(value) + await promptField(index).press('Tab') + await expect(promptField(index)).toHaveValue(value) + } + + // The outage has to outlast the sender's retry budget: that is the + // window PM-2392 lives in, and a shorter one proves nothing. Each + // refusal is also proof the page's reconnect ladder is still alive, + // so a later "no reconnect" cannot be a dead ladder instead. + await expect + .poll(() => hostSocket.refusedConnectionCount(), { + timeout: 90_000, + message: 'the page keeps trying while the agent is down' + }) + .toBeGreaterThanOrEqual(REFUSALS_OUTLASTING_RETRY_BUDGET) + for (const [index, value] of HUMAN_PROMPTS.entries()) + await expect(promptField(index)).toHaveValue(value) + }) + + await test.step('the agent comes back and catches the canvas up', async () => { + // What QA saw on Windows was `doc_subscribe` plus a FULL + // `doc_update` at seq 1, because the agent process that came back + // had rebuilt its document rather than resumed one. A delta + // catch-up carries nothing here and would hide the overwrite, so + // the restarted host answers with its whole state. + hostSocket.answerSubscribesWith(() => host.initialSync()) + hostSocket.endAgentOutage() + await expect + .poll(() => hostSocket.subscribeCount(), { timeout: 90_000 }) + .toBeGreaterThan(subscribesBeforeOutage) + + // The user-visible symptom first: the document's older value must + // not land back on the register the user just changed. + for (const [index, value] of HUMAN_PROMPTS.entries()) + await expect(promptField(index), { + message: 'the reconnect must not overwrite the offline edit' + }).toHaveValue(value, { timeout: 60_000 }) + + // And the edit has to reach the document, not merely stay on + // screen: a canvas that kept the value while the document kept the + // old one would lose it on the next projection anyway. + await expect + .poll(() => docPrompt(FIRST_NODE_ID), { + timeout: 60_000, + message: 'the held edit reaches the document on reconnect' + }) + .toBe(HUMAN_PROMPTS[0]) + await expect + .poll(() => docPrompt(SECOND_NODE_ID), { timeout: 60_000 }) + .toBe(HUMAN_PROMPTS[1]) + }) + + await test.step('the save after the reconnect still holds them', async () => { + await topbar.saveWorkflowAs(SAVED_NAME) + await expect + .poll(() => saved.savedContent()) + .toContain(HUMAN_PROMPTS[0]) + const content = saved.savedContent() ?? '' + for (const value of HUMAN_PROMPTS) expect(content).toContain(value) + for (const value of AGENT_PROMPTS) + expect(content).not.toContain(value) + }) + + await test.step('a reload reopens them from the server', async () => { + const restored = page.waitForResponse( + (response) => + response.request().method() === 'GET' && + decodeURIComponent(new URL(response.url()).pathname) === + `/api/userdata/workflows/${SAVED_NAME}.json`, + { timeout: 90_000 } + ) + await page.reload({ waitUntil: 'domcontentloaded' }) + await panel.open() + await panel.selectWorkflow(SAVED_NAME) + expect((await restored).ok()).toBe(true) + for (const [index, value] of HUMAN_PROMPTS.entries()) + await expect(promptField(index)).toHaveValue(value, { + timeout: 60_000 + }) + }) + + await test.step('the next agent turn reads them too', async () => { + const postsBefore = posts.length + await panel.sendMessage('Which prompts am I using?') + await expect.poll(() => posts.length).toBe(postsBefore + 1) + const draft = JSON.stringify(posts[postsBefore].draft ?? {}) + for (const value of HUMAN_PROMPTS) expect(draft).toContain(value) + for (const value of AGENT_PROMPTS) expect(draft).not.toContain(value) + }) + } + ) + } +) diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index a1241e9aa84..c12fa9ac476 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -210,7 +210,9 @@ describe('createOpSender', () => { }) it.for([ - ['undeliverable', false, WORKFLOW, 3_000, 'transport_exhausted'], + // A down transport no longer settles on its own: the batch is held for + // the socket's return, so the detach below is what reports it. + ['undeliverable', false, WORKFLOW, 3_000, 'detached'], ['undeliverable', true, null, 0, 'unbound'], ['unconfirmed', true, WORKFLOW, 0, 'detached'], ['unacknowledged', true, WORKFLOW, 20_000, 'result_timeout'] @@ -592,12 +594,59 @@ describe('createOpSender', () => { expect(settled[0].state).toBe('unconfirmed') }) - it('settles undeliverable after the transport retry budget', () => { + it('holds a batch the transport retry budget could not carry', () => { transportUp = false sender.enqueue([addNode(1)]) vi.advanceTimersByTime(500 * 6) + expect(sent).toHaveLength(0) + expect(settled).toEqual([]) + expect(sender.pending()).toBe(1) + }) + + it('resends the held batch unchanged when the transport returns', () => { + transportUp = false + sender.enqueue([addNode(1)]) + vi.advanceTimersByTime(500 * 6) + sender.enqueue([addNode(2)]) + vi.advanceTimersByTime(500 * 6) + + transportUp = true + sender.retryTransport() + // The first batch leaves immediately; the second is pumped behind it as + // soon as the first settles, so mint order survives the outage. + expect(sent.map((frame) => frame.ops.map((op) => op.op))).toEqual([ + ['add_node'] + ]) + const [firstOpId] = sent[0].ops.map((op) => op.op_id) + resultListener?.({ + ok: true, + seq: 42, + applied: [firstOpId], + skipped: [], + workflowId: WORKFLOW + }) + + expect(settled).toEqual([ + { + state: 'acknowledged', + ops: expect.any(Array), + result: expect.any(Object) + } + ]) + expect(sent).toHaveLength(2) + expect(sent[1].ops[0].op_id).not.toBe(firstOpId) + }) + + it('settles a held batch undeliverable when its doc is unbound', () => { + transportUp = false + sender.enqueue([addNode(1)]) + vi.advanceTimersByTime(500 * 6) + + boundWorkflow = null + sender.abortIfUnbound() + expect(settled).toEqual([ { state: 'undeliverable', ops: expect.any(Array) } ]) diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index f1fc97c2f41..5e27e941491 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -148,6 +148,15 @@ export interface OpSender { * `undeliverable` exactly as it would have. Idempotent. */ resume(): void + /** + * The socket is live again: re-transmit the batch the transport retry + * budget could not carry, with its original `op_id`s and stamps. A batch + * that exhausted the budget is held, not settled — an edit made while the + * agent process is down is owed delivery, not a lost-edit report (PM-2392, + * and the 2026-09-28 ruling that transport retry exhaustion is not a + * terminal discard). Idempotent, and a no-op while nothing is held. + */ + retryTransport(): void /** * Eager abort seam (FE #16637 residual): settle the in-flight batch NOW * ('unconfirmed' once transmitted, 'undeliverable' otherwise) if its @@ -206,6 +215,8 @@ interface InFlight extends WireBatch { reportedThrow: boolean resent: boolean parked: boolean + /** The retry budget ran out with the socket down; waiting for it to return. */ + awaitingTransport: boolean timer: ReturnType | null } @@ -326,10 +337,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { SEND_RETRY_INTERVAL_MS ) } else { - settleUnbound(batch, 'transport_exhausted') + // The socket is down for longer than the budget, which is what an + // agent process restart looks like. Hold the batch at the head of + // the queue instead of settling it: settling here both loses the + // edit and lifts its `LocalWidgetWrites` hold, so the reconnect + // catch-up frame then projects the pre-outage value back over what + // the user typed (PM-2392). `retryTransport()` resumes it. + batch.awaitingTransport = true } return } + batch.awaitingTransport = false batch.sends++ armResultTimeout(batch) } @@ -443,6 +461,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { reportedThrow: false, resent: false, parked: false, + awaitingTransport: false, timer: null } transmit(inFlight, 0) @@ -672,6 +691,11 @@ export function createOpSender(deps: OpSenderDeps): OpSender { transmit(inFlight, 0) } }, + retryTransport() { + if (!inFlight?.awaitingTransport) return + inFlight.awaitingTransport = false + transmit(inFlight, 0) + }, abortIfUnbound() { if (inFlight && deps.workflowId() !== inFlight.workflowId) { settleUnbound(inFlight, 'unbound') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 3af662d612a..87449dfeabd 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -2981,6 +2981,51 @@ describe('useAgentCrdtFollower', () => { unmount() }) + it('holds a widget edit made while the socket is down and resends it on the reconnect ack (PM-2392)', async () => { + vi.useFakeTimers() + const { recordDevEvent } = await import('./devPanelLog') + const edit: GraphOperation = { + op: 'set_widget', + node_id: '1', + widget: 'image', + value: 'human-choice.png' + } + const { enqueue, unmount } = mountFollower('wf-1') + clientState.sendOps.mockClear() + clientState.sendOps.mockReturnValue(false) + + enqueue([edit]) + await Promise.resolve() + // Five retries at 500 ms, then the whole budget again: the old sender + // settled the batch `undeliverable` here, which both lost the edit and + // lifted its LocalWidgetWrites hold. + vi.advanceTimersByTime(10_000) + + // One attempt plus the five retries, every one refused by the transport. + expect(clientState.sendOps).toHaveBeenCalledTimes(6) + expect(projectionState.noteLocalWrites).toHaveBeenCalledWith([edit]) + expect(projectionState.settleLocalWrites).not.toHaveBeenCalled() + expect( + vi + .mocked(recordDevEvent) + .mock.calls.filter(([event]) => event === 'human_ops_settled') + ).toEqual([]) + + clientState.sendOps.mockReturnValue(true) + agentSocket.open() + dispatchFrame('doc_subscribed', { ok: true, workflowId: 'wf-1', seq: 0 }) + + expect(clientState.sendOps).toHaveBeenCalledTimes(7) + const [, , refused] = clientState.sendOps.mock.calls[0] + const [, , resent] = clientState.sendOps.mock.calls[6] + expect(resent).toHaveLength(1) + expect(resent[0]).toMatchObject({ op: 'set_widget', node_id: '1' }) + // The same wire identity, not a re-mint (KEEP-ALIVE #2, FORECLOSE #7). + expect(resent[0].op_id).toBe(refused[0].op_id) + expect(resent[0].base_version).toBe(refused[0].base_version) + unmount() + }) + it('a refused subscription settles the transmitted in-flight batch unconfirmed immediately, without waiting the resend (residual of #16637)', async () => { vi.useFakeTimers() const { recordDevEvent } = await import('./devPanelLog') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 1dcd2207fe1..f4465649d8f 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -820,6 +820,13 @@ function startAgentCrdtFollower( lifecycle.onSubscribeConfirmed() forgetUnsyncedAtLoad(subscribedWorkflowId.value) resumeHeldOpsIfSubscribed() + // Edits made while the socket was down are held by the sender rather + // than discarded, so the confirmed subscription is where they go out + // (PM-2392). Driven from the ack, not from the socket-live edge: the + // host answers a subscribe before it reads the ops frame, so the + // catch-up `doc_update` is already on the wire ahead of the resend and + // the holds that protect those registers outlive it. + sender.retryTransport() } else { handleRejectedSubscription(detail) } From a10a3ca6d528ea5d853ebd354690dba9957a82a9 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 29 Oct 2026 04:22:20 -0700 Subject: [PATCH 2/2] test(agent): address PM-2392 review findings --- .../fixtures/agentFollowerHostSocket.ts | 16 +- .../fixtures/agentOfflineEditFixture.ts | 237 +++++++++++++++ .../agentOfflineEditSurvivesReconnect.spec.ts | 286 +++--------------- .../extensions/agent/crdt/opSender.test.ts | 4 - .../extensions/agent/crdt/opSender.ts | 16 +- .../agent/crdt/useAgentCrdtFollower.test.ts | 5 - .../agent/crdt/useAgentCrdtFollower.ts | 6 - 7 files changed, 284 insertions(+), 286 deletions(-) create mode 100644 browser_tests/fixtures/agentOfflineEditFixture.ts diff --git a/browser_tests/fixtures/agentFollowerHostSocket.ts b/browser_tests/fixtures/agentFollowerHostSocket.ts index 46ea5261e8e..b605918595a 100644 --- a/browser_tests/fixtures/agentFollowerHostSocket.ts +++ b/browser_tests/fixtures/agentFollowerHostSocket.ts @@ -231,9 +231,6 @@ export class AgentFollowerHostSocket { ) }) await this.page.routeWebSocket(AGENT_SOCKET_URL, (socket) => { - // A down agent process refuses the connection outright, so the page's - // reconnect ladder keeps backing off instead of landing on a host that - // answers. The refusal is not counted as a connection. if (this.agentProcessDown) { this.refusedConnections += 1 void socket.close() @@ -245,14 +242,7 @@ export class AgentFollowerHostSocket { }) } - /** - * Models the agent process going down: the live socket closes and every - * reconnect is refused until {@link endAgentOutage}. Unlike - * {@link disconnect}, which the page recovers from within its first backoff - * step, this holds the outage open for as long as the test needs — long - * enough to outlast the op sender's transport retry budget, which is what - * PM-2392 turns on. - */ + /** Refuses reconnect attempts until the outage ends. */ async beginAgentOutage(): Promise { this.agentProcessDown = true const socket = this.socket @@ -260,12 +250,12 @@ export class AgentFollowerHostSocket { await socket?.close() } - /** Lets the page's next reconnect attempt through. */ + /** Allows the next reconnect attempt. */ endAgentOutage(): void { this.agentProcessDown = false } - /** Reconnect attempts refused during an outage, so a test can prove the page kept trying. */ + /** Returns the number of refused reconnect attempts. */ refusedConnectionCount(): number { return this.refusedConnections } diff --git a/browser_tests/fixtures/agentOfflineEditFixture.ts b/browser_tests/fixtures/agentOfflineEditFixture.ts new file mode 100644 index 00000000000..62b9f6de208 --- /dev/null +++ b/browser_tests/fixtures/agentOfflineEditFixture.ts @@ -0,0 +1,237 @@ +import type { APIRequestContext, Locator, Page } from '@playwright/test' + +import type { WidgetCatalog } from '@comfyorg/comfy-multi-player' +import type { AgentPostMessageRequest } from '@comfyorg/ingest-types' +import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' + +import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { + agentTest, + bootAgentApp, + mockAgentTurnApi +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { Topbar } from '@e2e/fixtures/components/Topbar' +import { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' +import { + AGENT_NODE_IDS, + emptySeed +} from '@e2e/fixtures/data/agent/agentRemoteApply' +import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { mockSavedWorkflowPersistence } from '@e2e/fixtures/utils/savedWorkflowPersistence' + +const WORKFLOW_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10001' +const THREAD_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10002' +const FIRST_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10003' +const SECOND_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10004' +const NODE_IDS = [AGENT_NODE_IDS[0], AGENT_NODE_IDS[1]] as const +const AGENT_PROMPTS = ['agent prompt one', 'agent prompt two'] as const +const HUMAN_PROMPTS = ['human prompt one', 'human prompt two'] as const +const CLASS_TYPE = 'CLIPTextEncode' +const SAVED_NAME = 'pm-2392-offline-edit' + +const catalog: WidgetCatalog = { + types: { [CLASS_TYPE]: { widget_order: ['text'] } } +} + +type SavedWorkflow = Awaited> + +class AgentOfflineEditScenario { + public readonly agentPrompts = AGENT_PROMPTS + public readonly humanPrompts = HUMAN_PROMPTS + + private readonly comfyPage: ComfyPage + private readonly panel: AgentPanel + private readonly topbar: Topbar + + constructor( + private readonly page: Page, + request: APIRequestContext, + private readonly host: HostDoc, + private readonly hostSocket: AgentFollowerHostSocket, + private readonly saved: SavedWorkflow, + private readonly posts: AgentPostMessageRequest[] + ) { + this.comfyPage = new ComfyPage(page, request) + this.panel = new AgentPanel(page) + this.topbar = new Topbar(page) + } + + addPromptNode(nodeId: number, index: number): RecordedGraphOperation { + const pos = [120 + index * 320, 140] + return { + op: 'add_node', + pos, + node: { + id: nodeId, + pos, + mode: 0, + size: [300, 200], + type: CLASS_TYPE, + flags: {}, + order: index, + inputs: [], + outputs: [], + properties: {}, + widgets_values: [AGENT_PROMPTS[index]] + }, + node_id: nodeId, + class_type: CLASS_TYPE + } + } + + done(messageId: string): AgentWsEvent { + return { + type: 'agent_message_done', + data: { thread_id: THREAD_ID, message_id: messageId, usage: null } + } + } + + promptField(index: number): Locator { + return this.comfyPage.vueNodes + .getNodeLocator(String(NODE_IDS[index])) + .getByLabel('text', { exact: true }) + } + + async canvasPromptValues(): Promise { + return await Promise.all( + NODE_IDS.map((_, index) => this.promptField(index).inputValue()) + ) + } + + documentPromptValues(): unknown[] { + return NODE_IDS.map((nodeId) => { + const widgets = this.host.graph().nodes[String(nodeId)]?.widgets as + | Record + | undefined + return widgets?.text + }) + } + + subscribeCount(): number { + return this.hostSocket.subscribeCount() + } + + refusedConnectionCount(): number { + return this.hostSocket.refusedConnectionCount() + } + + postCount(): number { + return this.posts.length + } + + draftAt(index: number): string { + return JSON.stringify(this.posts[index]?.draft ?? {}) + } + + savedContent(): string { + return this.saved.savedContent() ?? '' + } + + async buildAgentPromptNodes(): Promise { + await this.panel.open() + await this.panel.selectWorkflow() + await this.panel.sendMessage('Add two prompt nodes.') + await this.hostSocket.waitForSubscribe() + this.hostSocket.send( + this.host.apply([ + this.addPromptNode(NODE_IDS[0], 0), + this.addPromptNode(NODE_IDS[1], 1) + ]) + ) + this.hostSocket.send(this.done(FIRST_MESSAGE_ID)) + } + + async beginOutageAndRetype(): Promise { + await this.hostSocket.beginAgentOutage() + for (const [index, value] of HUMAN_PROMPTS.entries()) { + await this.promptField(index).fill(value) + await this.promptField(index).press('Tab') + } + } + + restoreAgent(): void { + this.hostSocket.answerSubscribesWith(() => this.host.initialSync()) + this.hostSocket.endAgentOutage() + } + + async saveWorkflow(): Promise { + await this.topbar.saveWorkflowAs(SAVED_NAME) + } + + async reloadSavedWorkflow(): Promise { + const restored = this.page.waitForResponse( + (response) => + response.request().method() === 'GET' && + decodeURIComponent(new URL(response.url()).pathname) === + `/api/userdata/workflows/${SAVED_NAME}.json`, + { timeout: 90_000 } + ) + await this.page.reload({ waitUntil: 'domcontentloaded' }) + await this.panel.open() + await this.panel.selectWorkflow(SAVED_NAME) + return (await restored).ok() + } + + async askNextTurn(): Promise { + await this.panel.sendMessage('Which prompts am I using?') + } +} + +type AgentOfflineEditFixtures = { + offlineEdit: AgentOfflineEditScenario +} + +export const agentOfflineEditTest = agentTest.extend({ + offlineEdit: async ({ page, request, agentFlagEnabled }, use) => { + const host = new HostDoc(WORKFLOW_ID, emptySeed, catalog) + const hostSocket = new AgentFollowerHostSocket( + page, + WORKFLOW_ID, + host, + 'apply' + ) + await page.route('**/api/object_info', (route) => + route.fulfill(jsonRoute(agentReplayNodeDefs)) + ) + await hostSocket.install() + await mockAgentTurnApi(page, { + thread_id: THREAD_ID, + message_id: FIRST_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + + const posts: AgentPostMessageRequest[] = [] + await page.route('**/api/agent/threads/*/messages', (route) => { + if (route.request().method() !== 'POST') return route.fallback() + const parsed: unknown = JSON.parse(route.request().postData() ?? '{}') + posts.push(zAgentPostMessageRequest.parse(parsed)) + return route.fulfill( + jsonRoute({ + thread_id: THREAD_ID, + message_id: posts.length === 1 ? FIRST_MESSAGE_ID : SECOND_MESSAGE_ID, + workflow_id: WORKFLOW_ID + }) + ) + }) + await bootAgentApp(page, agentFlagEnabled, { + objectInfo: 'server', + settings: { 'Comfy.Graph.CanvasInfo': false } + }) + const saved = await mockSavedWorkflowPersistence(page, WORKFLOW_ID) + const scenario = new AgentOfflineEditScenario( + page, + request, + host, + hostSocket, + saved, + posts + ) + await use(scenario) + } +}) diff --git a/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts index 6f7bd0e5965..98c5a2eabb7 100644 --- a/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts +++ b/browser_tests/tests/agent/agentOfflineEditSurvivesReconnect.spec.ts @@ -1,118 +1,13 @@ import { expect } from '@playwright/test' -import type { WidgetCatalog } from '@comfyorg/comfy-multi-player' -import type { AgentPostMessageRequest } from '@comfyorg/ingest-types' -import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod' -import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' - -import { - agentTest as test, - bootAgentApp, - mockAgentTurnApi -} from '@e2e/fixtures/agentPanelFixture' -import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' -import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' -import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' -import { Topbar } from '@e2e/fixtures/components/Topbar' -import { ComfyPage } from '@e2e/fixtures/ComfyPage' -import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' -import { - AGENT_NODE_IDS, - emptySeed -} from '@e2e/fixtures/data/agent/agentRemoteApply' -import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs' -import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' -import { mockSavedWorkflowPersistence } from '@e2e/fixtures/utils/savedWorkflowPersistence' - -/** - * PM-2392: the agent process goes down, the user changes a widget by hand - * and saves, and the reconnect projects the agent document's older value - * back over the edit — in the live widget, in the next save, and in what the - * next agent turn reads. - * - * The report is about two Load Image nodes. The register this spec drives is - * a `CLIPTextEncode.text` widget instead, because the defect is a property of - * a last-writer-wins widget register and not of any node class, and because - * the Load Image picker only offers a filename the backend can actually serve - * — mocking one through the missing-media path would test the picker, not the - * reconnect. Everything else is the reported sequence: outage, hand edit, - * save, reconnect, reload, next turn. - */ - -const WORKFLOW_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10001' -const THREAD_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10002' -const FIRST_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10003' -const SECOND_MESSAGE_ID = '5a41f0b2-9c6e-4a71-8f03-2b6de4c10004' -const [FIRST_NODE_ID, SECOND_NODE_ID] = AGENT_NODE_IDS - -/** What the agent put in the document before the outage. */ -const AGENT_PROMPTS = ['agent prompt one', 'agent prompt two'] as const -/** What the user types by hand while the agent process is down. */ -const HUMAN_PROMPTS = ['human prompt one', 'human prompt two'] as const - -const CLASS_TYPE = 'CLIPTextEncode' -const SAVED_NAME = 'pm-2392-offline-edit' - -/** The follower only projects widgets for a class the pinned catalog names. */ -const catalog: WidgetCatalog = { - types: { [CLASS_TYPE]: { widget_order: ['text'] } } -} - -function addPromptNode(nodeId: number, index: number): RecordedGraphOperation { - const pos = [120 + index * 320, 140] - return { - op: 'add_node', - pos, - node: { - id: nodeId, - pos, - mode: 0, - size: [300, 200], - type: CLASS_TYPE, - flags: {}, - order: index, - inputs: [], - outputs: [], - properties: {}, - widgets_values: [AGENT_PROMPTS[index]] - }, - node_id: nodeId, - class_type: CLASS_TYPE - } -} - -function done(messageId: string): AgentWsEvent { - return { - type: 'agent_message_done', - data: { thread_id: THREAD_ID, message_id: messageId, usage: null } - } -} - -function parsePost(body: string | null): AgentPostMessageRequest { - const parsed: unknown = JSON.parse(body ?? '{}') - return zAgentPostMessageRequest.parse(parsed) -} - -/** - * How long the outage has to last, counted in refused reconnects rather than - * wall clock, so this never becomes a sleep that is "probably long enough". - * - * The op sender's transport retry budget is five retries 500 ms apart, i.e. - * 2.5 s. Below that the batch is still in its ladder and any reconnect - * carries it with no fix needed; above it the pre-fix sender settled the - * batch `undeliverable`, which is both the lost edit and the lifted - * `LocalWidgetWrites` hold that lets the reconnect frame put the agent's - * older value back on the canvas. The page's own reconnect ladder backs off - * 1 s, 2 s, 4 s, 8 s with 0.5-1.0 jitter, so four refusals cannot have taken - * less than 7.5 s — comfortably past the budget, and observed rather than - * assumed. - */ -const REFUSALS_OUTLASTING_RETRY_BUDGET = 4 +import { agentOfflineEditTest as test } from '@e2e/fixtures/agentOfflineEditFixture' test.describe( 'Human edits made while the agent process is down', { tag: ['@cloud', '@agent', '@vue-nodes', '@widget'] }, () => { + test.describe.configure({ timeout: 180_000 }) + test( 'survive the reconnect, the save, a reload and the next agent turn', { @@ -124,170 +19,75 @@ test.describe( } ] }, - async ({ page, request }) => { - test.setTimeout(180_000) - const comfyPage = new ComfyPage(page, request) - const host = new HostDoc(WORKFLOW_ID, emptySeed, catalog) - const hostSocket = new AgentFollowerHostSocket( - page, - WORKFLOW_ID, - host, - 'apply' - ) - const panel = new AgentPanel(page) - const topbar = new Topbar(page) - const posts: AgentPostMessageRequest[] = [] - const docPrompt = (nodeId: number): unknown => { - const widgets = host.graph().nodes[String(nodeId)]?.widgets as - | Record - | undefined - return widgets?.text - } - - await page.route('**/api/object_info', (route) => - route.fulfill(jsonRoute(agentReplayNodeDefs)) - ) - await hostSocket.install() - await mockAgentTurnApi(page, { - thread_id: THREAD_ID, - message_id: FIRST_MESSAGE_ID, - workflow_id: WORKFLOW_ID - }) - await page.route('**/api/agent/threads/*/messages', (route) => { - if (route.request().method() !== 'POST') return route.fallback() - posts.push(parsePost(route.request().postData())) - return route.fulfill( - jsonRoute({ - thread_id: THREAD_ID, - message_id: - posts.length === 1 ? FIRST_MESSAGE_ID : SECOND_MESSAGE_ID, - workflow_id: WORKFLOW_ID - }) - ) - }) - await bootAgentApp(page, true, { - objectInfo: 'server', - settings: { 'Comfy.Graph.CanvasInfo': false } - }) - const saved = await mockSavedWorkflowPersistence(page, WORKFLOW_ID) - - const promptField = (index: number) => - comfyPage.vueNodes - .getNodeLocator(String([FIRST_NODE_ID, SECOND_NODE_ID][index])) - .getByLabel('text', { exact: true }) - + async ({ offlineEdit }) => { await test.step('the agent builds two prompt nodes', async () => { - await panel.open() - await panel.selectWorkflow() - await panel.sendMessage('Add two prompt nodes.') - await hostSocket.waitForSubscribe() - - hostSocket.send( - host.apply([ - addPromptNode(FIRST_NODE_ID, 0), - addPromptNode(SECOND_NODE_ID, 1) - ]) - ) - hostSocket.send(done(FIRST_MESSAGE_ID)) - - for (const [index, value] of AGENT_PROMPTS.entries()) - await expect(promptField(index)).toHaveValue(value) + await offlineEdit.buildAgentPromptNodes() + await expect + .poll(() => offlineEdit.canvasPromptValues()) + .toEqual(offlineEdit.agentPrompts) }) - const subscribesBeforeOutage = hostSocket.subscribeCount() + const subscribesBeforeOutage = offlineEdit.subscribeCount() await test.step('the agent process goes down and the user retypes both prompts', async () => { - await hostSocket.beginAgentOutage() - - for (const [index, value] of HUMAN_PROMPTS.entries()) { - await promptField(index).fill(value) - await promptField(index).press('Tab') - await expect(promptField(index)).toHaveValue(value) - } - - // The outage has to outlast the sender's retry budget: that is the - // window PM-2392 lives in, and a shorter one proves nothing. Each - // refusal is also proof the page's reconnect ladder is still alive, - // so a later "no reconnect" cannot be a dead ladder instead. + await offlineEdit.beginOutageAndRetype() await expect - .poll(() => hostSocket.refusedConnectionCount(), { + .poll(() => offlineEdit.refusedConnectionCount(), { timeout: 90_000, message: 'the page keeps trying while the agent is down' }) - .toBeGreaterThanOrEqual(REFUSALS_OUTLASTING_RETRY_BUDGET) - for (const [index, value] of HUMAN_PROMPTS.entries()) - await expect(promptField(index)).toHaveValue(value) + .toBeGreaterThanOrEqual(4) + await expect + .poll(() => offlineEdit.canvasPromptValues()) + .toEqual(offlineEdit.humanPrompts) }) await test.step('the agent comes back and catches the canvas up', async () => { - // What QA saw on Windows was `doc_subscribe` plus a FULL - // `doc_update` at seq 1, because the agent process that came back - // had rebuilt its document rather than resumed one. A delta - // catch-up carries nothing here and would hide the overwrite, so - // the restarted host answers with its whole state. - hostSocket.answerSubscribesWith(() => host.initialSync()) - hostSocket.endAgentOutage() + offlineEdit.restoreAgent() await expect - .poll(() => hostSocket.subscribeCount(), { timeout: 90_000 }) + .poll(() => offlineEdit.subscribeCount(), { timeout: 90_000 }) .toBeGreaterThan(subscribesBeforeOutage) - - // The user-visible symptom first: the document's older value must - // not land back on the register the user just changed. - for (const [index, value] of HUMAN_PROMPTS.entries()) - await expect(promptField(index), { + await expect + .poll(() => offlineEdit.canvasPromptValues(), { + timeout: 60_000, message: 'the reconnect must not overwrite the offline edit' - }).toHaveValue(value, { timeout: 60_000 }) - - // And the edit has to reach the document, not merely stay on - // screen: a canvas that kept the value while the document kept the - // old one would lose it on the next projection anyway. + }) + .toEqual(offlineEdit.humanPrompts) await expect - .poll(() => docPrompt(FIRST_NODE_ID), { + .poll(() => offlineEdit.documentPromptValues(), { timeout: 60_000, message: 'the held edit reaches the document on reconnect' }) - .toBe(HUMAN_PROMPTS[0]) - await expect - .poll(() => docPrompt(SECOND_NODE_ID), { timeout: 60_000 }) - .toBe(HUMAN_PROMPTS[1]) + .toEqual(offlineEdit.humanPrompts) }) await test.step('the save after the reconnect still holds them', async () => { - await topbar.saveWorkflowAs(SAVED_NAME) + await offlineEdit.saveWorkflow() await expect - .poll(() => saved.savedContent()) - .toContain(HUMAN_PROMPTS[0]) - const content = saved.savedContent() ?? '' - for (const value of HUMAN_PROMPTS) expect(content).toContain(value) - for (const value of AGENT_PROMPTS) - expect(content).not.toContain(value) + .poll(() => offlineEdit.savedContent()) + .toContain(offlineEdit.humanPrompts[0]) + const content = offlineEdit.savedContent() + expect(content).toContain(offlineEdit.humanPrompts[1]) + expect(content).not.toContain(offlineEdit.agentPrompts[0]) + expect(content).not.toContain(offlineEdit.agentPrompts[1]) }) await test.step('a reload reopens them from the server', async () => { - const restored = page.waitForResponse( - (response) => - response.request().method() === 'GET' && - decodeURIComponent(new URL(response.url()).pathname) === - `/api/userdata/workflows/${SAVED_NAME}.json`, - { timeout: 90_000 } - ) - await page.reload({ waitUntil: 'domcontentloaded' }) - await panel.open() - await panel.selectWorkflow(SAVED_NAME) - expect((await restored).ok()).toBe(true) - for (const [index, value] of HUMAN_PROMPTS.entries()) - await expect(promptField(index)).toHaveValue(value, { - timeout: 60_000 - }) + expect(await offlineEdit.reloadSavedWorkflow()).toBe(true) + await expect + .poll(() => offlineEdit.canvasPromptValues(), { timeout: 60_000 }) + .toEqual(offlineEdit.humanPrompts) }) await test.step('the next agent turn reads them too', async () => { - const postsBefore = posts.length - await panel.sendMessage('Which prompts am I using?') - await expect.poll(() => posts.length).toBe(postsBefore + 1) - const draft = JSON.stringify(posts[postsBefore].draft ?? {}) - for (const value of HUMAN_PROMPTS) expect(draft).toContain(value) - for (const value of AGENT_PROMPTS) expect(draft).not.toContain(value) + const postsBefore = offlineEdit.postCount() + await offlineEdit.askNextTurn() + await expect.poll(() => offlineEdit.postCount()).toBe(postsBefore + 1) + const draft = offlineEdit.draftAt(postsBefore) + expect(draft).toContain(offlineEdit.humanPrompts[0]) + expect(draft).toContain(offlineEdit.humanPrompts[1]) + expect(draft).not.toContain(offlineEdit.agentPrompts[0]) + expect(draft).not.toContain(offlineEdit.agentPrompts[1]) }) } ) diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index c12fa9ac476..db426b2a04c 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -210,8 +210,6 @@ describe('createOpSender', () => { }) it.for([ - // A down transport no longer settles on its own: the batch is held for - // the socket's return, so the detach below is what reports it. ['undeliverable', false, WORKFLOW, 3_000, 'detached'], ['undeliverable', true, null, 0, 'unbound'], ['unconfirmed', true, WORKFLOW, 0, 'detached'], @@ -614,8 +612,6 @@ describe('createOpSender', () => { transportUp = true sender.retryTransport() - // The first batch leaves immediately; the second is pumped behind it as - // soon as the first settles, so mint order survives the outage. expect(sent.map((frame) => frame.ops.map((op) => op.op))).toEqual([ ['add_node'] ]) diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index 5e27e941491..21a3fd93b24 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -148,14 +148,7 @@ export interface OpSender { * `undeliverable` exactly as it would have. Idempotent. */ resume(): void - /** - * The socket is live again: re-transmit the batch the transport retry - * budget could not carry, with its original `op_id`s and stamps. A batch - * that exhausted the budget is held, not settled — an edit made while the - * agent process is down is owed delivery, not a lost-edit report (PM-2392, - * and the 2026-09-28 ruling that transport retry exhaustion is not a - * terminal discard). Idempotent, and a no-op while nothing is held. - */ + /** Retries a batch held after transport exhaustion. */ retryTransport(): void /** * Eager abort seam (FE #16637 residual): settle the in-flight batch NOW @@ -215,7 +208,6 @@ interface InFlight extends WireBatch { reportedThrow: boolean resent: boolean parked: boolean - /** The retry budget ran out with the socket down; waiting for it to return. */ awaitingTransport: boolean timer: ReturnType | null } @@ -337,12 +329,6 @@ export function createOpSender(deps: OpSenderDeps): OpSender { SEND_RETRY_INTERVAL_MS ) } else { - // The socket is down for longer than the budget, which is what an - // agent process restart looks like. Hold the batch at the head of - // the queue instead of settling it: settling here both loses the - // edit and lifts its `LocalWidgetWrites` hold, so the reconnect - // catch-up frame then projects the pre-outage value back over what - // the user typed (PM-2392). `retryTransport()` resumes it. batch.awaitingTransport = true } return diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 87449dfeabd..5a48b798c1c 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -2996,12 +2996,8 @@ describe('useAgentCrdtFollower', () => { enqueue([edit]) await Promise.resolve() - // Five retries at 500 ms, then the whole budget again: the old sender - // settled the batch `undeliverable` here, which both lost the edit and - // lifted its LocalWidgetWrites hold. vi.advanceTimersByTime(10_000) - // One attempt plus the five retries, every one refused by the transport. expect(clientState.sendOps).toHaveBeenCalledTimes(6) expect(projectionState.noteLocalWrites).toHaveBeenCalledWith([edit]) expect(projectionState.settleLocalWrites).not.toHaveBeenCalled() @@ -3020,7 +3016,6 @@ describe('useAgentCrdtFollower', () => { const [, , resent] = clientState.sendOps.mock.calls[6] expect(resent).toHaveLength(1) expect(resent[0]).toMatchObject({ op: 'set_widget', node_id: '1' }) - // The same wire identity, not a re-mint (KEEP-ALIVE #2, FORECLOSE #7). expect(resent[0].op_id).toBe(refused[0].op_id) expect(resent[0].base_version).toBe(refused[0].base_version) unmount() diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index f4465649d8f..46ef510b9dd 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -820,12 +820,6 @@ function startAgentCrdtFollower( lifecycle.onSubscribeConfirmed() forgetUnsyncedAtLoad(subscribedWorkflowId.value) resumeHeldOpsIfSubscribed() - // Edits made while the socket was down are held by the sender rather - // than discarded, so the confirmed subscription is where they go out - // (PM-2392). Driven from the ack, not from the socket-live edge: the - // host answers a subscribe before it reads the ops frame, so the - // catch-up `doc_update` is already on the wire ahead of the resend and - // the holds that protect those registers outlive it. sender.retryTransport() } else { handleRejectedSubscription(detail)