diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index c6541620775..7b68e234daf 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -6,6 +6,7 @@ import { z } from 'zod' import { createI18n } from 'vue-i18n' import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import type { ComfyNodeDef, ObjectInfoResponse } from '@/schemas/nodeDefSchema' import { toNodeId } from '@/types/nodeId' import type { @@ -16,12 +17,14 @@ import type { import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' import { parseServerDocFrame } from '@/workbench/extensions/agent/crdt/docFrameClient' import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' +import type { useAgentWorkflowTabBindingStore } from '@/workbench/extensions/agent/stores/agent/agentWorkflowTabBindingStore' import { agentTest, bootAgentApp, mockWorkflowPersistence } from '@e2e/fixtures/agentPanelFixture' +import type { HostFrame } from '@e2e/fixtures/agentConversationHostDoc' import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' import type { @@ -170,6 +173,35 @@ async function withTimeout( } } +type WorkflowBindingActions = Pick< + ReturnType, + 'tabPathFor' +> + +type WorkflowLookup = Pick< + ReturnType, + 'getWorkflowByPath' +> + +type WorkflowScopeAction = + | { kind: 'drop'; workflowId: string } + | { + kind: 'restore' + tabPath: string + graphId: string + } + +interface DroppedWorkflowScope { + tabPath: string + graphId: string +} + +interface ReplayResponseOptions { + beforeFirstGraphOps?: () => Promise + beforeGraphOps?: (ops: readonly RecordedGraphOperation[]) => Promise + waitForGraphOpsDelivery?: (ops: readonly RecordedGraphOperation[]) => boolean +} + // Runs one recorded prompt/response through the real panel over a routed /ws socket. export class AgentConversationHarness { readonly panel: Locator @@ -341,7 +373,7 @@ export class AgentConversationHarness { async replayResponse( turn = 0, - beforeFirstGraphOps?: () => Promise + options: ReplayResponseOptions = {} ): Promise { const startedAt = Date.now() const response = this.conversation.turns[turn].response @@ -354,8 +386,12 @@ export class AgentConversationHarness { this.hostSocket.send(this.stampTurn(entry.event, turn)) else { await this.hostSocket.waitForSubscribe() - if (index === firstGraphOps) await beforeFirstGraphOps?.() - this.hostSocket.send(this.host.apply(entry.ops)) + if (index === firstGraphOps) await options.beforeFirstGraphOps?.() + await options.beforeGraphOps?.(entry.ops) + await this.sendHostFrame( + this.host.apply(entry.ops), + options.waitForGraphOpsDelivery?.(entry.ops) === true + ) for (const id of Object.keys(this.host.graph().nodes)) this.seenIds.add(id) } @@ -370,7 +406,7 @@ export class AgentConversationHarness { for (const turn of this.conversation.turns.keys()) { const before = await this.panelCounts() await this.sendPrompt(turn) - await this.replayResponse(turn, beforeFirstGraphOps) + await this.replayResponse(turn, { beforeFirstGraphOps }) await this.waitForTurnComplete() await this.expectTurnRendered(turn, before) await this.expectCanvasReplayed(turn) @@ -939,31 +975,115 @@ export class AgentConversationHarness { await this.selectWorkflowTarget() } - // Sends one more doc_update that resyncs `widget` on `nodeId` to its - // current doc value — the same effect on a live widget as a stale echo, - // a reconnect resync, or an unrelated full-graph reconcile has whenever - // that frame's changed-widgets sweep happens to touch it. Lets a test - // race this deterministically against a live keystroke instead of - // waiting on the timing a real run happens to produce. - async resyncWidget(nodeId: string, widget: string): Promise { - const widgets = z - .record(z.string(), z.unknown()) - .optional() - .parse(this.host.graph().nodes[nodeId]?.widgets) - const value = widgets?.[widget] - if (value === undefined) - throw new Error(`Host widget ${nodeId}.${widget} does not exist`) - const operation = { - op: 'set_widget', - node_id: nodeId, - widget, - value, - old: value - } satisfies GraphOperation - const frame = this.host.apply([operation]) + async dropWorkflowScope(): Promise<() => Promise> { + const workflowId = this.conversation.workflow.id + const dropped = await this.runWorkflowScopeAction({ + kind: 'drop', + workflowId + }) + if (!dropped) + throw new Error(`no bound tab path for workflow ${workflowId}`) + + let restored = false + return async () => { + if (restored) return + await this.runWorkflowScopeAction({ kind: 'restore', ...dropped }) + restored = true + } + } + + async rememberRecoveryGraph(): Promise { + await this.page.evaluate(() => { + if (!window.app?.rootGraph) throw new Error('root graph is unavailable') + window.__agentRecoveryGraph = window.app.rootGraph + }) + } + + async isRecoveryGraphUnchanged(): Promise { + return await this.page.evaluate( + () => window.__agentRecoveryGraph === window.app?.rootGraph + ) + } + + private async runWorkflowScopeAction( + action: WorkflowScopeAction + ): Promise { + if (action.kind === 'drop') + return await this.dropWorkflowScopeInPage(action) + await this.restoreWorkflowScopeInPage(action) + return undefined + } + + private async dropWorkflowScopeInPage( + action: Extract + ): Promise { + return await this.page.evaluate((workflowId) => { + const isBinding = (value: unknown): value is WorkflowBindingActions => + typeof value === 'object' && + value !== null && + 'tabPathFor' in value && + typeof value.tabPathFor === 'function' + const isWorkflowLookup = (value: unknown): value is WorkflowLookup => + typeof value === 'object' && + value !== null && + 'getWorkflowByPath' in value && + typeof value.getWorkflowByPath === 'function' + + const mount = document.getElementById('vue-app') + const pinia = mount?.__vue_app__?.config.globalProperties.$pinia + if (!pinia) throw new Error('Pinia is not mounted') + const stores = [...pinia._s.values()] + const binding = stores.find(isBinding) + const workflows = stores.find(isWorkflowLookup) + if (!binding) + throw new Error('agentWorkflowTabBinding store is not mounted') + if (!workflows) throw new Error('workflow store is not mounted') + + const tabPath = binding.tabPathFor(workflowId) + if (!tabPath) + throw new Error(`no bound tab path for workflow ${workflowId}`) + const tab = workflows.getWorkflowByPath(tabPath) + if (!tab) throw new Error(`no open tab at ${tabPath}`) + const graphId = tab.activeState?.id + if (!graphId) throw new Error(`workflow at ${tabPath} has no graph id`) + delete tab.activeState?.id + return { tabPath, graphId } + }, action.workflowId) + } + + private async restoreWorkflowScopeInPage( + action: Extract + ): Promise { + await this.page.evaluate((action) => { + const isWorkflowLookup = (value: unknown): value is WorkflowLookup => + typeof value === 'object' && + value !== null && + 'getWorkflowByPath' in value && + typeof value.getWorkflowByPath === 'function' + const mount = document.getElementById('vue-app') + const pinia = mount?.__vue_app__?.config.globalProperties.$pinia + if (!pinia) throw new Error('Pinia is not mounted') + const workflows = [...pinia._s.values()].find(isWorkflowLookup) + if (!workflows) throw new Error('workflow store is not mounted') + const tab = workflows.getWorkflowByPath(action.tabPath) + if (!tab) throw new Error(`no open tab at ${action.tabPath}`) + if (!tab.activeState) + throw new Error(`workflow at ${action.tabPath} has no active state`) + tab.activeState.id = action.graphId + }, action) + } + + private async sendHostFrame( + frame: HostFrame, + waitForDelivery = false + ): Promise { + if (!waitForDelivery) { + this.hostSocket.send(frame) + return + } const parsedFrame = parseServerDocFrame(frame) if (parsedFrame?.type !== 'doc_update') - throw new Error('Host widget resync did not produce a doc_update') + throw new Error('Host operation did not produce a doc_update') const receipt = crypto.randomUUID() await this.page.evaluate( @@ -1015,6 +1135,25 @@ export class AgentConversationHarness { }, receipt) } } + + async resyncWidget(nodeId: string, widget: string): Promise { + const widgets = z + .record(z.string(), z.unknown()) + .optional() + .parse(this.host.graph().nodes[nodeId]?.widgets) + const value = widgets?.[widget] + if (value === undefined) + throw new Error(`Host widget ${nodeId}.${widget} does not exist`) + const operation = { + op: 'set_widget', + node_id: nodeId, + widget, + value, + old: value + } satisfies GraphOperation + const frame = this.host.apply([operation]) + await this.sendHostFrame(frame, true) + } } export type ReplayTiming = 'immediate' | 'recorded' diff --git a/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts b/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts index 1d5006eac04..251be326f47 100644 --- a/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts +++ b/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts @@ -388,13 +388,15 @@ test.describe( await agentConversation.sendPrompt(0) - await agentConversation.replayResponse(0, async () => { - await test.step('the user completes and leaves a widget edit', async () => { - await textField.fill(localValue) - await textField.press('Tab') - await expect(textField).not.toBeFocused() - await expect(textField).toHaveValue(localValue) - }) + await agentConversation.replayResponse(0, { + beforeFirstGraphOps: async () => { + await test.step('the user completes and leaves a widget edit', async () => { + await textField.fill(localValue) + await textField.press('Tab') + await expect(textField).not.toBeFocused() + await expect(textField).toHaveValue(localValue) + }) + } }) await agentConversation.waitForTurnComplete() diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts new file mode 100644 index 00000000000..99602c2dee1 --- /dev/null +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -0,0 +1,85 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' + +test.describe( + 'Agent delete and clear operations', + { tag: ['@cloud', '@vue-nodes'] }, + () => { + test.describe('delete_node', () => { + test.use({ conversationCase: 'agent-rec-add-set-delete' }) + + test('removes the requested node while preserving the workflow', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + + await agentConversation.rememberRecoveryGraph() + await agentConversation.runTurns() + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe(true) + }) + }) + + test.describe('clear_canvas', () => { + test.use({ conversationCase: 'agent-rec-clear-workflow' }) + + test('leaves the canvas empty after clearing the workflow', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + + await agentConversation.rememberRecoveryGraph() + await agentConversation.runTurns() + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe(true) + }) + + test('clears the canvas on its own after the clear frame is rejected for a dropped workflow scope', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + let restoreScope: (() => Promise) | undefined + const isClear = (ops: readonly RecordedGraphOperation[]) => + ops.some((op) => op.op === 'clear') + + await agentConversation.rememberRecoveryGraph() + try { + await test.step('deliver the clear while workflow scope is unavailable', async () => { + await agentConversation.sendPrompt(0) + await agentConversation.replayResponse(0, { + beforeGraphOps: async (ops) => { + if (!isClear(ops)) return + await expect(agentConversation.vueNodes.nodes).not.toHaveCount( + 0 + ) + restoreScope = await agentConversation.dropWorkflowScope() + }, + waitForGraphOpsDelivery: isClear + }) + }) + + if (!restoreScope) + throw new Error('the clear frame never dropped workflow scope') + const restore = restoreScope + + await test.step('keep the same graph stale after the rejected clear', async () => { + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe( + true + ) + }) + + await test.step('clear without another frame when scope returns', async () => { + await restore() + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe( + true + ) + }) + } finally { + await restoreScope?.() + } + }) + }) + } +) diff --git a/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts b/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts index 40538e12b07..e7f4f13e765 100644 --- a/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts +++ b/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts @@ -39,22 +39,24 @@ test.describe( await agentConversation.sendPrompt() - await agentConversation.replayResponse(0, async () => { - await test.step('user switches workflows before the agent edits arrive', async () => { - await expect(tabs).toHaveCount(1) - await expect( - topbar.getTab(0).and(topbar.getActiveTab()) - ).toBeVisible() - const inputs = await ksamplerInputs() - expect(inputs.steps).toBe(20) - expect(inputs.cfg).toBe(7) - await topbar.newWorkflowButton.click() - await expect(tabs).toHaveCount(2) - await expect( - topbar.getTab(1).and(topbar.getActiveTab()) - ).toBeVisible() - await expect(agentConversation.vueNodes.nodes).toHaveCount(0) - }) + await agentConversation.replayResponse(0, { + beforeFirstGraphOps: async () => { + await test.step('user switches workflows before the agent edits arrive', async () => { + await expect(tabs).toHaveCount(1) + await expect( + topbar.getTab(0).and(topbar.getActiveTab()) + ).toBeVisible() + const inputs = await ksamplerInputs() + expect(inputs.steps).toBe(20) + expect(inputs.cfg).toBe(7) + await topbar.newWorkflowButton.click() + await expect(tabs).toHaveCount(2) + await expect( + topbar.getTab(1).and(topbar.getActiveTab()) + ).toBeVisible() + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + }) + } }) await test.step('agent finishes editing the background workflow', async () => { diff --git a/browser_tests/types/globals.d.ts b/browser_tests/types/globals.d.ts index a0a4d37cb8b..54f8c7f4ea9 100644 --- a/browser_tests/types/globals.d.ts +++ b/browser_tests/types/globals.d.ts @@ -4,6 +4,7 @@ import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode' import type { LiteGraphGlobal } from '@/lib/litegraph/src/LiteGraphGlobal' import type { ComfyApp } from '@/scripts/app' import type { useWorkspaceStore } from '@/stores/workspaceStore' +import type { App } from 'vue' /** * Helper type for accessing nodes by ID in browser tests. @@ -30,6 +31,10 @@ interface CapturedMessages { } declare global { + interface HTMLElement { + __vue_app__?: App + } + interface Window { app?: ComfyApp graph?: LGraph @@ -60,6 +65,7 @@ declare global { * @see browser_tests/tests/agent/agentHumanAddTabSwitch.spec.ts */ __tabSwitchLens?: TabSwitchLens + __agentRecoveryGraph?: LGraph } const app: ComfyApp | undefined diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index e0736109b00..17b628fcad0 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -1,4 +1,4 @@ -import { mint, nodesMap } from '@comfyorg/comfy-multi-player' +import { linksMap, mint, nodesMap } from '@comfyorg/comfy-multi-player' import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' import * as Y from 'yjs' @@ -281,3 +281,168 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { ) }) }) + +describe('AgentCrdtProjection self-driven reconcile retry', () => { + it('sweeps the stale live node once a retry recovers from a rejected delete-all, without a new frame arriving', () => { + vi.useFakeTimers() + try { + const { graph, source } = buildLiveGraph() + const scope = graphScopeOf(graph) + let scopeAvailable = true + const mutations = createGraphMutations({ + getScope: () => (scopeAvailable ? scope : null), + layout, + placement: inertPlacementPort + }) + const host = mint( + toWorkflowJson(structuredClone(graph.serialize())), + CATALOG + ) + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + mutations, + () => graph, + () => follower.doc + ) + projection.bind(WORKFLOW_ID, follower) + let seq = 0 + const deliver = (update: Uint8Array) => { + follower.applyRemoteUpdate(update) + const applied = projection.applyFrame({ + workflowId: WORKFLOW_ID, + seq: ++seq, + update, + actor: 'agent:comfy:host', + opIds: [] + }) + // Mirrors `useAgentCrdtFollower`'s `applyAndReconcile`: the live + // graph is only ever swept for a frame that actually applied. + if (applied) projection.reconcileLiveGraph(WORKFLOW_ID) + return applied + } + + // Catch-up frame: the doc starts out matching the live graph, so the + // pre-existing `source` node is adopted as the live adapter for + // record id 1. + deliver(Y.encodeStateAsUpdate(host)) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // The agent deletes every node from the doc (a delete-all), but scope + // briefly can't resolve the bound workflow tab, so the batch is + // rejected: nothing is swept, and the stale node stays live. + const before = Y.encodeStateVector(host) + host.transact(() => { + nodesMap(host).clear() + linksMap(host).clear() + }) + scopeAvailable = false + const deleteAllUpdate = Y.encodeStateAsUpdate(host, before) + expect(deliver(deleteAllUpdate)).toBe(false) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Scope recovers, but no further frame ever arrives (e.g. the user + // never sends another agent message). The self-driven retry must + // both commit the store-side reconcile AND sweep the live graph + // through the same pipeline a normal frame gets, or the stale node + // would still be there to serialize into the next save. + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(graph.getNodeById(toNodeId(1))).toBeNull() + expect(layout.deleteNodes).toHaveBeenCalledWith( + scope, + [toNodeId(1)], + expect.anything() + ) + + projection.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('removes the stale live node once a later timer retries a live-graph sweep that threw once', () => { + vi.useFakeTimers() + try { + const { graph, source } = buildLiveGraph() + const scope = graphScopeOf(graph) + let scopeAvailable = true + const mutations = createGraphMutations({ + getScope: () => (scopeAvailable ? scope : null), + layout, + placement: inertPlacementPort + }) + const host = mint( + toWorkflowJson(structuredClone(graph.serialize())), + CATALOG + ) + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + mutations, + () => graph, + () => follower.doc + ) + let seq = 0 + const deliver = (update: Uint8Array) => { + follower.applyRemoteUpdate(update) + const applied = projection.applyFrame({ + workflowId: WORKFLOW_ID, + seq: ++seq, + update, + actor: 'agent:comfy:host', + opIds: [] + }) + if (applied) projection.reconcileLiveGraph(WORKFLOW_ID) + return applied + } + + projection.bind(WORKFLOW_ID, follower) + deliver(Y.encodeStateAsUpdate(host)) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Only now does the sweep start throwing, so the catch-up frame above + // (a normal, non-retry commit) is unaffected. + const sweepFailure = new Error('live sweep failed') + const sweepSpy = vi.spyOn(projection, 'reconcileLiveGraph') + sweepSpy.mockImplementationOnce(() => { + throw sweepFailure + }) + + const before = Y.encodeStateVector(host) + host.transact(() => { + nodesMap(host).clear() + linksMap(host).clear() + }) + scopeAvailable = false + const deleteAllUpdate = Y.encodeStateAsUpdate(host, before) + expect(deliver(deleteAllUpdate)).toBe(false) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Scope recovers: the retry's batch commits, but its live-graph sweep + // throws once. The store has already converged, but the throw must + // not be silently treated as done - the stale node is still live. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + expect(sweepSpy).toHaveBeenCalledTimes(1) + + // A later timer retries only the sweep - never the already-committed + // mutation - and this time it actually removes the live node. The + // live-sweep retry runs on the slow (2s) cadence, not the fast (200ms) + // batch-retry cadence. + vi.advanceTimersByTime(2_000) + expect(graph.getNodeById(toNodeId(1))).toBeNull() + expect(sweepSpy).toHaveBeenCalledTimes(2) + + projection.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index 86c3484b0a2..80a176caea9 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -27,7 +27,9 @@ export class AgentCrdtProjection { private readonly getFollowerDoc: () => Y.Doc, intent?: LocalIntent ) { - this.adapter = new EcsFollowerAdapter(mutations, intent) + this.adapter = new EcsFollowerAdapter(mutations, intent, (workflowId) => + this.reconcileLiveGraph(workflowId) + ) } bind(workflowId: string, follower: FollowerDoc): void { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 19716dc09a5..7fda437ea6b 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -11,9 +11,11 @@ import * as Y from 'yjs' import { createGraphMutations } from './graphMutations' import { inertPlacementPort } from './__fixtures__/inertPlacementPort' import type { GraphMutations } from './graphMutations' +import type { reportError as reportErrorFn } from '@/platform/telemetry/reportError' import { useLinkStore } from '@/stores/linkStore' import { useNodeDataStore } from '@/stores/nodeDataStore' import { useWidgetValueStore } from '@/stores/widgetValueStore' +import type { RemoteMutationContext } from '@/types/graphMutationContext' import { toOwningGraphId, toRootGraphId } from '@/types/graphScopeId' import { toLinkId } from '@/types/linkId' import type { NodeId } from '@/types/nodeId' @@ -24,6 +26,14 @@ import type { DocUpdate } from './docFrameClient' import { EcsFollowerAdapter } from './ecsFollowerAdapter' import { FollowerDoc } from './followerDoc' +const telemetryState = vi.hoisted(() => ({ + reportError: vi.fn() +})) + +vi.mock(import('@/platform/telemetry/reportError'), () => ({ + reportError: telemetryState.reportError +})) + const catalog: WidgetCatalog = { types: { Source: { widget_order: ['seed', 'stale'] }, @@ -473,18 +483,821 @@ describe('EcsFollowerAdapter integration', () => { host.destroy() }) + it('proactively reconciles once scope becomes available again, without waiting for a new frame', () => { + vi.useFakeTimers() + try { + const deleteLayouts = vi.fn() + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + const context = { + source: 'agent-remote', + actor: 'local-hydration', + opId: 'local-seed' + } satisfies RemoteMutationContext + scopeAvailable = true + mutations.addNode( + { + id: 99, + type: 'Sink', + widgets_values: { stale: 9 }, + inputs: [], + outputs: [] + }, + context + ) + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + // Store-only: the adapter has no notion of the live graph, so its + // owner (`AgentCrdtProjection`) is the one that would route a + // successful retry into `reconcileLiveGraph`. Assert the adapter + // actually invokes that seam once the retry commits, not just that + // the ECS store converges - a retry that only updates stores but + // never asks anything to sweep the canvas leaves a stale node + // rendered (and savable) even though this test's own store-level + // assertions below would still pass. + const onReconcileRetryCommitted = vi.fn() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // First frame: the batch is rejected (no scope available yet), so the + // reconciliation is dropped and local-only node 99 survives. A + // distinctive actor/opIds prove the eventual replay is attributed to + // this frame, not the generic 'replay' fallback a synthetic frame + // would also satisfy. + scopeAvailable = false + deleteLayouts.mockClear() + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 7, + update, + actor: 'agent:distinctive-actor', + opIds: ['distinctive-op-id'] + }) + ).toBe(false) + expect( + useNodeDataStore() + .getGraphNodesFor('root', 'root') + .map(({ id }) => id) + ).toEqual([toNodeId(99)]) + expect(onReconcileRetryCommitted).not.toHaveBeenCalled() + + // Stay unavailable past the old 20-attempt/4-second retry budget. No + // new frame arrives, so recovery must remain armed at its slower rate. + vi.advanceTimersByTime(5_000) + expect( + useNodeDataStore() + .getGraphNodesFor('root', 'root') + .map(({ id }) => id) + ).toEqual([toNodeId(99)]) + + scopeAvailable = true + vi.advanceTimersByTime(2_000) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ + actor: 'agent:distinctive-actor', + opId: 'distinctive-op-id', + opIds: ['distinctive-op-id'] + }) + ) + expect(onReconcileRetryCommitted).toHaveBeenCalledExactlyOnceWith('wf') + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('retries at a fixed 200ms cadence for exactly 20 attempts, then 2s, and a fresh episode restarts at 200ms', () => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'local-hydration', + opId: 'local-seed' + } satisfies RemoteMutationContext + scopeAvailable = true + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batchResult: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batchResult(batchContext, define) + } + } + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(trackedMutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // Episode 1: rejected for lack of scope, which never returns during + // the fast tier. + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + // Exactly 20 fast attempts at a fixed 200ms cadence. + vi.advanceTimersByTime(20 * 200) + expect(batchCalls).toHaveLength(21) + + // The slow tier is a fixed 2s, not another 200ms: a hot-loop-forever + // implementation would already have fired several more batches by the + // 1999ms mark. + vi.advanceTimersByTime(1_999) + expect(batchCalls).toHaveLength(21) + vi.advanceTimersByTime(1) + expect(batchCalls).toHaveLength(22) + + // Let episode 1 finally commit so none of its budget leaks into + // episode 2. + scopeAvailable = true + vi.advanceTimersByTime(2_000) + expect(batchCalls).toHaveLength(23) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + + // Episode 2: a fresh rejection gets the full fast budget back, so its + // first retry fires at 200ms again instead of inheriting episode 1's + // slow cadence or its spent attempt count. + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(24) + vi.advanceTimersByTime(199) + expect(batchCalls).toHaveLength(24) + vi.advanceTimersByTime(1) + expect(batchCalls).toHaveLength(25) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('reports and keeps retrying when a timer-driven retry batch itself throws', () => { + vi.useFakeTimers() + try { + telemetryState.reportError.mockClear() + let scopeAvailable = false + const realMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'seed', + opId: 'seed' + } satisfies RemoteMutationContext + scopeAvailable = true + realMutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + scopeAvailable = false + + const failure = new Error('retry batch threw') + let batchCallCount = 0 + // Call 1 is the initial (caller-driven) rejection; call 2 is the + // timer's own first retry attempt - the one DrJKL's finding says the + // existing suite never reaches. Call 3 is the next retry, which must + // still fire on the existing budget afterward. + const throwingMutations: GraphMutations = { + ...realMutations, + batchResult: (batchContext, define) => { + batchCallCount += 1 + if (batchCallCount === 2) throw failure + return realMutations.batchResult(batchContext, define) + } + } + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(throwingMutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCallCount).toBe(1) + + // The timer's own retry throws. It must not escape as an unhandled + // error, and must report the exact failure and reschedule. + expect(() => vi.advanceTimersByTime(200)).not.toThrow() + expect(batchCallCount).toBe(2) + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( + failure, + { + errorType: 'error_agent_reconcile_retry_threw', + context: { workflowId: 'wf' } + } + ) + + // The chain survives the throw: the next timer-driven attempt commits + // and converges, without any new frame ever arriving. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(batchCallCount).toBe(3) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('keeps recovery armed and never replays the committed batch when the live-graph sweep callback throws', () => { + vi.useFakeTimers() + try { + telemetryState.reportError.mockClear() + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'local-hydration', + opId: 'local-seed' + } satisfies RemoteMutationContext + scopeAvailable = true + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + scopeAvailable = false + + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batchResult: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batchResult(batchContext, define) + } + } + const sweepFailure = new Error('live sweep failed') + let sweepCalls = 0 + const onReconcileRetryCommitted = vi.fn(() => { + sweepCalls += 1 + if (sweepCalls === 1) throw sweepFailure + }) + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + trackedMutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + // The retry batch commits, but the live-graph sweep callback throws on + // its first call. The throw must not replay the already-committed + // mutation - only the sweep is retried. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(batchCalls).toHaveLength(2) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( + sweepFailure, + { + errorType: 'error_agent_reconcile_live_sweep_threw', + context: { workflowId: 'wf' } + } + ) + + // A later timer removes the live node's store record by retrying only + // the sweep, never the mutation batch. + vi.advanceTimersByTime(2_000) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + expect(batchCalls).toHaveLength(2) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('attributes recovery to every frame folded into its authoritative snapshot', () => { + vi.useFakeTimers() + try { + let scopeAvailable = true + const deleteLayouts = vi.fn() + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + { source: 'agent-remote', actor: 'seed', opId: 'seed' } + ) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + scopeAvailable = false + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 7, + update, + actor: 'agent:first-frame', + opIds: ['first-op'] + }) + ).toBe(false) + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 8, + update, + actor: 'agent:later-frame', + opIds: ['later-op'] + }) + ).toBe(false) + + scopeAvailable = true + vi.advanceTimersByTime(200) + + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ + actor: 'agent-reconcile', + opId: 'later-op', + opIds: ['first-op', 'later-op'] + }) + ) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a no-scope retry armed before %s cannot mutate or callback after it', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batchResult: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batchResult(batchContext, define) + } + } + const onReconcileRetryCommitted = vi.fn() + + const host = mint( + { + nodes: [{ id: 1, type: 'Source', pos: [0, 0], outputs: [] }], + links: [] + }, + catalog + ) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + trackedMutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // Reject for lack of scope, arming a fast retry. + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + batchCalls.length = 0 + + // Scope returning afterward must never resurrect the cleared, + // discarded, or unbound state: deleting the new cancellation calls + // would otherwise let this fire a batch (or the callback) later. + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(batchCalls).toHaveLength(0) + expect(onReconcileRetryCommitted).not.toHaveBeenCalled() + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a failed live sweep armed before %s cannot callback after it', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const onReconcileRetryCommitted = vi.fn(() => { + throw new Error('live sweep failed') + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + + vi.advanceTimersByTime(5_000) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + + it.for(['clearForReset', 'discardPending'] as const)( + '%s called by a throwing live sweep cannot be followed by another sweep', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const onReconcileRetryCommitted = vi.fn(() => { + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else { + adapter.discardPending('wf') + } + throw new Error('live sweep failed after cancellation') + }) + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a second, later live-sweep failure cannot orphan an earlier sweep-retry timer past %s', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const onReconcileRetryCommitted = vi.fn(() => { + throw new Error('live sweep failed') + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // First rejection episode: its batch retry commits at 200ms, and its + // live-graph sweep throws, arming a sweep-retry timer 2s out. + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + + // If the first sweep-retry timer were orphaned, it would still fire + // ~2s after the first sweep failure and invoke the callback again + // even though recovery was just cancelled. + vi.advanceTimersByTime(5_000) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + + it('retries from the batch rejection reason when scope appears immediately afterward', () => { + vi.useFakeTimers() + try { + const seedMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => scope, + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + seedMutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + { source: 'agent-remote', actor: 'seed', opId: 'seed' } + ) + + let scopeReads = 0 + const deleteLayouts = vi.fn() + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (++scopeReads === 1 ? null : scope), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + vi.advanceTimersByTime(200) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ opId: 'replay' }) + ) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('does not arm a self-driven retry for a deterministic rejection scope can never fix', () => { + vi.useFakeTimers() + try { + const host = mint( + { + nodes: [ + { + id: 1, + type: 'Source', + pos: [0, 0], + outputs: [{ name: 'out', type: 'IMAGE', links: [9] }], + inputs: [] + }, + { + id: 2, + type: 'Sink', + pos: [300, 0], + inputs: [{ name: 'in', type: 'IMAGE', link: 9 }], + outputs: [] + } + ], + links: [[9, 1, 0, 2, 0, 'IMAGE']] + }, + catalog + ) + const follower = new FollowerDoc() + const realMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => scope, + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const batchCalls: unknown[] = [] + const mutations: GraphMutations = { + ...realMutations, + batchResult: (context, define) => { + batchCalls.push(context) + return realMutations.batchResult(context, define) + } + } + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const bootstrap = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(bootstrap) + expect( + adapter.applyFrame({ workflowId: 'wf', seq: 1, update: bootstrap }) + ).toBe(true) + batchCalls.length = 0 + + // Retype node 1 to a type with no output, while the doc still carries + // link 9 originating from its old output slot 0: `prepare()`'s stale + // link revalidation rejects this deterministically (see the "FEC-4" + // test below). Scope stays available throughout. + const before = Y.encodeStateVector(host) + const retypeOp = { + op_id: 'retype', + actor: 'agent:test', + base_version: 2, + stamp: [2, 'agent:test'], + op: 'add_node', + node_id: 1, + class_type: 'Sink', + pos: [0, 0], + node: { + id: 1, + type: 'Sink', + pos: [0, 0], + inputs: [{ name: 'in', type: 'IMAGE', link: null }], + outputs: [] + } + } satisfies Op + applyOps(host, [retypeOp], catalog) + const update = Y.encodeStateAsUpdate(host, before) + follower.applyRemoteUpdate(update) + const rejected = vi + .spyOn(console, 'error') + .mockImplementation(() => undefined) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + rejected.mockRestore() + + // If the adapter blindly armed a retry for every rejection, this + // advance would fire it and call `batch()` again with the identical, + // still-invalid doc state - 20 wasted whole-document reconciles at a + // fixed interval that retrying can never turn into a commit. + vi.advanceTimersByTime(5_000) + expect(batchCalls).toHaveLength(1) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + it('replays authoritative state through real mutations after a batch throws', () => { const mutations = createGraphMutations({ placement: inertPlacementPort, getScope: () => scope, layout: { createNode: vi.fn(), deleteNodes: vi.fn() } }) - const realBatch = mutations.batch.bind(mutations) + const realBatch = mutations.batchResult.bind(mutations) const failure = new Error('projection failed') let throwNextBatch = true const throwingMutations: GraphMutations = { ...mutations, - batch: (context, define) => { + batchResult: (context, define) => { if (throwNextBatch) { throwNextBatch = false throw failure @@ -1406,12 +2219,13 @@ describe('EcsFollowerAdapter integration', () => { clearSemanticGraph: () => undefined } return { - batch: (_context, define) => { + batch: () => true, + batchResult: (_context, define) => { events.push(`${workflowId}:start`) define(noopBatch) if (workflowId === 'wf-a') adapter.applyFrame(frameB) events.push(`${workflowId}:end`) - return true + return { kind: 'committed' } }, addNode: () => true, setWidget: () => true, diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 74d52ef53ef..218d904c471 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -6,6 +6,8 @@ import { import * as Y from 'yjs' import type { + GraphMutationBatch, + GraphMutationBatchResult, GraphMutations, SemanticLinkPayload, SemanticNodePayload @@ -29,6 +31,17 @@ import type { FollowerDoc } from './followerDoc' type NodeRootAction = 'add' | 'update' | 'delete' +/** + * A rejected batch (no scope available) arms `reconcileNextFrame`, but that + * only fires on the next *incoming* frame, which may never arrive (e.g. the + * user never sends another agent message). This timed retry loop gives the + * scope race a fast, self-driven chance to resolve instead of leaving the + * stale doc/live mismatch on screen until some unrelated later frame lands. + */ +const RECONCILE_FAST_RETRY_LIMIT = 20 +const RECONCILE_RETRY_INTERVAL_MS = 200 +const RECONCILE_SLOW_RETRY_INTERVAL_MS = 2_000 + /** * Node-map keys whose by-key edits trigger a field resync. Structural keys * (`inputs`, `outputs`, `pos`, `size`, widget storage) are excluded: slots @@ -351,6 +364,20 @@ function frameContext(update: DocUpdate): RemoteMutationContext { } } +function mergeRecoveryFrame(first: DocUpdate, latest: DocUpdate): DocUpdate { + const opIds = [...new Set([...(first.opIds ?? []), ...(latest.opIds ?? [])])] + return { + ...latest, + actor: + first.actor === latest.actor + ? first.actor + : first.actor || latest.actor + ? 'agent-reconcile' + : undefined, + ...(opIds.length > 0 ? { opIds } : { opIds: undefined }) + } +} + interface TargetSession { readonly workflowId: string readonly follower: FollowerDoc @@ -370,6 +397,38 @@ interface TargetSession { onLinksChanged: (event: Y.YMapEvent) => void reconcileNextFrame: boolean applying: boolean + batchRecovery: BatchRecoveryState + liveSweepRecovery: LiveSweepRecoveryState +} + +type BatchRecoveryState = + | { kind: 'idle' } + | { + kind: 'scheduled' + timer: ReturnType + attempt: number + frame: DocUpdate + } + | { kind: 'applying'; attempt: number; frame: DocUpdate } + +type LiveSweepRecoveryState = + | { kind: 'idle' } + | { kind: 'running' } + | { kind: 'scheduled'; timer: ReturnType } + +type BatchChanges = { + batch: GraphMutationBatch + session: TargetSession + doc: Y.Doc + definitions: () => SubgraphDefinitionIndex + reconcile: boolean + removedLinkIds: number[] + nodeActions: Map + replacedWidgetMaps: Set + replacedOpaqueWidgets: Set + changedNodeFields: Set + changedWidgets: Map> + changedLinks: Map } /** @@ -383,7 +442,10 @@ export class EcsFollowerAdapter { constructor( private readonly mutations: MutationsForTarget, - private readonly intent: LocalIntent = NO_LOCAL_INTENT + private readonly intent: LocalIntent = NO_LOCAL_INTENT, + private readonly onReconcileRetryCommitted: ( + workflowId: string + ) => void = () => undefined ) {} bind(workflowId: string, follower: FollowerDoc): void { @@ -397,6 +459,8 @@ export class EcsFollowerAdapter { unbind(workflowId: string): void { const session = this.targets.get(workflowId) if (!session) return + this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) session.nodes.unobserveDeep(session.onNodesChanged) session.links.unobserve(session.onLinksChanged) this.targets.delete(workflowId) @@ -429,12 +493,22 @@ export class EcsFollowerAdapter { const session = this.targets.get(workflowId) if (!session) return false this.discardSessionPending(session) + this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) + session.reconcileNextFrame = false return session.mutations.clearSemanticGraph(context) } discardPending(workflowId: string): void { const session = this.targets.get(workflowId) - if (session) this.discardSessionPending(session) + if (!session) return + this.discardSessionPending(session) + // Same reasoning as `clearForReset`: a schema-rejected frame is a + // fail-closed gate, and a stale retry firing later would silently + // re-apply the very state that gate just refused. + this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) + session.reconcileNextFrame = false } destroy(): void { @@ -465,6 +539,8 @@ export class EcsFollowerAdapter { frameQueue: [], reconcileNextFrame: true, applying: false, + batchRecovery: { kind: 'idle' }, + liveSweepRecovery: { kind: 'idle' }, onNodesChanged: (_events): void => undefined, onLinksChanged: (_event): void => undefined } @@ -531,158 +607,344 @@ export class EcsFollowerAdapter { const linkId = resolveLinkMapKey(id) return linkId === null ? [] : [linkId] }) - const committed = session.mutations.batch(frameContext(update), (batch) => { - // A SubgraphNode host that is already live must never be rebuilt from - // its doc entry: `reconcileNode` (and delete + `addNode`) replaces the - // host's input list in place, which drops the `widgetId` / - // `_subgraphSlot` bindings its promoted widgets hang off, leaving the - // host with no widgets at all. Resync the host's scalar fields (title, - // mode, flags, properties, colors) and promoted values only; - // `readSemanticNode` has already keyed the values from the definition. - const isHost = (payload: SemanticNodePayload) => - definitions().has(payload.type) - const upsertNode = ( - payload: SemanticNodePayload, - mode: 'add' | 'reconcile' - ) => { - if (mode === 'add') batch.addNode(payload) - else if (isHost(payload)) batch.reconcileNodeFields(payload) - else batch.reconcileNode(payload) - } + const result = session.mutations.batchResult( + frameContext(update), + (batch) => + this.applyBatchChanges({ + batch, + session, + doc, + definitions, + reconcile, + removedLinkIds, + nodeActions, + replacedWidgetMaps, + replacedOpaqueWidgets, + changedNodeFields, + changedWidgets, + changedLinks + }) + ) - if (reconcile) { - const pendingDeletes = this.intent.pendingDeletes(session.workflowId) - const nodes = [...session.nodes.keys()] - .filter((id) => !pendingDeletes.has(id)) - .flatMap((id) => { - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - return payload ? [payload] : [] - }) - const links = excludeIncompatibleLinks( - [...session.links.keys()].flatMap((id) => { - const link = readSemanticLink( - doc, - id, - definitions(), - session.reportedErrors - ) - return link && - !pendingDeletes.has(String(link.originNodeId)) && - !pendingDeletes.has(String(link.targetNodeId)) - ? [link] - : [] - }), - session.reportedErrors - ) - batch.removeMissing( - nodes.map(({ id }) => toNodeId(id)), - links.map(({ id }) => id) - ) - for (const payload of nodes) upsertNode(payload, 'reconcile') - for (const link of links) batch.connect(link) - return - } + // The pending sets were snapshotted and cleared before `batch` ran, so a + // rejected batch (no scope, or validation failure) has already lost the + // incremental record of this frame. Arm a full reconcile for the next + // frame so the dropped edits are re-read from the doc instead of falling + // through to incremental handling that never revisits them. + const committed = result.kind === 'committed' + session.reconcileNextFrame = !committed + this.handleReconcileOutcome(session, update, result) + return committed + } - batch.removeLinks(removedLinkIds) - const payloads = new Map( - [...nodeActions] - .filter(([, action]) => action !== 'delete') - .map( - ([id]) => - [ - id, - readSemanticNode(doc, id, definitions, session.reportedErrors) - ] as const - ) - ) - for (const [id, action] of nodeActions) { - if (action === 'delete') { - batch.deleteNode(toNodeId(id)) - continue - } - const payload = payloads.get(id) - if (action === 'update' && !(payload && isHost(payload))) - batch.deleteNode(toNodeId(id)) - } - for (const [id, payload] of payloads) { - if (!payload) continue - upsertNode(payload, nodeActions.get(id) === 'add' ? 'add' : 'reconcile') - } - // A node whose widget storage was replaced wholesale, either the named - // `widgets` map or the positional `__widgets_opaque` array (cmp writes - // both in one transaction when a host's storage flips to opaque, and - // deletes the opaque array when it flips back), is re-read in full. - for (const id of new Set([ - ...replacedWidgetMaps, - ...replacedOpaqueWidgets - ])) { - if (nodeActions.has(id)) continue + private applyBatchChanges(args: BatchChanges): void { + if (args.reconcile) this.applyFullReconcile(args) + else this.applyIncrementalChanges(args) + } + + private applyFullReconcile({ + batch, + session, + doc, + definitions + }: Pick): void { + const pendingDeletes = this.intent.pendingDeletes(session.workflowId) + const nodes = [...session.nodes.keys()] + .filter((id) => !pendingDeletes.has(id)) + .flatMap((id) => { const payload = readSemanticNode( doc, id, definitions, session.reportedErrors ) - if (payload) upsertNode(payload, 'reconcile') + return payload ? [payload] : [] + }) + const links = excludeIncompatibleLinks( + [...session.links.keys()].flatMap((id) => { + const link = readSemanticLink( + doc, + id, + definitions(), + session.reportedErrors + ) + return link && + !pendingDeletes.has(String(link.originNodeId)) && + !pendingDeletes.has(String(link.targetNodeId)) + ? [link] + : [] + }), + session.reportedErrors + ) + batch.removeMissing( + nodes.map(({ id }) => toNodeId(id)), + links.map(({ id }) => id) + ) + for (const payload of nodes) + this.upsertNode(batch, payload, definitions, 'reconcile') + for (const link of links) batch.connect(link) + } + + private applyIncrementalChanges(args: BatchChanges): void { + const { batch, session, doc, definitions, nodeActions } = args + batch.removeLinks(args.removedLinkIds) + const payloads = new Map( + [...nodeActions] + .filter(([, action]) => action !== 'delete') + .map( + ([id]) => + [ + id, + readSemanticNode(doc, id, definitions, session.reportedErrors) + ] as const + ) + ) + this.applyNodeActions(batch, definitions, nodeActions, payloads) + this.upsertChangedNodes(batch, definitions, nodeActions, payloads) + this.applyChangedNodeValues(args) + const incomingLinks = excludeIncompatibleLinks( + [...args.changedLinks.values()].filter( + (link): link is SemanticLinkPayload => link !== null + ), + session.reportedErrors + ) + for (const link of incomingLinks) batch.connect(link) + } + + private applyNodeActions( + batch: GraphMutationBatch, + definitions: () => SubgraphDefinitionIndex, + nodeActions: Map, + payloads: Map + ): void { + for (const [id, action] of nodeActions) { + if (action === 'delete') { + batch.deleteNode(toNodeId(id)) + continue } - // A node whose scalar fields were edited by key (title, mode, flags, - // properties, colors) is re-read so a live host resyncs those fields - // without rebuilding its promoted widgets or slots. - for (const id of changedNodeFields) { - if ( - nodeActions.has(id) || - replacedWidgetMaps.has(id) || - replacedOpaqueWidgets.has(id) + const payload = payloads.get(id) + if (action === 'update' && !(payload && definitions().has(payload.type))) + batch.deleteNode(toNodeId(id)) + } + } + + private upsertChangedNodes( + batch: GraphMutationBatch, + definitions: () => SubgraphDefinitionIndex, + nodeActions: Map, + payloads: Map + ): void { + for (const [id, payload] of payloads) { + if (payload) + this.upsertNode( + batch, + payload, + definitions, + nodeActions.get(id) === 'add' ? 'add' : 'reconcile' ) - continue + } + } + + private applyChangedNodeValues(args: BatchChanges): void { + const { + batch, + session, + doc, + definitions, + nodeActions, + replacedWidgetMaps, + replacedOpaqueWidgets + } = args + for (const id of new Set([ + ...replacedWidgetMaps, + ...replacedOpaqueWidgets + ])) { + if (nodeActions.has(id)) continue + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + } + for (const id of args.changedNodeFields) { + if ( + nodeActions.has(id) || + replacedWidgetMaps.has(id) || + replacedOpaqueWidgets.has(id) + ) + continue + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + } + this.applyChangedWidgets(args) + } + + private applyChangedWidgets(args: BatchChanges): void { + const { + batch, + session, + doc, + definitions, + nodeActions, + replacedWidgetMaps + } = args + for (const [id, names] of args.changedWidgets) { + if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue + const widgets = session.nodes.get(id)?.get('widgets') + if (!(widgets instanceof Y.Map)) continue + if ([...names].some((name) => !widgets.has(name))) { const payload = readSemanticNode( doc, id, definitions, session.reportedErrors ) - if (payload) upsertNode(payload, 'reconcile') + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + continue } - for (const [id, names] of changedWidgets) { - if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue - const node = session.nodes.get(id) - const widgets = node?.get('widgets') - if (!(widgets instanceof Y.Map)) continue - if ([...names].some((name) => !widgets.has(name))) { - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors + for (const name of names) + batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) + } + } + + private upsertNode( + batch: GraphMutationBatch, + payload: SemanticNodePayload, + definitions: () => SubgraphDefinitionIndex, + mode: 'add' | 'reconcile' + ): void { + if (mode === 'add') batch.addNode(payload) + else if (definitions().has(payload.type)) batch.reconcileNodeFields(payload) + else batch.reconcileNode(payload) + } + + private handleReconcileOutcome( + session: TargetSession, + update: DocUpdate, + result: GraphMutationBatchResult + ): void { + if (result.kind === 'committed') { + this.clearReconcileRetry(session) + return + } + if (result.reason === 'validation') { + if (session.batchRecovery.kind === 'applying') + session.batchRecovery = { kind: 'idle' } + return + } + + if (session.batchRecovery.kind === 'scheduled') { + session.batchRecovery.frame = mergeRecoveryFrame( + session.batchRecovery.frame, + update + ) + return + } + if (session.batchRecovery.kind === 'applying') { + this.scheduleReconcileRetry( + session, + session.batchRecovery.frame, + session.batchRecovery.attempt + ) + return + } + this.scheduleReconcileRetry(session, update, 0) + } + + private scheduleReconcileRetry( + session: TargetSession, + frame: DocUpdate, + previousAttempts: number + ): void { + const attempt = previousAttempts + 1 + const delay = + attempt <= RECONCILE_FAST_RETRY_LIMIT + ? RECONCILE_RETRY_INTERVAL_MS + : RECONCILE_SLOW_RETRY_INTERVAL_MS + const scheduled: Extract = { + kind: 'scheduled', + timer: setTimeout(() => { + if (this.targets.get(session.workflowId) !== session) return + if (session.batchRecovery !== scheduled) return + if (!session.reconcileNextFrame) return + const applying = { + kind: 'applying', + attempt: scheduled.attempt, + frame: scheduled.frame + } satisfies BatchRecoveryState + session.batchRecovery = applying + let committed: boolean + try { + committed = this.applyFrame(applying.frame) + } catch (error) { + reportError( + error instanceof Error ? error : new Error(String(error)), + { + errorType: 'error_agent_reconcile_retry_threw', + context: { workflowId: session.workflowId } + } ) - if (payload) upsertNode(payload, 'reconcile') - continue + if (session.batchRecovery === applying) + this.scheduleReconcileRetry( + session, + applying.frame, + applying.attempt + ) + return } - for (const name of names) { - batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) + if (committed) this.runLiveGraphSweep(session) + }, delay), + attempt, + frame + } + session.batchRecovery = scheduled + } + + private runLiveGraphSweep(session: TargetSession): void { + if (session.liveSweepRecovery.kind === 'scheduled') + clearTimeout(session.liveSweepRecovery.timer) + const running = { kind: 'running' } satisfies LiveSweepRecoveryState + session.liveSweepRecovery = running + try { + this.onReconcileRetryCommitted(session.workflowId) + if (session.liveSweepRecovery === running) + session.liveSweepRecovery = { kind: 'idle' } + } catch (error) { + reportError(error instanceof Error ? error : new Error(String(error)), { + errorType: 'error_agent_reconcile_live_sweep_threw', + context: { workflowId: session.workflowId } + }) + if (session.liveSweepRecovery !== running) return + const scheduled: Extract = + { + kind: 'scheduled', + timer: setTimeout(() => { + if (this.targets.get(session.workflowId) !== session) return + if (session.liveSweepRecovery !== scheduled) return + this.runLiveGraphSweep(session) + }, RECONCILE_SLOW_RETRY_INTERVAL_MS) } - } - const incomingLinks = excludeIncompatibleLinks( - [...changedLinks.values()].filter( - (link): link is SemanticLinkPayload => link !== null - ), - session.reportedErrors - ) - for (const link of incomingLinks) batch.connect(link) - }) + session.liveSweepRecovery = scheduled + } + } - // The pending sets were snapshotted and cleared before `batch` ran, so a - // rejected batch (no scope, or validation failure) has already lost the - // incremental record of this frame. Arm a full reconcile for the next - // frame so the dropped edits are re-read from the doc instead of falling - // through to incremental handling that never revisits them. - session.reconcileNextFrame = !committed - return committed + private clearReconcileRetry(session: TargetSession): void { + if (session.batchRecovery.kind === 'scheduled') + clearTimeout(session.batchRecovery.timer) + session.batchRecovery = { kind: 'idle' } + } + + private clearLiveSweepRetry(session: TargetSession): void { + if (session.liveSweepRecovery.kind === 'scheduled') + clearTimeout(session.liveSweepRecovery.timer) + session.liveSweepRecovery = { kind: 'idle' } } private discardSessionPending(session: TargetSession): void { diff --git a/src/workbench/extensions/agent/crdt/followerSubscription.test.ts b/src/workbench/extensions/agent/crdt/followerSubscription.test.ts index 4150b1c4428..7b01aa9aa5e 100644 --- a/src/workbench/extensions/agent/crdt/followerSubscription.test.ts +++ b/src/workbench/extensions/agent/crdt/followerSubscription.test.ts @@ -29,7 +29,7 @@ import type { DocFrameTransport, DocOp, DocUpdate } from './docFrameClient' import { DocFrameClient, encodeBase64 } from './docFrameClient' import { EcsFollowerAdapter } from './ecsFollowerAdapter' import { FollowerDoc } from './followerDoc' -import type { GraphMutations } from './graphMutations' +import type { GraphMutationBatchResult, GraphMutations } from './graphMutations' import { LayoutFollowerBridge } from './layoutFollowerBridge' import { FollowerSchemaError, assertReadableSchema } from './schemaGuard' @@ -127,6 +127,12 @@ describe('follower commit boundary', () => { }) const mutations = { batch: vi.fn(() => false), + batchResult: vi.fn( + (): GraphMutationBatchResult => ({ + kind: 'rejected', + reason: 'validation' + }) + ), addNode: vi.fn(() => false), setWidget: vi.fn(() => false), connect: vi.fn(() => false), diff --git a/src/workbench/extensions/agent/crdt/graphMutations.ts b/src/workbench/extensions/agent/crdt/graphMutations.ts index 78a235a2d3e..9551cc3cd13 100644 --- a/src/workbench/extensions/agent/crdt/graphMutations.ts +++ b/src/workbench/extensions/agent/crdt/graphMutations.ts @@ -176,11 +176,19 @@ export interface GraphMutationBatch { clearSemanticGraph(): void } +export type GraphMutationBatchResult = + | { readonly kind: 'committed' } + | { readonly kind: 'rejected'; readonly reason: 'no-scope' | 'validation' } + export interface GraphMutations { batch( context: RemoteMutationContext, define: (batch: GraphMutationBatch) => void ): boolean + batchResult( + context: RemoteMutationContext, + define: (batch: GraphMutationBatch) => void + ): GraphMutationBatchResult addNode(payload: SemanticNodePayload, context: RemoteMutationContext): boolean setWidget( nodeId: NodeId, @@ -2301,54 +2309,65 @@ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { } } + const runBatch = ( + context: RemoteMutationContext, + define: (batch: GraphMutationBatch) => void + ): GraphMutationBatchResult => { + const scope = deps.getScope() + if (!scope) return { kind: 'rejected', reason: 'no-scope' } + const queued: QueuedMutation[] = [] + define({ + addNode(payload) { + queued.push({ kind: 'addNode', payload }) + }, + reconcileNode(payload) { + queued.push({ kind: 'reconcileNode', payload }) + }, + reconcileNodeFields(payload) { + queued.push({ kind: 'reconcileNodeFields', payload }) + }, + setWidget(nodeId, name, value) { + queued.push({ kind: 'setWidget', nodeId, name, value }) + }, + connect(link) { + queued.push({ kind: 'connect', link }) + }, + removeMissing(retainedNodeIds, retainedLinkIds) { + queued.push({ + kind: 'removeMissing', + retainedNodeIds, + retainedLinkIds + }) + }, + removeLinks(linkIds) { + queued.push({ kind: 'removeLinks', linkIds }) + }, + deleteNode(nodeId, removedLinkIds = []) { + queued.push({ kind: 'deleteNode', nodeId, removedLinkIds }) + }, + clearSemanticGraph() { + queued.push({ kind: 'clearSemanticGraph' }) + } + }) + const existingIds = nodeStore + .getGraphNodesFor(scope.rootGraphId, scope.owningGraphId) + .map((node) => node.id) + const memory = autogrowMemory.draft() + const prepared = prepare(memory, scope, queued) + if (typeof prepared === 'string') { + fail(prepared) + return { kind: 'rejected', reason: 'validation' } + } + offsetInsertedBatch(scope, existingIds, prepared) + commit(scope, prepared, context, memory) + return { kind: 'committed' } + } + const graphMutations: GraphMutations = { batch(context, define) { - const scope = deps.getScope() - if (!scope) return false - const queued: QueuedMutation[] = [] - define({ - addNode(payload) { - queued.push({ kind: 'addNode', payload }) - }, - reconcileNode(payload) { - queued.push({ kind: 'reconcileNode', payload }) - }, - reconcileNodeFields(payload) { - queued.push({ kind: 'reconcileNodeFields', payload }) - }, - setWidget(nodeId, name, value) { - queued.push({ kind: 'setWidget', nodeId, name, value }) - }, - connect(link) { - queued.push({ kind: 'connect', link }) - }, - removeMissing(retainedNodeIds, retainedLinkIds) { - queued.push({ - kind: 'removeMissing', - retainedNodeIds, - retainedLinkIds - }) - }, - removeLinks(linkIds) { - queued.push({ kind: 'removeLinks', linkIds }) - }, - deleteNode(nodeId, removedLinkIds = []) { - queued.push({ kind: 'deleteNode', nodeId, removedLinkIds }) - }, - clearSemanticGraph() { - queued.push({ kind: 'clearSemanticGraph' }) - } - }) - const existingIds = nodeStore - .getGraphNodesFor(scope.rootGraphId, scope.owningGraphId) - .map((node) => node.id) - const memory = autogrowMemory.draft() - const prepared = prepare(memory, scope, queued) - if (typeof prepared === 'string') return fail(prepared) - offsetInsertedBatch(scope, existingIds, prepared) - commit(scope, prepared, context, memory) - return true + return runBatch(context, define).kind === 'committed' }, + batchResult: runBatch, addNode(payload, context) { return graphMutations.batch(context, (batch) => batch.addNode(payload)) },