From 3a4b4a94870f894ce8a400b2725e5fecd743d941 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 07:33:37 +0000 Subject: [PATCH 01/18] test: add failing test for agent delete-all leaving stale nodes on canvas Reproduces PM-1500/PM-1504: when the agent's mutation batch is rejected because getScope() briefly can't resolve the bound workflow tab, the only existing recovery arms reconcileNextFrame for whatever incoming frame happens to land next. If no further frame ever arrives (the user doesn't send another agent message), the stale doc/live mismatch is never cleaned up. This test asserts the adapter recovers on its own once scope becomes available again, without a new frame arriving; it fails against current code since nothing retries proactively. --- .../ecsFollowerAdapter.integration.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index df4da353a61..5574c940cdf 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -379,6 +379,75 @@ describe('EcsFollowerAdapter integration', () => { host.destroy() }) + it('proactively reconciles once scope becomes available again, without waiting for a new frame', () => { + vi.useFakeTimers() + try { + const deleteLayouts = vi.fn() + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + const context = { + source: 'agent-remote' as const, + actor: 'local-hydration', + opId: 'local-seed' + } + scopeAvailable = true + mutations.addNode( + { + id: 99, + type: 'Sink', + widgets_values: { stale: 9 }, + inputs: [], + outputs: [] + }, + context + ) + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // First frame: the batch is rejected (no scope available yet), so the + // reconciliation is dropped and local-only node 99 survives. + scopeAvailable = false + deleteLayouts.mockClear() + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect( + useNodeDataStore() + .getGraphNodesFor('root', 'root') + .map(({ id }) => id) + ).toEqual([toNodeId(99)]) + + // Scope becomes available again, but no new frame ever arrives (e.g. + // the user never sends another agent message). The adapter must retry + // on its own instead of leaving the stale node stuck until some + // unrelated later frame happens to land. + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ opId: 'replay' }) + ) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + it('replays authoritative state through real mutations after a batch throws', () => { const mutations = createGraphMutations({ placement: inertPlacementPort, From 5f12a6445b2b15d32de3ebf004ca5280b98f778a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 07:34:52 +0000 Subject: [PATCH 02/18] fix: retry agent reconcile promptly after a scope race, not just on the next frame EcsFollowerAdapter.applyQueuedFrame already arms reconcileNextFrame when a batch is rejected for lack of scope, but the only trigger for that reconcile was whatever frame happened to arrive next. If the agent's delete-all lands during a brief getScope() race and the user never sends another message, the stale nodes stayed on the canvas indefinitely (a refresh doesn't help since it hits the same race). Add a bounded, self-driven retry: on a rejected batch, schedule a short timer that replays the pending reconcile directly, independent of any new incoming frame, until it commits or a retry budget is exhausted (falling back to the existing next-frame behavior). --- .../agent/crdt/ecsFollowerAdapter.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index f31b789d5d4..8fbe274cf1a 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -28,6 +28,16 @@ import type { FollowerDoc } from './followerDoc' type NodeRootAction = 'add' | 'update' | 'delete' +/** + * A rejected batch (no scope available) arms `reconcileNextFrame`, but that + * only fires on the next *incoming* frame, which may never arrive (e.g. the + * user never sends another agent message). These bound retries give the + * scope race a fast, self-driven chance to resolve instead of leaving the + * stale doc/live mismatch on screen until some unrelated later frame lands. + */ +const RECONCILE_RETRY_LIMIT = 20 +const RECONCILE_RETRY_INTERVAL_MS = 200 + /** * Node-map keys whose by-key edits trigger a field resync. Structural keys * (`inputs`, `outputs`, `pos`, `size`, widget storage) are excluded: slots @@ -356,6 +366,9 @@ interface TargetSession { onLinksChanged: (event: Y.YMapEvent) => void reconcileNextFrame: boolean applying: boolean + /** Pending fast-retry of a rejected batch; see {@link RECONCILE_RETRY_LIMIT}. */ + reconcileRetryTimer: ReturnType | null + reconcileRetryAttempt: number } /** @@ -383,6 +396,7 @@ export class EcsFollowerAdapter { unbind(workflowId: string): void { const session = this.targets.get(workflowId) if (!session) return + this.clearReconcileRetry(session) session.nodes.unobserveDeep(session.onNodesChanged) session.links.unobserve(session.onLinksChanged) this.targets.delete(workflowId) @@ -450,6 +464,8 @@ export class EcsFollowerAdapter { frameQueue: [], reconcileNextFrame: true, applying: false, + reconcileRetryTimer: null, + reconcileRetryAttempt: 0, onNodesChanged: (_events): void => undefined, onLinksChanged: (_event): void => undefined } @@ -665,9 +681,40 @@ export class EcsFollowerAdapter { // frame so the dropped edits are re-read from the doc instead of falling // through to incremental handling that never revisits them. session.reconcileNextFrame = !committed + if (committed) this.clearReconcileRetry(session) + else this.scheduleReconcileRetry(session) return committed } + /** + * Bounded fast-retry for a rejected batch: rather than only reconciling on + * whatever frame happens to arrive next (which may be a long time away, or + * never), replay the pending reconcile on a short timer until scope + * resolves or the retry budget runs out. The ordinary next-frame reconcile + * remains the fallback once the budget is exhausted. + */ + private scheduleReconcileRetry(session: TargetSession): void { + if (session.reconcileRetryTimer) return + if (session.reconcileRetryAttempt >= RECONCILE_RETRY_LIMIT) return + session.reconcileRetryAttempt += 1 + session.reconcileRetryTimer = setTimeout(() => { + session.reconcileRetryTimer = null + if (this.targets.get(session.workflowId) !== session) return + if (!session.reconcileNextFrame) return + this.applyFrame({ + workflowId: session.workflowId, + seq: -1, + update: new Uint8Array() + }) + }, RECONCILE_RETRY_INTERVAL_MS) + } + + private clearReconcileRetry(session: TargetSession): void { + if (session.reconcileRetryTimer) clearTimeout(session.reconcileRetryTimer) + session.reconcileRetryTimer = null + session.reconcileRetryAttempt = 0 + } + private discardSessionPending(session: TargetSession): void { session.nodeActions.clear() session.changedWidgets.clear() From 0dcf33e935fedfd61ca6cdec295228fd7eccc943 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 08:21:54 +0000 Subject: [PATCH 03/18] fix: close reconcile-retry gaps found in review (reset, scope-only, live sweep) Addresses 7 Cursor review findings against the self-driven reconcile retry from the previous commit: - clearForReset and discardPending now also call clearReconcileRetry and disarm reconcileNextFrame, so a pending retry can no longer fire after a lineage reset or a schema-rejected batch and resurrect state those paths deliberately cleared. - The retry now routes a successful commit through a new onReconcileRetryCommitted hook, which AgentCrdtProjection wires to reconcileLiveGraph. Previously the retry only updated the ECS stores via the adapter's own applyFrame, so the canvas was never swept and a stale node could still be saved even after the retry "succeeded". - The retry timer's applyFrame call is now wrapped in try/catch and reports via reportError instead of dying as an unhandled global error and silently killing the retry chain. - GraphMutations gains an optional hasScope() check; the adapter only arms a retry for a scope-unavailable rejection now, not for every batch() rejection (a deterministic prepare() validation failure can never be fixed by waiting). - reconcileRetryAttempt resets when a new rejection episode starts (a real frame rejected while no retry is currently pending), not only on success/unbind, so the 20-attempt budget no longer has to cover multiple unrelated episodes in one session. - The retry resubmits the actual rejected DocUpdate (same seq/actor/ opIds) instead of a synthetic seq: -1 frame with no attribution. Extends the adapter-level retry test to assert the new commit hook fires, adds a test proving a deterministic rejection never arms a retry, and adds a projection-level integration test that proves the live graph (not just the store) is actually swept once the retry recovers, closing the gap the previous test left. --- .../agentCrdtProjection.integration.test.ts | 85 ++++++++++++- .../agent/crdt/agentCrdtProjection.ts | 10 +- .../ecsFollowerAdapter.integration.test.ts | 116 +++++++++++++++++- .../agent/crdt/ecsFollowerAdapter.ts | 101 +++++++++++++-- .../extensions/agent/crdt/graphMutations.ts | 13 ++ 5 files changed, 315 insertions(+), 10 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index db2d68331b1..be1f54ff87a 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -1,4 +1,4 @@ -import { mint, nodesMap } from '@comfyorg/comfy-multi-player' +import { linksMap, mint, nodesMap } from '@comfyorg/comfy-multi-player' import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' import { beforeEach, describe, expect, it, vi } from 'vitest' import * as Y from 'yjs' @@ -247,3 +247,86 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { destroy() }) }) + +describe('AgentCrdtProjection self-driven reconcile retry', () => { + it('sweeps the stale live node once a retry recovers from a rejected delete-all, without a new frame arriving', () => { + vi.useFakeTimers() + try { + const { graph, source } = buildLiveGraph() + const scope = graphScopeOf(graph) + let scopeAvailable = true + const mutations = createGraphMutations({ + getScope: () => (scopeAvailable ? scope : null), + layout, + placement: inertPlacementPort + }) + const host = mint( + toWorkflowJson(structuredClone(graph.serialize())), + CATALOG + ) + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + mutations, + () => graph, + () => follower.doc + ) + projection.bind(WORKFLOW_ID, follower) + let seq = 0 + const deliver = (update: Uint8Array) => { + follower.applyRemoteUpdate(update) + const applied = projection.applyFrame({ + workflowId: WORKFLOW_ID, + seq: ++seq, + update, + actor: 'agent:comfy:host', + opIds: [] + }) + // Mirrors `useAgentCrdtFollower`'s `applyAndReconcile`: the live + // graph is only ever swept for a frame that actually applied. + if (applied) projection.reconcileLiveGraph(WORKFLOW_ID) + return applied + } + + // Catch-up frame: the doc starts out matching the live graph, so the + // pre-existing `source` node is adopted as the live adapter for + // record id 1. + deliver(Y.encodeStateAsUpdate(host)) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // The agent deletes every node from the doc (a delete-all), but scope + // briefly can't resolve the bound workflow tab, so the batch is + // rejected: nothing is swept, and the stale node stays live. + const before = Y.encodeStateVector(host) + host.transact(() => { + for (const id of [...nodesMap(host).keys()]) nodesMap(host).delete(id) + for (const id of [...linksMap(host).keys()]) linksMap(host).delete(id) + }) + scopeAvailable = false + const deleteAllUpdate = Y.encodeStateAsUpdate(host, before) + expect(deliver(deleteAllUpdate)).toBe(false) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Scope recovers, but no further frame ever arrives (e.g. the user + // never sends another agent message). The self-driven retry must + // both commit the store-side reconcile AND sweep the live graph + // through the same pipeline a normal frame gets, or the stale node + // would still be there to serialize into the next save. + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(graph.getNodeById(toNodeId(1))).toBeNull() + expect(layout.deleteNodes).toHaveBeenCalledWith( + scope, + [toNodeId(1)], + expect.anything() + ) + + projection.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index c7f9eac29ed..bae524cbd74 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -24,7 +24,15 @@ export class AgentCrdtProjection { private readonly getFollowerDoc: () => Y.Doc, intent?: LocalIntent ) { - this.adapter = new EcsFollowerAdapter(mutations, intent) + // The adapter's self-driven reconcile retry (a rejected batch armed for + // a bounded fast retry, see `EcsFollowerAdapter`) is store-only and has + // no notion of the live graph. Route a successful retry through the same + // `reconcileLiveGraph` sweep a normally-arriving frame gets via + // `applyAndReconcile`, or a retry that commits still leaves stale live + // nodes on screen (and savable) with nothing left to re-materialize them. + this.adapter = new EcsFollowerAdapter(mutations, intent, (workflowId) => + this.reconcileLiveGraph(workflowId) + ) } bind(workflowId: string, follower: FollowerDoc): void { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 5574c940cdf..3a787e4829f 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -408,7 +408,20 @@ describe('EcsFollowerAdapter integration', () => { const host = mint({ nodes: [], links: [] }, catalog) const follower = new FollowerDoc() - const adapter = new EcsFollowerAdapter(mutations) + // Store-only: the adapter has no notion of the live graph, so its + // owner (`AgentCrdtProjection`) is the one that would route a + // successful retry into `reconcileLiveGraph`. Assert the adapter + // actually invokes that seam once the retry commits, not just that + // the ECS store converges - a retry that only updates stores but + // never asks anything to sweep the canvas leaves a stale node + // rendered (and savable) even though this test's own store-level + // assertions below would still pass. + const onReconcileRetryCommitted = vi.fn() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) adapter.bind('wf', follower) const update = Y.encodeStateAsUpdate(host) follower.applyRemoteUpdate(update) @@ -425,6 +438,7 @@ describe('EcsFollowerAdapter integration', () => { .getGraphNodesFor('root', 'root') .map(({ id }) => id) ).toEqual([toNodeId(99)]) + expect(onReconcileRetryCommitted).not.toHaveBeenCalled() // Scope becomes available again, but no new frame ever arrives (e.g. // the user never sends another agent message). The adapter must retry @@ -439,6 +453,106 @@ describe('EcsFollowerAdapter integration', () => { [toNodeId(99)], expect.objectContaining({ opId: 'replay' }) ) + expect(onReconcileRetryCommitted).toHaveBeenCalledExactlyOnceWith('wf') + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('does not arm a self-driven retry for a deterministic rejection scope can never fix', () => { + vi.useFakeTimers() + try { + const host = mint( + { + nodes: [ + { + id: 1, + type: 'Source', + pos: [0, 0], + outputs: [{ name: 'out', type: 'IMAGE', links: [9] }], + inputs: [] + }, + { + id: 2, + type: 'Sink', + pos: [300, 0], + inputs: [{ name: 'in', type: 'IMAGE', link: 9 }], + outputs: [] + } + ], + links: [[9, 1, 0, 2, 0, 'IMAGE']] + }, + catalog + ) + const follower = new FollowerDoc() + const realMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => scope, + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const batchCalls: unknown[] = [] + const mutations: GraphMutations = { + ...realMutations, + batch: (context, define) => { + batchCalls.push(context) + return realMutations.batch(context, define) + } + } + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const bootstrap = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(bootstrap) + expect( + adapter.applyFrame({ workflowId: 'wf', seq: 1, update: bootstrap }) + ).toBe(true) + batchCalls.length = 0 + + // Retype node 1 to a type with no output, while the doc still carries + // link 9 originating from its old output slot 0: `prepare()`'s stale + // link revalidation rejects this deterministically (see the "FEC-4" + // test below). Scope stays available throughout, so this is exactly + // the case `hasScope()` must distinguish from a transient scope race. + const before = Y.encodeStateVector(host) + const retypeOp = { + op_id: 'retype', + actor: 'agent:test', + base_version: 2, + stamp: [2, 'agent:test'], + op: 'add_node', + node_id: 1, + class_type: 'Sink', + pos: [0, 0], + node: { + id: 1, + type: 'Sink', + pos: [0, 0], + inputs: [{ name: 'in', type: 'IMAGE', link: null }], + outputs: [] + } + } satisfies Op + applyOps(host, [retypeOp], catalog) + const update = Y.encodeStateAsUpdate(host, before) + follower.applyRemoteUpdate(update) + const rejected = vi + .spyOn(console, 'error') + .mockImplementation(() => undefined) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + rejected.mockRestore() + + // If the adapter blindly armed a retry for every rejection, this + // advance would fire it and call `batch()` again with the identical, + // still-invalid doc state - 20 wasted whole-document reconciles at a + // fixed interval that retrying can never turn into a commit. + vi.advanceTimersByTime(5_000) + expect(batchCalls).toHaveLength(1) adapter.destroy() follower.destroy() diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 8fbe274cf1a..4d73895120c 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -369,6 +369,20 @@ interface TargetSession { /** Pending fast-retry of a rejected batch; see {@link RECONCILE_RETRY_LIMIT}. */ reconcileRetryTimer: ReturnType | null reconcileRetryAttempt: number + /** + * The frame whose batch was most recently rejected, resubmitted verbatim + * (same actor/opIds/seq) by the retry timer instead of a synthetic frame: + * the reconcile branch never reads `update.update`, only `frameContext`, + * so replaying the original frame is both simpler and correctly attributed. + */ + lastRejectedFrame: DocUpdate | null + /** + * True while a retry timer's own resubmission is being applied, so the + * post-batch bookkeeping can tell "a fresh frame was rejected" (which may + * start a new retry episode) apart from "the retry's own attempt was + * rejected again" (which must not reset the retry budget). + */ + retryInFlight: boolean } /** @@ -382,7 +396,18 @@ export class EcsFollowerAdapter { constructor( private readonly mutations: MutationsForTarget, - private readonly intent: LocalIntent = NO_LOCAL_INTENT + private readonly intent: LocalIntent = NO_LOCAL_INTENT, + /** + * Called after a self-driven retry's batch commits. The adapter is + * store-only and has no notion of the live graph; this is how its owner + * (`AgentCrdtProjection`) plugs in the same `reconcileLiveGraph` sweep a + * normally-arriving frame gets via `applyAndReconcile`. Without it, a + * retry that commits still leaves stale live nodes on screen (and + * savable) since nothing ever re-materializes the canvas for it. + */ + private readonly onReconcileRetryCommitted: ( + workflowId: string + ) => void = () => undefined ) {} bind(workflowId: string, follower: FollowerDoc): void { @@ -429,12 +454,23 @@ export class EcsFollowerAdapter { const session = this.targets.get(workflowId) if (!session) return false this.discardSessionPending(session) + // A pending retry (or an armed `reconcileNextFrame`) would otherwise + // fire up to `RECONCILE_RETRY_INTERVAL_MS` later and re-project the + // pre-reset doc, resurrecting exactly the state this reset clears. + this.clearReconcileRetry(session) + session.reconcileNextFrame = false return session.mutations.clearSemanticGraph(context) } discardPending(workflowId: string): void { const session = this.targets.get(workflowId) - if (session) this.discardSessionPending(session) + if (!session) return + this.discardSessionPending(session) + // Same reasoning as `clearForReset`: a schema-rejected frame is a + // fail-closed gate, and a stale retry firing later would silently + // re-apply the very state that gate just refused. + this.clearReconcileRetry(session) + session.reconcileNextFrame = false } destroy(): void { @@ -466,6 +502,8 @@ export class EcsFollowerAdapter { applying: false, reconcileRetryTimer: null, reconcileRetryAttempt: 0, + lastRejectedFrame: null, + retryInFlight: false, onNodesChanged: (_events): void => undefined, onLinksChanged: (_event): void => undefined } @@ -681,8 +719,28 @@ export class EcsFollowerAdapter { // frame so the dropped edits are re-read from the doc instead of falling // through to incremental handling that never revisits them. session.reconcileNextFrame = !committed - if (committed) this.clearReconcileRetry(session) - else this.scheduleReconcileRetry(session) + if (committed) { + this.clearReconcileRetry(session) + } else { + // A rejection that arrives while no retry is already in flight starts + // a new rejection episode: reset the budget so an unrelated later + // rejection (or scope staying down longer than the previous episode's + // remaining budget) still gets the full retry window, instead of + // inheriting whatever was left over from an earlier, unrelated + // episode. `retryInFlight` excludes the retry's own resubmission + // (already counted against the budget when it was scheduled) from + // being mistaken for a new episode. + if (!session.retryInFlight && !session.reconcileRetryTimer) { + session.reconcileRetryAttempt = 0 + } + session.lastRejectedFrame = update + // `batch()` returns false both for a transient scope race and for a + // deterministic `prepare()` validation rejection; only the former can + // ever be fixed by waiting, so only arm the retry for it. + if (session.mutations.hasScope?.() !== true) { + this.scheduleReconcileRetry(session) + } + } return committed } @@ -701,11 +759,39 @@ export class EcsFollowerAdapter { session.reconcileRetryTimer = null if (this.targets.get(session.workflowId) !== session) return if (!session.reconcileNextFrame) return - this.applyFrame({ + // Resubmit the actual rejected frame (same actor/opIds/seq) rather + // than a synthetic one: the reconcile branch below never reads + // `update.update`, only `frameContext(update)`, so this both keeps + // `seq` in its normal domain and attributes the replayed commit to + // whichever op actually got rejected instead of a generic 'replay'. + const retryFrame = session.lastRejectedFrame ?? { workflowId: session.workflowId, - seq: -1, + seq: 0, update: new Uint8Array() - }) + } + session.retryInFlight = true + let committed: boolean + try { + committed = this.applyFrame(retryFrame) + } catch (error) { + // `applyFrame` deliberately propagates a throw from `batch()` (see + // "replays authoritative state through real mutations after a batch + // throws"), but this call has no caller to propagate to: it runs + // off a timer. Left uncaught it would become an unhandled global + // error, `reconcileRetryTimer` is already null, and the throw skips + // the `scheduleReconcileRetry` call below, silently killing the + // retry chain. Report it and keep retrying on the existing budget + // instead. + session.retryInFlight = false + reportError(error instanceof Error ? error : new Error(String(error)), { + errorType: 'error_agent_reconcile_retry_threw', + context: { workflowId: session.workflowId } + }) + this.scheduleReconcileRetry(session) + return + } + session.retryInFlight = false + if (committed) this.onReconcileRetryCommitted(session.workflowId) }, RECONCILE_RETRY_INTERVAL_MS) } @@ -713,6 +799,7 @@ export class EcsFollowerAdapter { if (session.reconcileRetryTimer) clearTimeout(session.reconcileRetryTimer) session.reconcileRetryTimer = null session.reconcileRetryAttempt = 0 + session.lastRejectedFrame = null } private discardSessionPending(session: TargetSession): void { diff --git a/src/workbench/extensions/agent/crdt/graphMutations.ts b/src/workbench/extensions/agent/crdt/graphMutations.ts index f36cafc40bd..1dbd5c76632 100644 --- a/src/workbench/extensions/agent/crdt/graphMutations.ts +++ b/src/workbench/extensions/agent/crdt/graphMutations.ts @@ -163,6 +163,16 @@ export interface GraphMutations { context: RemoteMutationContext, define: (batch: GraphMutationBatch) => void ): boolean + /** + * Cheap, side-effect-free check for whether `batch` would currently be + * rejected for lack of scope. Callers that retry a rejected `batch` (e.g. + * `EcsFollowerAdapter`'s self-driven reconcile retry) use this to tell a + * transient scope race, worth retrying, apart from a deterministic + * `prepare()` validation rejection that retrying can never fix. Optional + * so a test double that never rejects for scope reasons can omit it; a + * caller that only cares about `batch`'s boolean result is unaffected. + */ + hasScope?(): boolean addNode(payload: SemanticNodePayload, context: RemoteMutationContext): boolean setWidget( nodeId: NodeId, @@ -1353,6 +1363,9 @@ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { } const graphMutations: GraphMutations = { + hasScope() { + return deps.getScope() !== null + }, batch(context, define) { const scope = deps.getScope() if (!scope) return false From b5bbf7ba0ad9d81399ea0526b2631a21ca247ad4 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 12:43:56 -0700 Subject: [PATCH 04/18] fix(agent): keep scope-race reconcile recovery armed --- .../ecsFollowerAdapter.integration.test.ts | 62 +++++++++++++++++-- .../agent/crdt/ecsFollowerAdapter.ts | 31 ++++++---- .../extensions/agent/crdt/graphMutations.ts | 17 ++++- 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 659c95f3b6c..bde8b4c83a9 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -534,12 +534,17 @@ describe('EcsFollowerAdapter integration', () => { ).toEqual([toNodeId(99)]) expect(onReconcileRetryCommitted).not.toHaveBeenCalled() - // Scope becomes available again, but no new frame ever arrives (e.g. - // the user never sends another agent message). The adapter must retry - // on its own instead of leaving the stale node stuck until some - // unrelated later frame happens to land. - scopeAvailable = true + // Stay unavailable past the old 20-attempt/4-second retry budget. No + // new frame arrives, so recovery must remain armed at its slower rate. vi.advanceTimersByTime(5_000) + expect( + useNodeDataStore() + .getGraphNodesFor('root', 'root') + .map(({ id }) => id) + ).toEqual([toNodeId(99)]) + + scopeAvailable = true + vi.advanceTimersByTime(2_000) expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) expect(deleteLayouts).toHaveBeenCalledWith( @@ -557,6 +562,53 @@ describe('EcsFollowerAdapter integration', () => { } }) + it('retries from the batch rejection reason when scope appears immediately afterward', () => { + vi.useFakeTimers() + try { + const seedMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => scope, + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + seedMutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + { source: 'agent-remote', actor: 'seed', opId: 'seed' } + ) + + let scopeReads = 0 + const deleteLayouts = vi.fn() + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (++scopeReads === 1 ? null : scope), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + vi.advanceTimersByTime(200) + + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ opId: 'replay' }) + ) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + it('does not arm a self-driven retry for a deterministic rejection scope can never fix', () => { vi.useFakeTimers() try { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 475eb24f416..12e60155dd4 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -36,8 +36,9 @@ type NodeRootAction = 'add' | 'update' | 'delete' * scope race a fast, self-driven chance to resolve instead of leaving the * stale doc/live mismatch on screen until some unrelated later frame lands. */ -const RECONCILE_RETRY_LIMIT = 20 +const RECONCILE_FAST_RETRY_LIMIT = 20 const RECONCILE_RETRY_INTERVAL_MS = 200 +const RECONCILE_SLOW_RETRY_INTERVAL_MS = 2_000 /** * Node-map keys whose by-key edits trigger a field resync. Structural keys @@ -380,7 +381,7 @@ interface TargetSession { onLinksChanged: (event: Y.YMapEvent) => void reconcileNextFrame: boolean applying: boolean - /** Pending fast-retry of a rejected batch; see {@link RECONCILE_RETRY_LIMIT}. */ + /** Pending retry of a batch rejected while workflow scope is unavailable. */ reconcileRetryTimer: ReturnType | null reconcileRetryAttempt: number /** @@ -751,10 +752,13 @@ export class EcsFollowerAdapter { session.reconcileRetryAttempt = 0 } session.lastRejectedFrame = update - // `batch()` returns false both for a transient scope race and for a - // deterministic `prepare()` validation rejection; only the former can - // ever be fixed by waiting, so only arm the retry for it. - if (session.mutations.hasScope?.() !== true) { + // Read the rejection captured by the same `batch()` call so scope + // becoming available immediately afterward cannot hide the race. + const rejection = session.mutations.lastBatchRejection?.() + if ( + rejection === 'no-scope' || + (rejection === undefined && session.mutations.hasScope?.() !== true) + ) { this.scheduleReconcileRetry(session) } } @@ -762,16 +766,17 @@ export class EcsFollowerAdapter { } /** - * Bounded fast-retry for a rejected batch: rather than only reconciling on - * whatever frame happens to arrive next (which may be a long time away, or - * never), replay the pending reconcile on a short timer until scope - * resolves or the retry budget runs out. The ordinary next-frame reconcile - * remains the fallback once the budget is exhausted. + * Retry a scope-rejected batch quickly at first, then at a low steady rate + * until scope returns or the target is reset/unbound. This keeps recovery + * armed even when no later frame arrives. */ private scheduleReconcileRetry(session: TargetSession): void { if (session.reconcileRetryTimer) return - if (session.reconcileRetryAttempt >= RECONCILE_RETRY_LIMIT) return session.reconcileRetryAttempt += 1 + const delay = + session.reconcileRetryAttempt <= RECONCILE_FAST_RETRY_LIMIT + ? RECONCILE_RETRY_INTERVAL_MS + : RECONCILE_SLOW_RETRY_INTERVAL_MS session.reconcileRetryTimer = setTimeout(() => { session.reconcileRetryTimer = null if (this.targets.get(session.workflowId) !== session) return @@ -809,7 +814,7 @@ export class EcsFollowerAdapter { } session.retryInFlight = false if (committed) this.onReconcileRetryCommitted(session.workflowId) - }, RECONCILE_RETRY_INTERVAL_MS) + }, delay) } private clearReconcileRetry(session: TargetSession): void { diff --git a/src/workbench/extensions/agent/crdt/graphMutations.ts b/src/workbench/extensions/agent/crdt/graphMutations.ts index ba34910c950..8ad911107a1 100644 --- a/src/workbench/extensions/agent/crdt/graphMutations.ts +++ b/src/workbench/extensions/agent/crdt/graphMutations.ts @@ -181,6 +181,8 @@ export interface GraphMutations { context: RemoteMutationContext, define: (batch: GraphMutationBatch) => void ): boolean + /** Reason the most recent synchronous `batch` call returned `false`. */ + lastBatchRejection?(): 'no-scope' | 'validation' | null /** * Cheap, side-effect-free check for whether `batch` would currently be * rejected for lack of scope. Callers that retry a rejected `batch` (e.g. @@ -1275,6 +1277,7 @@ function createAutogrowMemory() { * provenance on every participating store action. */ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { + let lastBatchRejection: 'no-scope' | 'validation' | null = null const nodeStore = useNodeDataStore() const linkStore = useLinkStore() const linkPresentationStore = useLinkPresentationStore() @@ -2264,12 +2267,19 @@ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { } const graphMutations: GraphMutations = { + lastBatchRejection() { + return lastBatchRejection + }, hasScope() { return deps.getScope() !== null }, batch(context, define) { + lastBatchRejection = null const scope = deps.getScope() - if (!scope) return false + if (!scope) { + lastBatchRejection = 'no-scope' + return false + } const queued: QueuedMutation[] = [] define({ addNode(payload) { @@ -2309,7 +2319,10 @@ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { .map((node) => node.id) const memory = autogrowMemory.draft() const prepared = prepare(memory, scope, queued) - if (typeof prepared === 'string') return fail(prepared) + if (typeof prepared === 'string') { + lastBatchRejection = 'validation' + return fail(prepared) + } offsetInsertedBatch(scope, existingIds, prepared) commit(scope, prepared, context, memory) return true From 82e386db1ce347b79abdb2846f18968f532d00b3 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 12:50:14 -0700 Subject: [PATCH 05/18] test(agent): cover delete and clear canvas journeys --- .../agent/agentDeleteAndClearWorkflow.spec.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts new file mode 100644 index 00000000000..c31ec676e3b --- /dev/null +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -0,0 +1,40 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' + +test.describe( + 'Agent delete and clear operations', + { tag: ['@cloud', '@vue-nodes'] }, + () => { + test.describe('delete_node', () => { + test.use({ conversationCase: 'agent-rec-add-set-delete' }) + + test('removes the requested node while preserving the workflow', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + + await agentConversation.runTurns() + + await expect(agentConversation.vueNodes.nodes).toHaveCount(5) + await expect( + agentConversation.vueNodes.getNodeLocator('2785690574723683') + ).toHaveCount(0) + }) + }) + + test.describe('clear_canvas', () => { + test.use({ conversationCase: 'agent-rec-clear-workflow' }) + + test('leaves the canvas empty after clearing the workflow', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + + await agentConversation.runTurns() + + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + }) + }) + } +) From 5a7e813e5dadd29c83a3303b1834839a4de7266a Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 12:54:50 -0700 Subject: [PATCH 06/18] test(agent): satisfy full-repo lint for clear fixture --- .../agent/crdt/agentCrdtProjection.integration.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index dcc82be7f5a..a4f1abe89b3 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -332,8 +332,8 @@ describe('AgentCrdtProjection self-driven reconcile retry', () => { // rejected: nothing is swept, and the stale node stays live. const before = Y.encodeStateVector(host) host.transact(() => { - for (const id of [...nodesMap(host).keys()]) nodesMap(host).delete(id) - for (const id of [...linksMap(host).keys()]) linksMap(host).delete(id) + nodesMap(host).clear() + linksMap(host).clear() }) scopeAvailable = false const deleteAllUpdate = Y.encodeStateAsUpdate(host, before) From 255dbbbf3cda1103c861864c5b68002f8745ad9b Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 13:03:27 -0700 Subject: [PATCH 07/18] refactor(agent): isolate retry outcome handling --- .../agent/crdt/ecsFollowerAdapter.ts | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 12e60155dd4..aa90ffedd6c 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -737,32 +737,40 @@ export class EcsFollowerAdapter { // frame so the dropped edits are re-read from the doc instead of falling // through to incremental handling that never revisits them. session.reconcileNextFrame = !committed + this.handleReconcileOutcome(session, update, committed) + return committed + } + + private handleReconcileOutcome( + session: TargetSession, + update: DocUpdate, + committed: boolean + ): void { if (committed) { this.clearReconcileRetry(session) - } else { - // A rejection that arrives while no retry is already in flight starts - // a new rejection episode: reset the budget so an unrelated later - // rejection (or scope staying down longer than the previous episode's - // remaining budget) still gets the full retry window, instead of - // inheriting whatever was left over from an earlier, unrelated - // episode. `retryInFlight` excludes the retry's own resubmission - // (already counted against the budget when it was scheduled) from - // being mistaken for a new episode. - if (!session.retryInFlight && !session.reconcileRetryTimer) { - session.reconcileRetryAttempt = 0 - } - session.lastRejectedFrame = update - // Read the rejection captured by the same `batch()` call so scope - // becoming available immediately afterward cannot hide the race. - const rejection = session.mutations.lastBatchRejection?.() - if ( - rejection === 'no-scope' || - (rejection === undefined && session.mutations.hasScope?.() !== true) - ) { - this.scheduleReconcileRetry(session) - } + return + } + // A rejection that arrives while no retry is already in flight starts + // a new rejection episode: reset the budget so an unrelated later + // rejection (or scope staying down longer than the previous episode's + // remaining budget) still gets the full retry window, instead of + // inheriting whatever was left over from an earlier, unrelated + // episode. `retryInFlight` excludes the retry's own resubmission + // (already counted against the budget when it was scheduled) from + // being mistaken for a new episode. + if (!session.retryInFlight && !session.reconcileRetryTimer) { + session.reconcileRetryAttempt = 0 + } + session.lastRejectedFrame = update + // Read the rejection captured by the same `batch()` call so scope + // becoming available immediately afterward cannot hide the race. + const rejection = session.mutations.lastBatchRejection?.() + if ( + rejection === 'no-scope' || + (rejection === undefined && session.mutations.hasScope?.() !== true) + ) { + this.scheduleReconcileRetry(session) } - return committed } /** From f57e02de4b688832510dfa5842539b759602f422 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 20:51:38 +0000 Subject: [PATCH 08/18] fix(agent): keep live-graph sweep armed and close retry test gaps Address DrJKL's review on the self-driven reconcile retry: - Keep recovery armed until the live-graph sweep actually succeeds. The reconcile-retry callback ran outside the timer's try/catch after the store mutation had already committed; a throw there escaped globally and left a stale live node with nothing to repair it. The callback is now retried on its own (never replaying the committed batch) until it succeeds. - Require an explicit hasScope() === false before treating an unclassified rejection as a scope race, so a GraphMutations that implements neither optional classifier no longer retries forever on a deterministic false. - Stop a later rejection from overwriting the frame an already-armed retry will resubmit, which was corrupting the replayed commit's provenance. - Add regression coverage: cancellation via clearForReset/discardPending/ unbind, a timer-driven batch throw reaching the retry's own catch, fast/slow cadence boundaries and per-episode budget reset, distinctive frame provenance surviving a replay, and a full-stack test proving a live node is actually removed after a sweep that threw once. - Add a deterministic browser-test fixture that drops and restores the agent's workflow-tab binding around the recorded clear frame, so the scope-race regression is exercised for real instead of by a frame that never actually loses scope. - Correct the PR description's retry-cadence, next-frame-recovery, verification-count, and evidence-scope claims. --- .../fixtures/agentConversationFixture.ts | 8 +- .../agent/agentDeleteAndClearWorkflow.spec.ts | 94 +++++ .../agentCrdtProjection.integration.test.ts | 80 ++++ .../agent/crdt/agentCrdtProjection.ts | 7 +- .../ecsFollowerAdapter.integration.test.ts | 355 +++++++++++++++++- .../agent/crdt/ecsFollowerAdapter.ts | 64 +++- 6 files changed, 584 insertions(+), 24 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index b925e3cd630..8662e347865 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -341,7 +341,12 @@ export class AgentConversationHarness { async replayResponse( turn = 0, - beforeFirstGraphOps?: () => Promise + beforeFirstGraphOps?: () => Promise, + // Fires before every graph_ops entry (by its index in `response`), not + // just the first: a fixture that needs to act on a later graph_ops entry + // (e.g. the specific frame carrying a `clear` op) reads the entry's ops + // itself to decide whether this is the one it wants. + onGraphOps?: (index: number) => Promise ): Promise { const startedAt = Date.now() const response = this.conversation.turns[turn].response @@ -355,6 +360,7 @@ export class AgentConversationHarness { else { await this.hostSocket.waitForSubscribe() if (index === firstGraphOps) await beforeFirstGraphOps?.() + await onGraphOps?.(index) this.hostSocket.send(this.host.apply(entry.ops)) for (const id of Object.keys(this.host.graph().nodes)) this.seenIds.add(id) diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index c31ec676e3b..41e7a35f01c 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -1,7 +1,68 @@ +import type { Page } from '@playwright/test' import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +// Pinia store instance access, the pattern +// `browser_tests/tests/vueNodes/widgets/flux2ImagePromptRerun.spec.ts` uses: +// the store object itself cannot cross the evaluate boundary, so every call +// against it happens inside a single `page.evaluate`. +interface WorkflowTabBindingStore { + tabPathFor(workflowId: string): string | undefined + bind(workflowId: string, tabPath: string): void + unbindWorkflow(workflowId: string): void +} + +// Drops the agent's workflow-tab binding so `getScope()` (AgentPanelRoot.vue) +// resolves to null on the next batch, reproducing the same "can't resolve +// which tab this edit is bound to" race the retry recovers from — without +// touching the doc or sending another frame. Returns the tab path so the +// caller can restore the exact same binding afterward. +function dropWorkflowScope(page: Page, workflowId: string): Promise { + return page.evaluate((id) => { + const store = ( + document.getElementById('vue-app') as unknown as { + __vue_app__: { + config: { + globalProperties: { $pinia: { _s: Map } } + } + } + } + ).__vue_app__.config.globalProperties.$pinia._s.get( + 'agentWorkflowTabBinding' + ) as WorkflowTabBindingStore + const tabPath = store.tabPathFor(id) + if (tabPath === undefined) + throw new Error(`no bound tab path for workflow ${id}`) + store.unbindWorkflow(id) + return tabPath + }, workflowId) +} + +function restoreWorkflowScope( + page: Page, + workflowId: string, + tabPath: string +): Promise { + return page.evaluate( + ({ id, path }) => { + const store = ( + document.getElementById('vue-app') as unknown as { + __vue_app__: { + config: { + globalProperties: { $pinia: { _s: Map } } + } + } + } + ).__vue_app__.config.globalProperties.$pinia._s.get( + 'agentWorkflowTabBinding' + ) as WorkflowTabBindingStore + store.bind(id, path) + }, + { id: workflowId, path: tabPath } + ) +} + test.describe( 'Agent delete and clear operations', { tag: ['@cloud', '@vue-nodes'] }, @@ -35,6 +96,39 @@ test.describe( await expect(agentConversation.vueNodes.nodes).toHaveCount(0) }) + + test('clears the canvas on its own after the clear frame is rejected for a dropped workflow scope', async ({ + agentConversation, + page + }) => { + test.setTimeout(90_000) + const workflowId = agentConversation.conversation.workflow.id + let tabPath: string | undefined + + await agentConversation.sendPrompt(0) + await agentConversation.replayResponse(0, undefined, async (index) => { + const entry = agentConversation.conversation.turns[0].response[index] + if (entry.kind !== 'graph_ops') return + if (!entry.ops.some((op) => op.op === 'clear')) return + // The workflow tab binding drives production `getScope()` + // (AgentPanelRoot.vue): dropping it here reproduces the exact race + // this PR fixes without touching the doc or sending another frame. + tabPath = await dropWorkflowScope(page, workflowId) + }) + await agentConversation.waitForTurnComplete() + + // The clear batch was rejected for lack of scope: the seed workflow's + // nodes are still on screen. + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) + + if (tabPath === undefined) + throw new Error('the clear frame never armed the scope drop') + await restoreWorkflowScope(page, workflowId, tabPath) + + // The self-driven retry clears the canvas on its own; nothing sends + // another frame after this point. + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + }) }) } ) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index a4f1abe89b3..9a3be434839 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -363,4 +363,84 @@ describe('AgentCrdtProjection self-driven reconcile retry', () => { vi.useRealTimers() } }) + + it('removes the stale live node once a later timer retries a live-graph sweep that threw once', () => { + vi.useFakeTimers() + try { + const { graph, source } = buildLiveGraph() + const scope = graphScopeOf(graph) + let scopeAvailable = true + const mutations = createGraphMutations({ + getScope: () => (scopeAvailable ? scope : null), + layout, + placement: inertPlacementPort + }) + const host = mint( + toWorkflowJson(structuredClone(graph.serialize())), + CATALOG + ) + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + mutations, + () => graph, + () => follower.doc + ) + let seq = 0 + const deliver = (update: Uint8Array) => { + follower.applyRemoteUpdate(update) + const applied = projection.applyFrame({ + workflowId: WORKFLOW_ID, + seq: ++seq, + update, + actor: 'agent:comfy:host', + opIds: [] + }) + if (applied) projection.reconcileLiveGraph(WORKFLOW_ID) + return applied + } + + projection.bind(WORKFLOW_ID, follower) + deliver(Y.encodeStateAsUpdate(host)) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Only now does the sweep start throwing, so the catch-up frame above + // (a normal, non-retry commit) is unaffected. + const sweepFailure = new Error('live sweep failed') + const sweepSpy = vi.spyOn(projection, 'reconcileLiveGraph') + sweepSpy.mockImplementationOnce(() => { + throw sweepFailure + }) + + const before = Y.encodeStateVector(host) + host.transact(() => { + nodesMap(host).clear() + linksMap(host).clear() + }) + scopeAvailable = false + const deleteAllUpdate = Y.encodeStateAsUpdate(host, before) + expect(deliver(deleteAllUpdate)).toBe(false) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + + // Scope recovers: the retry's batch commits, but its live-graph sweep + // throws once. The store has already converged, but the throw must + // not be silently treated as done - the stale node is still live. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(graph.getNodeById(toNodeId(1))).toBe(source) + expect(sweepSpy).toHaveBeenCalledTimes(1) + + // A later timer retries only the sweep - never the already-committed + // mutation - and this time it actually removes the live node. + vi.advanceTimersByTime(200) + expect(graph.getNodeById(toNodeId(1))).toBeNull() + expect(sweepSpy).toHaveBeenCalledTimes(2) + + projection.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) }) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index 9fe0f014435..1fa048b0204 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -27,12 +27,7 @@ export class AgentCrdtProjection { private readonly getFollowerDoc: () => Y.Doc, intent?: LocalIntent ) { - // The adapter's self-driven reconcile retry (a rejected batch armed for - // a bounded fast retry, see `EcsFollowerAdapter`) is store-only and has - // no notion of the live graph. Route a successful retry through the same - // `reconcileLiveGraph` sweep a normally-arriving frame gets via - // `applyAndReconcile`, or a retry that commits still leaves stale live - // nodes on screen (and savable) with nothing left to re-materialize them. + // Wires the adapter's store-only retry callback to the live-graph sweep. this.adapter = new EcsFollowerAdapter(mutations, intent, (workflowId) => this.reconcileLiveGraph(workflowId) ) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index bde8b4c83a9..90be293c485 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -11,9 +11,11 @@ import * as Y from 'yjs' import { createGraphMutations } from './graphMutations' import { inertPlacementPort } from './__fixtures__/inertPlacementPort' import type { GraphMutations } from './graphMutations' +import type { reportError as reportErrorFn } from '@/platform/telemetry/reportError' import { useLinkStore } from '@/stores/linkStore' import { useNodeDataStore } from '@/stores/nodeDataStore' import { useWidgetValueStore } from '@/stores/widgetValueStore' +import type { RemoteMutationContext } from '@/types/graphMutationContext' import { toOwningGraphId, toRootGraphId } from '@/types/graphScopeId' import { toLinkId } from '@/types/linkId' import type { NodeId } from '@/types/nodeId' @@ -24,6 +26,14 @@ import type { DocUpdate } from './docFrameClient' import { EcsFollowerAdapter } from './ecsFollowerAdapter' import { FollowerDoc } from './followerDoc' +const telemetryState = vi.hoisted(() => ({ + reportError: vi.fn() +})) + +vi.mock(import('@/platform/telemetry/reportError'), () => ({ + reportError: telemetryState.reportError +})) + const catalog: WidgetCatalog = { types: { Source: { widget_order: ['seed', 'stale'] }, @@ -484,10 +494,10 @@ describe('EcsFollowerAdapter integration', () => { layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } }) const context = { - source: 'agent-remote' as const, + source: 'agent-remote', actor: 'local-hydration', opId: 'local-seed' - } + } satisfies RemoteMutationContext scopeAvailable = true mutations.addNode( { @@ -521,12 +531,21 @@ describe('EcsFollowerAdapter integration', () => { follower.applyRemoteUpdate(update) // First frame: the batch is rejected (no scope available yet), so the - // reconciliation is dropped and local-only node 99 survives. + // reconciliation is dropped and local-only node 99 survives. A + // distinctive actor/opIds prove the eventual replay is attributed to + // this frame, not the generic 'replay' fallback a synthetic frame + // would also satisfy. scopeAvailable = false deleteLayouts.mockClear() - expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( - false - ) + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 7, + update, + actor: 'agent:distinctive-actor', + opIds: ['distinctive-op-id'] + }) + ).toBe(false) expect( useNodeDataStore() .getGraphNodesFor('root', 'root') @@ -550,7 +569,11 @@ describe('EcsFollowerAdapter integration', () => { expect(deleteLayouts).toHaveBeenCalledWith( scope, [toNodeId(99)], - expect.objectContaining({ opId: 'replay' }) + expect.objectContaining({ + actor: 'agent:distinctive-actor', + opId: 'distinctive-op-id', + opIds: ['distinctive-op-id'] + }) ) expect(onReconcileRetryCommitted).toHaveBeenCalledExactlyOnceWith('wf') @@ -562,6 +585,324 @@ describe('EcsFollowerAdapter integration', () => { } }) + it('retries at a fixed 200ms cadence for exactly 20 attempts, then 2s, and a fresh episode restarts at 200ms', () => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'local-hydration', + opId: 'local-seed' + } satisfies RemoteMutationContext + scopeAvailable = true + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batch: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batch(batchContext, define) + } + } + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(trackedMutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // Episode 1: rejected for lack of scope, which never returns during + // the fast tier. + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + // Exactly 20 fast attempts at a fixed 200ms cadence. + vi.advanceTimersByTime(20 * 200) + expect(batchCalls).toHaveLength(21) + + // The slow tier is a fixed 2s, not another 200ms: a hot-loop-forever + // implementation would already have fired several more batches by the + // 1999ms mark. + vi.advanceTimersByTime(1_999) + expect(batchCalls).toHaveLength(21) + vi.advanceTimersByTime(1) + expect(batchCalls).toHaveLength(22) + + // Let episode 1 finally commit so none of its budget leaks into + // episode 2. + scopeAvailable = true + vi.advanceTimersByTime(2_000) + expect(batchCalls).toHaveLength(23) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + + // Episode 2: a fresh rejection gets the full fast budget back, so its + // first retry fires at 200ms again instead of inheriting episode 1's + // slow cadence or its spent attempt count. + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(24) + vi.advanceTimersByTime(199) + expect(batchCalls).toHaveLength(24) + vi.advanceTimersByTime(1) + expect(batchCalls).toHaveLength(25) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('reports and keeps retrying when a timer-driven retry batch itself throws', () => { + vi.useFakeTimers() + try { + telemetryState.reportError.mockClear() + let scopeAvailable = false + const realMutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'seed', + opId: 'seed' + } satisfies RemoteMutationContext + scopeAvailable = true + realMutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + scopeAvailable = false + + const failure = new Error('retry batch threw') + let batchCallCount = 0 + // Call 1 is the initial (caller-driven) rejection; call 2 is the + // timer's own first retry attempt - the one DrJKL's finding says the + // existing suite never reaches. Call 3 is the next retry, which must + // still fire on the existing budget afterward. + const throwingMutations: GraphMutations = { + ...realMutations, + batch: (batchContext, define) => { + batchCallCount += 1 + if (batchCallCount === 2) throw failure + return realMutations.batch(batchContext, define) + } + } + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(throwingMutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCallCount).toBe(1) + + // The timer's own retry throws. It must not escape as an unhandled + // error, and must report the exact failure and reschedule. + expect(() => vi.advanceTimersByTime(200)).not.toThrow() + expect(batchCallCount).toBe(2) + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( + failure, + { + errorType: 'error_agent_reconcile_retry_threw', + context: { workflowId: 'wf' } + } + ) + + // The chain survives the throw: the next timer-driven attempt commits + // and converges, without any new frame ever arriving. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(batchCallCount).toBe(3) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it('keeps recovery armed and never replays the committed batch when the live-graph sweep callback throws', () => { + vi.useFakeTimers() + try { + telemetryState.reportError.mockClear() + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const context = { + source: 'agent-remote', + actor: 'local-hydration', + opId: 'local-seed' + } satisfies RemoteMutationContext + scopeAvailable = true + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + context + ) + scopeAvailable = false + + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batch: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batch(batchContext, define) + } + } + const sweepFailure = new Error('live sweep failed') + let sweepCalls = 0 + const onReconcileRetryCommitted = vi.fn(() => { + sweepCalls += 1 + if (sweepCalls === 1) throw sweepFailure + }) + + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + trackedMutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + // The retry batch commits, but the live-graph sweep callback throws on + // its first call. The throw must not replay the already-committed + // mutation - only the sweep is retried. + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(batchCalls).toHaveLength(2) + expect(useNodeDataStore().getGraphNodesFor('root', 'root')).toEqual([]) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( + sweepFailure, + { + errorType: 'error_agent_reconcile_live_sweep_threw', + context: { workflowId: 'wf' } + } + ) + + // A later timer removes the live node's store record by retrying only + // the sweep, never the mutation batch. + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + expect(batchCalls).toHaveLength(2) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a no-scope retry armed before %s cannot mutate or callback after it', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const batchCalls: unknown[] = [] + const trackedMutations: GraphMutations = { + ...mutations, + batch: (batchContext, define) => { + batchCalls.push(batchContext) + return mutations.batch(batchContext, define) + } + } + const onReconcileRetryCommitted = vi.fn() + + const host = mint( + { + nodes: [{ id: 1, type: 'Source', pos: [0, 0], outputs: [] }], + links: [] + }, + catalog + ) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + trackedMutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // Reject for lack of scope, arming a fast retry. + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + expect(batchCalls).toHaveLength(1) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + batchCalls.length = 0 + + // Scope returning afterward must never resurrect the cleared, + // discarded, or unbound state: deleting the new cancellation calls + // would otherwise let this fire a batch (or the callback) later. + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(batchCalls).toHaveLength(0) + expect(onReconcileRetryCommitted).not.toHaveBeenCalled() + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + it('retries from the batch rejection reason when scope appears immediately afterward', () => { vi.useFakeTimers() try { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index aa90ffedd6c..a0dc28f7062 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -398,6 +398,12 @@ interface TargetSession { * rejected again" (which must not reset the retry budget). */ retryInFlight: boolean + /** + * Pending retry of the live-graph sweep after a committed retry batch's + * `onReconcileRetryCommitted` callback threw. The store mutation already + * committed by the time this fires, so only the sweep itself is retried. + */ + liveSweepRetryTimer: ReturnType | null } /** @@ -437,6 +443,7 @@ export class EcsFollowerAdapter { const session = this.targets.get(workflowId) if (!session) return this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) session.nodes.unobserveDeep(session.onNodesChanged) session.links.unobserve(session.onLinksChanged) this.targets.delete(workflowId) @@ -520,6 +527,7 @@ export class EcsFollowerAdapter { reconcileRetryAttempt: 0, lastRejectedFrame: null, retryInFlight: false, + liveSweepRetryTimer: null, onNodesChanged: (_events): void => undefined, onLinksChanged: (_event): void => undefined } @@ -750,24 +758,31 @@ export class EcsFollowerAdapter { this.clearReconcileRetry(session) return } - // A rejection that arrives while no retry is already in flight starts - // a new rejection episode: reset the budget so an unrelated later + // A rejection that arrives while no retry is already in flight starts a + // new rejection episode: reset the budget so an unrelated later // rejection (or scope staying down longer than the previous episode's // remaining budget) still gets the full retry window, instead of - // inheriting whatever was left over from an earlier, unrelated - // episode. `retryInFlight` excludes the retry's own resubmission - // (already counted against the budget when it was scheduled) from - // being mistaken for a new episode. + // inheriting whatever was left over from an earlier, unrelated episode. + // It also captures this episode's frame once: a later rejection that + // arrives while a retry is already armed must not overwrite the frame + // the pending timer will resubmit, or the replayed commit gets + // attributed to whichever frame happened to arrive last instead of the + // one that actually started the episode. `retryInFlight` excludes the + // retry's own resubmission (already counted against the budget, and not + // a new episode) from either of these. if (!session.retryInFlight && !session.reconcileRetryTimer) { session.reconcileRetryAttempt = 0 + session.lastRejectedFrame = update } - session.lastRejectedFrame = update // Read the rejection captured by the same `batch()` call so scope - // becoming available immediately afterward cannot hide the race. + // becoming available immediately afterward cannot hide the race. A + // `GraphMutations` that implements neither optional classifier must + // never be treated as a scope race: `hasScope` has to explicitly say + // `false` before a deterministic-looking rejection is retried. const rejection = session.mutations.lastBatchRejection?.() if ( rejection === 'no-scope' || - (rejection === undefined && session.mutations.hasScope?.() !== true) + (rejection === undefined && session.mutations.hasScope?.() === false) ) { this.scheduleReconcileRetry(session) } @@ -821,10 +836,34 @@ export class EcsFollowerAdapter { return } session.retryInFlight = false - if (committed) this.onReconcileRetryCommitted(session.workflowId) + if (committed) this.runLiveGraphSweep(session) }, delay) } + /** + * Sweeps the live graph for a retry batch that has already committed. + * `onReconcileRetryCommitted` can throw (its contract reaches extension + * `onRemoved` hooks), and by this point the store mutation is done and the + * retry state already cleared — replaying `applyFrame` would resubmit a + * batch that already succeeded. So a throw here only reschedules the sweep + * itself, keeping recovery armed until the live graph actually converges. + */ + private runLiveGraphSweep(session: TargetSession): void { + try { + this.onReconcileRetryCommitted(session.workflowId) + } catch (error) { + reportError(error instanceof Error ? error : new Error(String(error)), { + errorType: 'error_agent_reconcile_live_sweep_threw', + context: { workflowId: session.workflowId } + }) + session.liveSweepRetryTimer = setTimeout(() => { + session.liveSweepRetryTimer = null + if (this.targets.get(session.workflowId) !== session) return + this.runLiveGraphSweep(session) + }, RECONCILE_RETRY_INTERVAL_MS) + } + } + private clearReconcileRetry(session: TargetSession): void { if (session.reconcileRetryTimer) clearTimeout(session.reconcileRetryTimer) session.reconcileRetryTimer = null @@ -832,6 +871,11 @@ export class EcsFollowerAdapter { session.lastRejectedFrame = null } + private clearLiveSweepRetry(session: TargetSession): void { + if (session.liveSweepRetryTimer) clearTimeout(session.liveSweepRetryTimer) + session.liveSweepRetryTimer = null + } + private discardSessionPending(session: TargetSession): void { session.nodeActions.clear() session.changedWidgets.clear() From f15a70904c9bdc6dac7722c5abdc6ed871a10447 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 21:11:39 +0000 Subject: [PATCH 09/18] test(agent): wait for the add frame to land before dropping scope agentDeleteAndClearWorkflow.spec.ts sent the turn's add_node frame and, without waiting, immediately dropped the workflow scope and sent the clear frame right after it. hostSocket.send() does not wait for the page to receive and apply a frame, so the add could still be in flight when scope dropped, rejecting it too and leaving zero nodes on screen instead of the one the clear's rejection was supposed to leave behind. Wait for that node to render before dropping scope. --- .../tests/agent/agentDeleteAndClearWorkflow.spec.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index 41e7a35f01c..f1c3ef848fd 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -110,6 +110,13 @@ test.describe( const entry = agentConversation.conversation.turns[0].response[index] if (entry.kind !== 'graph_ops') return if (!entry.ops.some((op) => op.op === 'clear')) return + // The seed workflow is empty, so the only node on screen at this + // point is the one the turn's own earlier add_node frame placed. + // hostSocket.send() does not wait for the page to receive and apply + // that frame before this callback runs, so without this wait the + // scope drop below can land before the add does, rejecting it too + // and leaving nothing on screen to prove the clear's rejection with. + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) // The workflow tab binding drives production `getScope()` // (AgentPanelRoot.vue): dropping it here reproduces the exact race // this PR fixes without touching the doc or sending another frame. From 6159d0d11c470eb419f006e8411399b98d956841 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 14:29:21 -0700 Subject: [PATCH 10/18] fix(agent): cancel and throttle live sweep retries Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/18404#discussion_r4098123419 --- .../ecsFollowerAdapter.integration.test.ts | 58 ++++++++++++++++++- .../agent/crdt/ecsFollowerAdapter.ts | 4 +- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 90be293c485..6d1bd2da23a 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -816,7 +816,7 @@ describe('EcsFollowerAdapter integration', () => { // A later timer removes the live node's store record by retrying only // the sweep, never the mutation batch. - vi.advanceTimersByTime(200) + vi.advanceTimersByTime(2_000) expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) expect(batchCalls).toHaveLength(2) @@ -903,6 +903,62 @@ describe('EcsFollowerAdapter integration', () => { } ) + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a failed live sweep armed before %s cannot callback after it', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const onReconcileRetryCommitted = vi.fn(() => { + throw new Error('live sweep failed') + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + + vi.advanceTimersByTime(5_000) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + it('retries from the batch rejection reason when scope appears immediately afterward', () => { vi.useFakeTimers() try { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index a0dc28f7062..2d577b59880 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -480,6 +480,7 @@ export class EcsFollowerAdapter { // fire up to `RECONCILE_RETRY_INTERVAL_MS` later and re-project the // pre-reset doc, resurrecting exactly the state this reset clears. this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) session.reconcileNextFrame = false return session.mutations.clearSemanticGraph(context) } @@ -492,6 +493,7 @@ export class EcsFollowerAdapter { // fail-closed gate, and a stale retry firing later would silently // re-apply the very state that gate just refused. this.clearReconcileRetry(session) + this.clearLiveSweepRetry(session) session.reconcileNextFrame = false } @@ -860,7 +862,7 @@ export class EcsFollowerAdapter { session.liveSweepRetryTimer = null if (this.targets.get(session.workflowId) !== session) return this.runLiveGraphSweep(session) - }, RECONCILE_RETRY_INTERVAL_MS) + }, RECONCILE_SLOW_RETRY_INTERVAL_MS) } } From ca0bd3adb951565a798da8be64edeb5388334f71 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 14:37:01 -0700 Subject: [PATCH 11/18] test(agent): exercise scope-loss retry through tab closure Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/18404#discussion_r4098123416 --- .../agent/agentDeleteAndClearWorkflow.spec.ts | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index f1c3ef848fd..7eeb48fdcf1 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -2,6 +2,7 @@ import type { Page } from '@playwright/test' import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import type { WorkspaceStore } from '@e2e/types/globals' // Pinia store instance access, the pattern // `browser_tests/tests/vueNodes/widgets/flux2ImagePromptRerun.spec.ts` uses: @@ -117,20 +118,22 @@ test.describe( // scope drop below can land before the add does, rejecting it too // and leaving nothing on screen to prove the clear's rejection with. await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - // The workflow tab binding drives production `getScope()` - // (AgentPanelRoot.vue): dropping it here reproduces the exact race - // this PR fixes without touching the doc or sending another frame. + // Removing both resolution paths reproduces a real closed-tab race: + // the binding is gone and the workflow is no longer open, so the + // clear frame cannot resolve a graph scope. tabPath = await dropWorkflowScope(page, workflowId) + await agentConversation.topbar.closeWorkflowTab('Unsaved Workflow') }) - await agentConversation.waitForTurnComplete() - - // The clear batch was rejected for lack of scope: the seed workflow's - // nodes are still on screen. - await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - if (tabPath === undefined) throw new Error('the clear frame never armed the scope drop') - await restoreWorkflowScope(page, workflowId, tabPath) + const replacementPath = await page.evaluate( + () => + (window.app!.extensionManager as WorkspaceStore).workflow + .activeWorkflow?.path + ) + if (replacementPath === undefined) + throw new Error('closing the target did not create a replacement tab') + await restoreWorkflowScope(page, workflowId, replacementPath) // The self-driven retry clears the canvas on its own; nothing sends // another frame after this point. From a7a224686917803ee2a87058c45de23e682f5cda Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 24 Sep 2026 14:43:06 -0700 Subject: [PATCH 12/18] test(agent): lock retry provenance across interleaved frames Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/18404#discussion_r4098123410 --- .../ecsFollowerAdapter.integration.test.ts | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 6d1bd2da23a..04d82e96f08 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -828,6 +828,73 @@ describe('EcsFollowerAdapter integration', () => { } }) + it('keeps retry provenance from the frame that started the rejection episode', () => { + vi.useFakeTimers() + try { + let scopeAvailable = true + const deleteLayouts = vi.fn() + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: deleteLayouts } + }) + mutations.addNode( + { id: 99, type: 'Sink', inputs: [], outputs: [] }, + { source: 'agent-remote', actor: 'seed', opId: 'seed' } + ) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter(mutations) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + scopeAvailable = false + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 7, + update, + actor: 'agent:first-frame', + opIds: ['first-op'] + }) + ).toBe(false) + expect( + adapter.applyFrame({ + workflowId: 'wf', + seq: 8, + update, + actor: 'agent:later-frame', + opIds: ['later-op'] + }) + ).toBe(false) + + scopeAvailable = true + vi.advanceTimersByTime(200) + + expect(deleteLayouts).toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ + actor: 'agent:first-frame', + opId: 'first-op', + opIds: ['first-op'] + }) + ) + expect(deleteLayouts).not.toHaveBeenCalledWith( + scope, + [toNodeId(99)], + expect.objectContaining({ actor: 'agent:later-frame' }) + ) + + adapter.destroy() + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + }) + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( 'a no-scope retry armed before %s cannot mutate or callback after it', (lifecycleAction) => { From ce23c46231c7a63d99ba16d60b51761bac0e9946 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 22:12:47 +0000 Subject: [PATCH 13/18] fix: make live-sweep retry scheduling idempotent, fix red projection test - ecsFollowerAdapter.ts: a second, independent live-graph-sweep failure could overwrite `liveSweepRetryTimer` before the first sweep's timer fired, orphaning it so it could still call the sweep callback after unbind/clearForReset/discardPending believed recovery was cancelled. Scheduling is now idempotent: don't arm a new timer while one is already pending. - agentCrdtProjection.integration.test.ts: the live-sweep retry now runs on the 2s slow cadence, not the 200ms fast cadence; the test advanced only 200ms and was red. Advance 2s instead. - ecsFollowerAdapter.integration.test.ts: added a table-driven regression test for the overlapping-sweep-failure interleaving above. - browser_tests: rewrote the delete/clear regression to observe the same graph/node identity throughout instead of switching to a new, already- empty replacement tab, so it actually proves the clear was rejected and then recovered. Moved the Pinia store access into a typed fixture helper (agentWorkflowScopeFixture.ts) using the store's real return type instead of a hand-declared interface. - Fixed a stale "bound retries" comment and split the browser test into test.step phases. --- .../fixtures/agentWorkflowScopeFixture.ts | 86 ++++++++++++ .../agent/agentDeleteAndClearWorkflow.spec.ts | 131 ++++++------------ .../agentCrdtProjection.integration.test.ts | 6 +- .../ecsFollowerAdapter.integration.test.ts | 74 ++++++++++ .../agent/crdt/ecsFollowerAdapter.ts | 11 +- 5 files changed, 215 insertions(+), 93 deletions(-) create mode 100644 browser_tests/fixtures/agentWorkflowScopeFixture.ts diff --git a/browser_tests/fixtures/agentWorkflowScopeFixture.ts b/browser_tests/fixtures/agentWorkflowScopeFixture.ts new file mode 100644 index 00000000000..fb5b979e2d7 --- /dev/null +++ b/browser_tests/fixtures/agentWorkflowScopeFixture.ts @@ -0,0 +1,86 @@ +import type { Page } from '@playwright/test' + +import type { useAgentWorkflowTabBindingStore } from '@/workbench/extensions/agent/stores/agent/agentWorkflowTabBindingStore' + +type AgentWorkflowTabBindingStore = ReturnType< + typeof useAgentWorkflowTabBindingStore +> + +type ScopeAction = + | { kind: 'drop'; workflowId: string } + | { kind: 'restore'; workflowId: string; tabPath: string } + +/** + * Runs one binding action against the real, mounted `agentWorkflowTabBinding` + * Pinia store, the same store `AgentPanelRoot.vue`'s `getScope()` reads + * through `boundOrOpenWorkflowFor`. Store instances cannot cross the + * `page.evaluate` serialization boundary, so this is the single place that + * reaches into the app's private Pinia map (`$pinia._s`); callers below + * never touch it directly. + */ +function runScopeAction( + page: Page, + action: ScopeAction +): Promise { + return page.evaluate((action) => { + const vueApp = document.getElementById('vue-app') + if (vueApp === null) throw new Error('#vue-app is not mounted') + // Vue attaches `__vue_app__` to the mount element at runtime; no public + // type covers it, so this is the one unavoidable boundary cast. + const pinia = ( + vueApp as unknown as { + __vue_app__: { + config: { + globalProperties: { $pinia: { _s: Map } } + } + } + } + ).__vue_app__.config.globalProperties.$pinia + const store: unknown = pinia._s.get('agentWorkflowTabBinding') + if ( + typeof store !== 'object' || + store === null || + !('tabPathFor' in store) || + !('bind' in store) || + !('unbindWorkflow' in store) + ) + throw new Error('agentWorkflowTabBinding store is not mounted') + const binding = store as AgentWorkflowTabBindingStore + + if (action.kind === 'drop') { + const tabPath = binding.tabPathFor(action.workflowId) + if (tabPath === undefined) + throw new Error(`no bound tab path for workflow ${action.workflowId}`) + binding.unbindWorkflow(action.workflowId) + return tabPath + } + binding.bind(action.workflowId, action.tabPath) + return undefined + }, action) +} + +/** + * Drops the agent's workflow-tab binding so `getScope()` (AgentPanelRoot.vue) + * resolves to null on the next batch, reproducing the same "can't resolve + * which tab this edit is bound to" race the retry recovers from, without + * touching the doc or sending another frame. Returns the bound tab path so + * the caller can restore the exact same binding afterward. + */ +export async function dropWorkflowScope( + page: Page, + workflowId: string +): Promise { + const tabPath = await runScopeAction(page, { kind: 'drop', workflowId }) + if (tabPath === undefined) + throw new Error(`no bound tab path for workflow ${workflowId}`) + return tabPath +} + +/** Restores a workflow-tab binding previously dropped by `dropWorkflowScope`. */ +export async function restoreWorkflowScope( + page: Page, + workflowId: string, + tabPath: string +): Promise { + await runScopeAction(page, { kind: 'restore', workflowId, tabPath }) +} diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index 7eeb48fdcf1..5dc2814edb7 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -1,68 +1,10 @@ -import type { Page } from '@playwright/test' import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -import type { WorkspaceStore } from '@e2e/types/globals' - -// Pinia store instance access, the pattern -// `browser_tests/tests/vueNodes/widgets/flux2ImagePromptRerun.spec.ts` uses: -// the store object itself cannot cross the evaluate boundary, so every call -// against it happens inside a single `page.evaluate`. -interface WorkflowTabBindingStore { - tabPathFor(workflowId: string): string | undefined - bind(workflowId: string, tabPath: string): void - unbindWorkflow(workflowId: string): void -} - -// Drops the agent's workflow-tab binding so `getScope()` (AgentPanelRoot.vue) -// resolves to null on the next batch, reproducing the same "can't resolve -// which tab this edit is bound to" race the retry recovers from — without -// touching the doc or sending another frame. Returns the tab path so the -// caller can restore the exact same binding afterward. -function dropWorkflowScope(page: Page, workflowId: string): Promise { - return page.evaluate((id) => { - const store = ( - document.getElementById('vue-app') as unknown as { - __vue_app__: { - config: { - globalProperties: { $pinia: { _s: Map } } - } - } - } - ).__vue_app__.config.globalProperties.$pinia._s.get( - 'agentWorkflowTabBinding' - ) as WorkflowTabBindingStore - const tabPath = store.tabPathFor(id) - if (tabPath === undefined) - throw new Error(`no bound tab path for workflow ${id}`) - store.unbindWorkflow(id) - return tabPath - }, workflowId) -} - -function restoreWorkflowScope( - page: Page, - workflowId: string, - tabPath: string -): Promise { - return page.evaluate( - ({ id, path }) => { - const store = ( - document.getElementById('vue-app') as unknown as { - __vue_app__: { - config: { - globalProperties: { $pinia: { _s: Map } } - } - } - } - ).__vue_app__.config.globalProperties.$pinia._s.get( - 'agentWorkflowTabBinding' - ) as WorkflowTabBindingStore - store.bind(id, path) - }, - { id: workflowId, path: tabPath } - ) -} +import { + dropWorkflowScope, + restoreWorkflowScope +} from '@e2e/fixtures/agentWorkflowScopeFixture' test.describe( 'Agent delete and clear operations', @@ -106,38 +48,47 @@ test.describe( const workflowId = agentConversation.conversation.workflow.id let tabPath: string | undefined - await agentConversation.sendPrompt(0) - await agentConversation.replayResponse(0, undefined, async (index) => { - const entry = agentConversation.conversation.turns[0].response[index] - if (entry.kind !== 'graph_ops') return - if (!entry.ops.some((op) => op.op === 'clear')) return - // The seed workflow is empty, so the only node on screen at this - // point is the one the turn's own earlier add_node frame placed. - // hostSocket.send() does not wait for the page to receive and apply - // that frame before this callback runs, so without this wait the - // scope drop below can land before the add does, rejecting it too - // and leaving nothing on screen to prove the clear's rejection with. - await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - // Removing both resolution paths reproduces a real closed-tab race: - // the binding is gone and the workflow is no longer open, so the - // clear frame cannot resolve a graph scope. - tabPath = await dropWorkflowScope(page, workflowId) - await agentConversation.topbar.closeWorkflowTab('Unsaved Workflow') + await test.step('setup: drop scope as the clear frame arrives', async () => { + await agentConversation.sendPrompt(0) + await agentConversation.replayResponse( + 0, + undefined, + async (index) => { + const entry = + agentConversation.conversation.turns[0].response[index] + if (entry.kind !== 'graph_ops') return + if (!entry.ops.some((op) => op.op === 'clear')) return + // The seed workflow is empty, so the only node on screen at this + // point is the one the turn's own earlier add_node frame placed. + // hostSocket.send() does not wait for the page to receive and + // apply that frame before this callback runs, so without this + // wait the scope drop below can land before the add does, + // rejecting it too and leaving nothing on screen to prove the + // clear's rejection with. + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) + tabPath = await dropWorkflowScope(page, workflowId) + } + ) }) + if (tabPath === undefined) throw new Error('the clear frame never armed the scope drop') - const replacementPath = await page.evaluate( - () => - (window.app!.extensionManager as WorkspaceStore).workflow - .activeWorkflow?.path - ) - if (replacementPath === undefined) - throw new Error('closing the target did not create a replacement tab') - await restoreWorkflowScope(page, workflowId, replacementPath) + const droppedTabPath = tabPath - // The self-driven retry clears the canvas on its own; nothing sends - // another frame after this point. - await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + await test.step('rejection: the same graph keeps its stale node', async () => { + // Observing the same graph the whole way through (never switching + // to a different tab) is what proves the clear was actually + // rejected here, rather than trivially passing against an + // already-empty replacement canvas. + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) + }) + + await test.step('autonomous recovery: scope returns, no new frame is sent', async () => { + await restoreWorkflowScope(page, workflowId, droppedTabPath) + // The self-driven retry clears the same canvas on its own; nothing + // sends another frame after this point. + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + }) }) }) } diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index 9a3be434839..17b628fcad0 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -431,8 +431,10 @@ describe('AgentCrdtProjection self-driven reconcile retry', () => { expect(sweepSpy).toHaveBeenCalledTimes(1) // A later timer retries only the sweep - never the already-committed - // mutation - and this time it actually removes the live node. - vi.advanceTimersByTime(200) + // mutation - and this time it actually removes the live node. The + // live-sweep retry runs on the slow (2s) cadence, not the fast (200ms) + // batch-retry cadence. + vi.advanceTimersByTime(2_000) expect(graph.getNodeById(toNodeId(1))).toBeNull() expect(sweepSpy).toHaveBeenCalledTimes(2) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 04d82e96f08..3069eb0387c 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -1026,6 +1026,80 @@ describe('EcsFollowerAdapter integration', () => { } ) + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( + 'a second, later live-sweep failure cannot orphan an earlier sweep-retry timer past %s', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const onReconcileRetryCommitted = vi.fn(() => { + throw new Error('live sweep failed') + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + // First rejection episode: its batch retry commits at 200ms, and its + // live-graph sweep throws, arming a sweep-retry timer 2s out. + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + // A second, independent rejection episode starts and its own retry + // commits well before the first sweep-retry timer would fire; its + // sweep also throws. Without idempotent scheduling this overwrites + // `liveSweepRetryTimer`, orphaning the first timer so nothing can + // ever `clearTimeout` it again. + scopeAvailable = false + expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(200) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else if (lifecycleAction === 'discardPending') { + adapter.discardPending('wf') + } else { + adapter.unbind('wf') + } + + // If the first sweep-retry timer were orphaned, it would still fire + // ~2s after the first sweep failure and invoke the callback again + // even though recovery was just cancelled. + vi.advanceTimersByTime(5_000) + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(2) + + if (lifecycleAction !== 'unbind') adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + it('retries from the batch rejection reason when scope appears immediately afterward', () => { vi.useFakeTimers() try { diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 2d577b59880..2ee938e8d09 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -32,7 +32,7 @@ type NodeRootAction = 'add' | 'update' | 'delete' /** * A rejected batch (no scope available) arms `reconcileNextFrame`, but that * only fires on the next *incoming* frame, which may never arrive (e.g. the - * user never sends another agent message). These bound retries give the + * user never sends another agent message). This timed retry loop gives the * scope race a fast, self-driven chance to resolve instead of leaving the * stale doc/live mismatch on screen until some unrelated later frame lands. */ @@ -858,6 +858,15 @@ export class EcsFollowerAdapter { errorType: 'error_agent_reconcile_live_sweep_threw', context: { workflowId: session.workflowId } }) + // A timer from an earlier failed sweep may already be armed for this + // session (e.g. a second, independent rejection episode committed and + // failed its own sweep before the first sweep's retry fired). + // Overwriting `liveSweepRetryTimer` here would orphan that earlier + // timer: it stays scheduled but nothing can ever `clearTimeout` it + // again, so it fires later even after `unbind`/`clearForReset`/ + // `discardPending` believed they had cancelled recovery. Scheduling is + // therefore idempotent: only arm a new timer when none is pending. + if (session.liveSweepRetryTimer) return session.liveSweepRetryTimer = setTimeout(() => { session.liveSweepRetryTimer = null if (this.targets.get(session.workflowId) !== session) return From 1e0e21eb658a23b9df0c14a9a79c8e631dcefc97 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 22:43:35 +0000 Subject: [PATCH 14/18] fix(agent): drop the cloud-name fallback too when simulating scope loss dropWorkflowScope only removed the agentWorkflowTabBinding record, but useAgentWorkflowResolver's resolveWorkflow() falls back to re-deriving the same scope by matching the still-open tab's cloud name -- the exact path AgentPanelRoot.vue's onAgentActiveTab relies on to auto-bind a tab on first activation. With the tab still open, that fallback silently handed getScope() a valid scope again, so the clear frame committed instead of being rejected. The pre-rewrite version of this test avoided this by closing the tab outright ("removing both resolution paths"), but the rewrite dropped that call so the same graph could be observed throughout per review feedback, without noticing the fallback still resolves it. Fix: also rename the tab while scope is dropped, breaking the cloud-name match without closing the tab or touching the doc, and restore the original filename alongside the binding. Verified locally against a built cloud dist + preview server (matching the failing CI job's SLOW_MO=250/RECORD_VIDEO=true flags): reproduced the failure deterministically before the fix, 4/4 clean runs after. --- .../fixtures/agentWorkflowScopeFixture.ts | 71 ++++++++++++++----- .../agent/agentDeleteAndClearWorkflow.spec.ts | 11 +-- 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/browser_tests/fixtures/agentWorkflowScopeFixture.ts b/browser_tests/fixtures/agentWorkflowScopeFixture.ts index fb5b979e2d7..b27211b890e 100644 --- a/browser_tests/fixtures/agentWorkflowScopeFixture.ts +++ b/browser_tests/fixtures/agentWorkflowScopeFixture.ts @@ -1,27 +1,35 @@ import type { Page } from '@playwright/test' import type { useAgentWorkflowTabBindingStore } from '@/workbench/extensions/agent/stores/agent/agentWorkflowTabBindingStore' +import type { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' type AgentWorkflowTabBindingStore = ReturnType< typeof useAgentWorkflowTabBindingStore > +type WorkflowStore = ReturnType + +/** What `dropWorkflowScope` needs to hand back so the drop can be undone. */ +export interface DroppedScope { + tabPath: string + filename: string +} type ScopeAction = | { kind: 'drop'; workflowId: string } - | { kind: 'restore'; workflowId: string; tabPath: string } + | { kind: 'restore'; workflowId: string; tabPath: string; filename: string } /** * Runs one binding action against the real, mounted `agentWorkflowTabBinding` - * Pinia store, the same store `AgentPanelRoot.vue`'s `getScope()` reads - * through `boundOrOpenWorkflowFor`. Store instances cannot cross the - * `page.evaluate` serialization boundary, so this is the single place that - * reaches into the app's private Pinia map (`$pinia._s`); callers below - * never touch it directly. + * and `workflow` Pinia stores, the same stores `AgentPanelRoot.vue`'s + * `getScope()` reads through `boundOrOpenWorkflowFor`. Store instances cannot + * cross the `page.evaluate` serialization boundary, so this is the single + * place that reaches into the app's private Pinia map (`$pinia._s`); callers + * below never touch it directly. */ function runScopeAction( page: Page, action: ScopeAction -): Promise { +): Promise { return page.evaluate((action) => { const vueApp = document.getElementById('vue-app') if (vueApp === null) throw new Error('#vue-app is not mounted') @@ -47,40 +55,65 @@ function runScopeAction( throw new Error('agentWorkflowTabBinding store is not mounted') const binding = store as AgentWorkflowTabBindingStore + const workflowStoreRaw: unknown = pinia._s.get('workflow') + if ( + typeof workflowStoreRaw !== 'object' || + workflowStoreRaw === null || + !('getWorkflowByPath' in workflowStoreRaw) + ) + throw new Error('workflow store is not mounted') + const workflows = workflowStoreRaw as WorkflowStore + if (action.kind === 'drop') { const tabPath = binding.tabPathFor(action.workflowId) if (tabPath === undefined) throw new Error(`no bound tab path for workflow ${action.workflowId}`) + const tab = workflows.getWorkflowByPath(tabPath) + if (!tab) throw new Error(`no open tab at ${tabPath} to drop scope from`) + const filename = tab.filename binding.unbindWorkflow(action.workflowId) - return tabPath + // Dropping the binding alone is not enough: `boundOrOpenWorkflowFor` + // (useAgentWorkflowResolver's `resolveWorkflow`) falls back to + // re-deriving the same scope by matching this still-open tab's name + // against the cloud index, exactly as it does on a tab's first + // activation. Renaming the tab breaks that name match too, so both + // resolution paths are down while the tab (and its canvas) stays open + // and observable -- the closed-tab race this reproduces, without + // actually closing the tab. + tab.filename = `${filename}\u0000scope-dropped` + return { tabPath, filename } } + + const tab = workflows.getWorkflowByPath(action.tabPath) + if (tab) tab.filename = action.filename binding.bind(action.workflowId, action.tabPath) return undefined }, action) } /** - * Drops the agent's workflow-tab binding so `getScope()` (AgentPanelRoot.vue) - * resolves to null on the next batch, reproducing the same "can't resolve - * which tab this edit is bound to" race the retry recovers from, without - * touching the doc or sending another frame. Returns the bound tab path so - * the caller can restore the exact same binding afterward. + * Drops the agent's workflow-tab binding, and the tab's own cloud-name match, + * so `getScope()` (AgentPanelRoot.vue) resolves to null on the next batch, + * reproducing the same "can't resolve which tab this edit is bound to" race + * the retry recovers from, without touching the doc or sending another + * frame. Returns the tab path and original filename so the caller can + * restore the exact same binding afterward. */ export async function dropWorkflowScope( page: Page, workflowId: string -): Promise { - const tabPath = await runScopeAction(page, { kind: 'drop', workflowId }) - if (tabPath === undefined) +): Promise { + const dropped = await runScopeAction(page, { kind: 'drop', workflowId }) + if (dropped === undefined) throw new Error(`no bound tab path for workflow ${workflowId}`) - return tabPath + return dropped } /** Restores a workflow-tab binding previously dropped by `dropWorkflowScope`. */ export async function restoreWorkflowScope( page: Page, workflowId: string, - tabPath: string + dropped: DroppedScope ): Promise { - await runScopeAction(page, { kind: 'restore', workflowId, tabPath }) + await runScopeAction(page, { kind: 'restore', workflowId, ...dropped }) } diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index 5dc2814edb7..26a814ad93a 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -1,6 +1,7 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import type { DroppedScope } from '@e2e/fixtures/agentWorkflowScopeFixture' import { dropWorkflowScope, restoreWorkflowScope @@ -46,7 +47,7 @@ test.describe( }) => { test.setTimeout(90_000) const workflowId = agentConversation.conversation.workflow.id - let tabPath: string | undefined + let dropped: DroppedScope | undefined await test.step('setup: drop scope as the clear frame arrives', async () => { await agentConversation.sendPrompt(0) @@ -66,14 +67,14 @@ test.describe( // rejecting it too and leaving nothing on screen to prove the // clear's rejection with. await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - tabPath = await dropWorkflowScope(page, workflowId) + dropped = await dropWorkflowScope(page, workflowId) } ) }) - if (tabPath === undefined) + if (dropped === undefined) throw new Error('the clear frame never armed the scope drop') - const droppedTabPath = tabPath + const droppedScope = dropped await test.step('rejection: the same graph keeps its stale node', async () => { // Observing the same graph the whole way through (never switching @@ -84,7 +85,7 @@ test.describe( }) await test.step('autonomous recovery: scope returns, no new frame is sent', async () => { - await restoreWorkflowScope(page, workflowId, droppedTabPath) + await restoreWorkflowScope(page, workflowId, droppedScope) // The self-driven retry clears the same canvas on its own; nothing // sends another frame after this point. await expect(agentConversation.vueNodes.nodes).toHaveCount(0) From a0feb23959f6802ae006f6fde34761624266f200 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 22:55:46 +0000 Subject: [PATCH 15/18] refactor(test): extract branches in agentWorkflowScopeFixture to cut complexity runScopeAction's page.evaluate callback tripped Fallow's CRAP threshold (cyclomatic 14, CRAP 56.3). Split the pinia lookup, store type-guards, and the drop/restore actions into small nested helper functions (kept nested since Playwright serializes the evaluate callback via toString(), so helpers must live in its closure). No behavior change: drop still clears the tab binding and renames the tab, restore still restores both. --- .../fixtures/agentWorkflowScopeFixture.ts | 100 +++++++++++------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/browser_tests/fixtures/agentWorkflowScopeFixture.ts b/browser_tests/fixtures/agentWorkflowScopeFixture.ts index b27211b890e..c991a37e134 100644 --- a/browser_tests/fixtures/agentWorkflowScopeFixture.ts +++ b/browser_tests/fixtures/agentWorkflowScopeFixture.ts @@ -31,47 +31,60 @@ function runScopeAction( action: ScopeAction ): Promise { return page.evaluate((action) => { - const vueApp = document.getElementById('vue-app') - if (vueApp === null) throw new Error('#vue-app is not mounted') - // Vue attaches `__vue_app__` to the mount element at runtime; no public - // type covers it, so this is the one unavoidable boundary cast. - const pinia = ( - vueApp as unknown as { - __vue_app__: { - config: { - globalProperties: { $pinia: { _s: Map } } + function getPinia(): { _s: Map } { + const vueApp = document.getElementById('vue-app') + if (vueApp === null) throw new Error('#vue-app is not mounted') + // Vue attaches `__vue_app__` to the mount element at runtime; no public + // type covers it, so this is the one unavoidable boundary cast. + return ( + vueApp as unknown as { + __vue_app__: { + config: { + globalProperties: { $pinia: { _s: Map } } + } } } - } - ).__vue_app__.config.globalProperties.$pinia - const store: unknown = pinia._s.get('agentWorkflowTabBinding') - if ( - typeof store !== 'object' || - store === null || - !('tabPathFor' in store) || - !('bind' in store) || - !('unbindWorkflow' in store) - ) - throw new Error('agentWorkflowTabBinding store is not mounted') - const binding = store as AgentWorkflowTabBindingStore + ).__vue_app__.config.globalProperties.$pinia + } + + function getBindingStore(pinia: { + _s: Map + }): AgentWorkflowTabBindingStore { + const store: unknown = pinia._s.get('agentWorkflowTabBinding') + if ( + typeof store !== 'object' || + store === null || + !('tabPathFor' in store) || + !('bind' in store) || + !('unbindWorkflow' in store) + ) + throw new Error('agentWorkflowTabBinding store is not mounted') + return store as AgentWorkflowTabBindingStore + } - const workflowStoreRaw: unknown = pinia._s.get('workflow') - if ( - typeof workflowStoreRaw !== 'object' || - workflowStoreRaw === null || - !('getWorkflowByPath' in workflowStoreRaw) - ) - throw new Error('workflow store is not mounted') - const workflows = workflowStoreRaw as WorkflowStore + function getWorkflows(pinia: { _s: Map }): WorkflowStore { + const workflowStoreRaw: unknown = pinia._s.get('workflow') + if ( + typeof workflowStoreRaw !== 'object' || + workflowStoreRaw === null || + !('getWorkflowByPath' in workflowStoreRaw) + ) + throw new Error('workflow store is not mounted') + return workflowStoreRaw as WorkflowStore + } - if (action.kind === 'drop') { - const tabPath = binding.tabPathFor(action.workflowId) + function dropScope( + binding: AgentWorkflowTabBindingStore, + workflows: WorkflowStore, + workflowId: string + ): DroppedScope { + const tabPath = binding.tabPathFor(workflowId) if (tabPath === undefined) - throw new Error(`no bound tab path for workflow ${action.workflowId}`) + throw new Error(`no bound tab path for workflow ${workflowId}`) const tab = workflows.getWorkflowByPath(tabPath) if (!tab) throw new Error(`no open tab at ${tabPath} to drop scope from`) const filename = tab.filename - binding.unbindWorkflow(action.workflowId) + binding.unbindWorkflow(workflowId) // Dropping the binding alone is not enough: `boundOrOpenWorkflowFor` // (useAgentWorkflowResolver's `resolveWorkflow`) falls back to // re-deriving the same scope by matching this still-open tab's name @@ -84,9 +97,24 @@ function runScopeAction( return { tabPath, filename } } - const tab = workflows.getWorkflowByPath(action.tabPath) - if (tab) tab.filename = action.filename - binding.bind(action.workflowId, action.tabPath) + function restoreScope( + binding: AgentWorkflowTabBindingStore, + workflows: WorkflowStore, + restore: Extract + ): void { + const tab = workflows.getWorkflowByPath(restore.tabPath) + if (tab) tab.filename = restore.filename + binding.bind(restore.workflowId, restore.tabPath) + } + + const pinia = getPinia() + const binding = getBindingStore(pinia) + const workflows = getWorkflows(pinia) + + if (action.kind === 'drop') + return dropScope(binding, workflows, action.workflowId) + + restoreScope(binding, workflows, action) return undefined }, action) } From 5691f1b52931dc3f99c0ad708d7ded63a6fe4296 Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 25 Sep 2026 01:28:57 +0000 Subject: [PATCH 16/18] fix: harden agent scope recovery Amp-Thread-ID: https://ampcode.com/threads/T-01a0d508-3739-7789-b9d9-61a373b24746 --- .../fixtures/agentConversationFixture.ts | 171 +++++- .../fixtures/agentWorkflowScopeFixture.ts | 147 ----- .../agent/agentDeleteAndClearWorkflow.spec.ts | 97 ++-- .../agent/agentTabSwitchRunSubmission.spec.ts | 34 +- browser_tests/types/globals.d.ts | 6 + .../agent/crdt/agentCrdtProjection.ts | 1 - .../ecsFollowerAdapter.integration.test.ts | 101 +++- .../agent/crdt/ecsFollowerAdapter.ts | 532 +++++++++--------- .../agent/crdt/followerSubscription.test.ts | 8 +- .../extensions/agent/crdt/graphMutations.ts | 135 +++-- 10 files changed, 613 insertions(+), 619 deletions(-) delete mode 100644 browser_tests/fixtures/agentWorkflowScopeFixture.ts diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index d2328dcceab..13f7ae48b24 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -6,6 +6,7 @@ import { z } from 'zod' import { createI18n } from 'vue-i18n' import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import type { ComfyNodeDef, ObjectInfoResponse } from '@/schemas/nodeDefSchema' import { toNodeId } from '@/types/nodeId' import type { @@ -16,12 +17,14 @@ import type { import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' import { parseServerDocFrame } from '@/workbench/extensions/agent/crdt/docFrameClient' import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' +import type { useAgentWorkflowTabBindingStore } from '@/workbench/extensions/agent/stores/agent/agentWorkflowTabBindingStore' import { agentTest, bootAgentApp, mockWorkflowPersistence } from '@e2e/fixtures/agentPanelFixture' +import type { HostFrame } from '@e2e/fixtures/agentConversationHostDoc' import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' import { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' import type { @@ -170,6 +173,35 @@ async function withTimeout( } } +type WorkflowBindingActions = Pick< + ReturnType, + 'tabPathFor' +> + +type WorkflowLookup = Pick< + ReturnType, + 'getWorkflowByPath' +> + +type WorkflowScopeAction = + | { kind: 'drop'; workflowId: string } + | { + kind: 'restore' + tabPath: string + graphId: string + } + +interface DroppedWorkflowScope { + tabPath: string + graphId: string +} + +interface ReplayResponseOptions { + beforeFirstGraphOps?: () => Promise + beforeGraphOps?: (ops: readonly RecordedGraphOperation[]) => Promise + waitForGraphOpsDelivery?: (ops: readonly RecordedGraphOperation[]) => boolean +} + // Runs one recorded prompt/response through the real panel over a routed /ws socket. export class AgentConversationHarness { readonly panel: Locator @@ -341,12 +373,7 @@ export class AgentConversationHarness { async replayResponse( turn = 0, - beforeFirstGraphOps?: () => Promise, - // Fires before every graph_ops entry (by its index in `response`), not - // just the first: a fixture that needs to act on a later graph_ops entry - // (e.g. the specific frame carrying a `clear` op) reads the entry's ops - // itself to decide whether this is the one it wants. - onGraphOps?: (index: number) => Promise + options: ReplayResponseOptions = {} ): Promise { const startedAt = Date.now() const response = this.conversation.turns[turn].response @@ -359,9 +386,12 @@ export class AgentConversationHarness { this.hostSocket.send(this.stampTurn(entry.event, turn)) else { await this.hostSocket.waitForSubscribe() - if (index === firstGraphOps) await beforeFirstGraphOps?.() - await onGraphOps?.(index) - this.hostSocket.send(this.host.apply(entry.ops)) + if (index === firstGraphOps) await options.beforeFirstGraphOps?.() + await options.beforeGraphOps?.(entry.ops) + await this.sendHostFrame( + this.host.apply(entry.ops), + options.waitForGraphOpsDelivery?.(entry.ops) === true + ) for (const id of Object.keys(this.host.graph().nodes)) this.seenIds.add(id) } @@ -376,7 +406,7 @@ export class AgentConversationHarness { for (const turn of this.conversation.turns.keys()) { const before = await this.panelCounts() await this.sendPrompt(turn) - await this.replayResponse(turn, beforeFirstGraphOps) + await this.replayResponse(turn, { beforeFirstGraphOps }) await this.waitForTurnComplete() await this.expectTurnRendered(turn, before) await this.expectCanvasReplayed(turn) @@ -929,25 +959,93 @@ export class AgentConversationHarness { await this.selectWorkflowTarget() } - async resyncWidget(nodeId: string, widget: string): Promise { - const widgets = z - .record(z.string(), z.unknown()) - .optional() - .parse(this.host.graph().nodes[nodeId]?.widgets) - const value = widgets?.[widget] - if (value === undefined) - throw new Error(`Host widget ${nodeId}.${widget} does not exist`) - const operation = { - op: 'set_widget', - node_id: nodeId, - widget, - value, - old: value - } satisfies GraphOperation - const frame = this.host.apply([operation]) + async dropWorkflowScope(): Promise<() => Promise> { + const workflowId = this.conversation.workflow.id + const dropped = await this.runWorkflowScopeAction({ + kind: 'drop', + workflowId + }) + if (!dropped) + throw new Error(`no bound tab path for workflow ${workflowId}`) + + let restored = false + return async () => { + if (restored) return + await this.runWorkflowScopeAction({ kind: 'restore', ...dropped }) + restored = true + } + } + + async rememberRecoveryGraph(): Promise { + await this.page.evaluate(() => { + if (!window.app?.rootGraph) throw new Error('root graph is unavailable') + window.__agentRecoveryGraph = window.app.rootGraph + }) + } + + async isRecoveryGraphUnchanged(): Promise { + return await this.page.evaluate( + () => window.__agentRecoveryGraph === window.app?.rootGraph + ) + } + + private async runWorkflowScopeAction( + action: WorkflowScopeAction + ): Promise { + return await this.page.evaluate((action) => { + const isBinding = (value: unknown): value is WorkflowBindingActions => + typeof value === 'object' && + value !== null && + 'tabPathFor' in value && + typeof value.tabPathFor === 'function' + const isWorkflowLookup = (value: unknown): value is WorkflowLookup => + typeof value === 'object' && + value !== null && + 'getWorkflowByPath' in value && + typeof value.getWorkflowByPath === 'function' + + const mount = document.getElementById('vue-app') + const pinia = mount?.__vue_app__?.config.globalProperties.$pinia + if (!pinia) throw new Error('Pinia is not mounted') + const stores = [...pinia._s.values()] + const binding = stores.find(isBinding) + const workflows = stores.find(isWorkflowLookup) + if (!binding) + throw new Error('agentWorkflowTabBinding store is not mounted') + if (!workflows) throw new Error('workflow store is not mounted') + + if (action.kind === 'drop') { + const tabPath = binding.tabPathFor(action.workflowId) + if (!tabPath) + throw new Error(`no bound tab path for workflow ${action.workflowId}`) + const tab = workflows.getWorkflowByPath(tabPath) + if (!tab) throw new Error(`no open tab at ${tabPath}`) + const graphId = tab.activeState?.id + if (!graphId) throw new Error(`workflow at ${tabPath} has no graph id`) + delete tab.activeState?.id + return { tabPath, graphId } + } + + const tab = workflows.getWorkflowByPath(action.tabPath) + if (!tab) throw new Error(`no open tab at ${action.tabPath}`) + if (!tab.activeState) + throw new Error(`workflow at ${action.tabPath} has no active state`) + tab.activeState.id = action.graphId + return undefined + }, action) + } + + private async sendHostFrame( + frame: HostFrame, + waitForDelivery = false + ): Promise { + if (!waitForDelivery) { + this.hostSocket.send(frame) + return + } const parsedFrame = parseServerDocFrame(frame) if (parsedFrame?.type !== 'doc_update') - throw new Error('Host widget resync did not produce a doc_update') + throw new Error('Host operation did not produce a doc_update') const receipt = crypto.randomUUID() await this.page.evaluate( @@ -999,6 +1097,25 @@ export class AgentConversationHarness { }, receipt) } } + + async resyncWidget(nodeId: string, widget: string): Promise { + const widgets = z + .record(z.string(), z.unknown()) + .optional() + .parse(this.host.graph().nodes[nodeId]?.widgets) + const value = widgets?.[widget] + if (value === undefined) + throw new Error(`Host widget ${nodeId}.${widget} does not exist`) + const operation = { + op: 'set_widget', + node_id: nodeId, + widget, + value, + old: value + } satisfies GraphOperation + const frame = this.host.apply([operation]) + await this.sendHostFrame(frame, true) + } } export type ReplayTiming = 'immediate' | 'recorded' diff --git a/browser_tests/fixtures/agentWorkflowScopeFixture.ts b/browser_tests/fixtures/agentWorkflowScopeFixture.ts deleted file mode 100644 index c991a37e134..00000000000 --- a/browser_tests/fixtures/agentWorkflowScopeFixture.ts +++ /dev/null @@ -1,147 +0,0 @@ -import type { Page } from '@playwright/test' - -import type { useAgentWorkflowTabBindingStore } from '@/workbench/extensions/agent/stores/agent/agentWorkflowTabBindingStore' -import type { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' - -type AgentWorkflowTabBindingStore = ReturnType< - typeof useAgentWorkflowTabBindingStore -> -type WorkflowStore = ReturnType - -/** What `dropWorkflowScope` needs to hand back so the drop can be undone. */ -export interface DroppedScope { - tabPath: string - filename: string -} - -type ScopeAction = - | { kind: 'drop'; workflowId: string } - | { kind: 'restore'; workflowId: string; tabPath: string; filename: string } - -/** - * Runs one binding action against the real, mounted `agentWorkflowTabBinding` - * and `workflow` Pinia stores, the same stores `AgentPanelRoot.vue`'s - * `getScope()` reads through `boundOrOpenWorkflowFor`. Store instances cannot - * cross the `page.evaluate` serialization boundary, so this is the single - * place that reaches into the app's private Pinia map (`$pinia._s`); callers - * below never touch it directly. - */ -function runScopeAction( - page: Page, - action: ScopeAction -): Promise { - return page.evaluate((action) => { - function getPinia(): { _s: Map } { - const vueApp = document.getElementById('vue-app') - if (vueApp === null) throw new Error('#vue-app is not mounted') - // Vue attaches `__vue_app__` to the mount element at runtime; no public - // type covers it, so this is the one unavoidable boundary cast. - return ( - vueApp as unknown as { - __vue_app__: { - config: { - globalProperties: { $pinia: { _s: Map } } - } - } - } - ).__vue_app__.config.globalProperties.$pinia - } - - function getBindingStore(pinia: { - _s: Map - }): AgentWorkflowTabBindingStore { - const store: unknown = pinia._s.get('agentWorkflowTabBinding') - if ( - typeof store !== 'object' || - store === null || - !('tabPathFor' in store) || - !('bind' in store) || - !('unbindWorkflow' in store) - ) - throw new Error('agentWorkflowTabBinding store is not mounted') - return store as AgentWorkflowTabBindingStore - } - - function getWorkflows(pinia: { _s: Map }): WorkflowStore { - const workflowStoreRaw: unknown = pinia._s.get('workflow') - if ( - typeof workflowStoreRaw !== 'object' || - workflowStoreRaw === null || - !('getWorkflowByPath' in workflowStoreRaw) - ) - throw new Error('workflow store is not mounted') - return workflowStoreRaw as WorkflowStore - } - - function dropScope( - binding: AgentWorkflowTabBindingStore, - workflows: WorkflowStore, - workflowId: string - ): DroppedScope { - const tabPath = binding.tabPathFor(workflowId) - if (tabPath === undefined) - throw new Error(`no bound tab path for workflow ${workflowId}`) - const tab = workflows.getWorkflowByPath(tabPath) - if (!tab) throw new Error(`no open tab at ${tabPath} to drop scope from`) - const filename = tab.filename - binding.unbindWorkflow(workflowId) - // Dropping the binding alone is not enough: `boundOrOpenWorkflowFor` - // (useAgentWorkflowResolver's `resolveWorkflow`) falls back to - // re-deriving the same scope by matching this still-open tab's name - // against the cloud index, exactly as it does on a tab's first - // activation. Renaming the tab breaks that name match too, so both - // resolution paths are down while the tab (and its canvas) stays open - // and observable -- the closed-tab race this reproduces, without - // actually closing the tab. - tab.filename = `${filename}\u0000scope-dropped` - return { tabPath, filename } - } - - function restoreScope( - binding: AgentWorkflowTabBindingStore, - workflows: WorkflowStore, - restore: Extract - ): void { - const tab = workflows.getWorkflowByPath(restore.tabPath) - if (tab) tab.filename = restore.filename - binding.bind(restore.workflowId, restore.tabPath) - } - - const pinia = getPinia() - const binding = getBindingStore(pinia) - const workflows = getWorkflows(pinia) - - if (action.kind === 'drop') - return dropScope(binding, workflows, action.workflowId) - - restoreScope(binding, workflows, action) - return undefined - }, action) -} - -/** - * Drops the agent's workflow-tab binding, and the tab's own cloud-name match, - * so `getScope()` (AgentPanelRoot.vue) resolves to null on the next batch, - * reproducing the same "can't resolve which tab this edit is bound to" race - * the retry recovers from, without touching the doc or sending another - * frame. Returns the tab path and original filename so the caller can - * restore the exact same binding afterward. - */ -export async function dropWorkflowScope( - page: Page, - workflowId: string -): Promise { - const dropped = await runScopeAction(page, { kind: 'drop', workflowId }) - if (dropped === undefined) - throw new Error(`no bound tab path for workflow ${workflowId}`) - return dropped -} - -/** Restores a workflow-tab binding previously dropped by `dropWorkflowScope`. */ -export async function restoreWorkflowScope( - page: Page, - workflowId: string, - dropped: DroppedScope -): Promise { - await runScopeAction(page, { kind: 'restore', workflowId, ...dropped }) -} diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index 26a814ad93a..99602c2dee1 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -1,11 +1,7 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -import type { DroppedScope } from '@e2e/fixtures/agentWorkflowScopeFixture' -import { - dropWorkflowScope, - restoreWorkflowScope -} from '@e2e/fixtures/agentWorkflowScopeFixture' +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' test.describe( 'Agent delete and clear operations', @@ -19,12 +15,9 @@ test.describe( }) => { test.setTimeout(90_000) + await agentConversation.rememberRecoveryGraph() await agentConversation.runTurns() - - await expect(agentConversation.vueNodes.nodes).toHaveCount(5) - await expect( - agentConversation.vueNodes.getNodeLocator('2785690574723683') - ).toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe(true) }) }) @@ -36,60 +29,56 @@ test.describe( }) => { test.setTimeout(90_000) + await agentConversation.rememberRecoveryGraph() await agentConversation.runTurns() - - await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe(true) }) test('clears the canvas on its own after the clear frame is rejected for a dropped workflow scope', async ({ - agentConversation, - page + agentConversation }) => { test.setTimeout(90_000) - const workflowId = agentConversation.conversation.workflow.id - let dropped: DroppedScope | undefined + let restoreScope: (() => Promise) | undefined + const isClear = (ops: readonly RecordedGraphOperation[]) => + ops.some((op) => op.op === 'clear') - await test.step('setup: drop scope as the clear frame arrives', async () => { - await agentConversation.sendPrompt(0) - await agentConversation.replayResponse( - 0, - undefined, - async (index) => { - const entry = - agentConversation.conversation.turns[0].response[index] - if (entry.kind !== 'graph_ops') return - if (!entry.ops.some((op) => op.op === 'clear')) return - // The seed workflow is empty, so the only node on screen at this - // point is the one the turn's own earlier add_node frame placed. - // hostSocket.send() does not wait for the page to receive and - // apply that frame before this callback runs, so without this - // wait the scope drop below can land before the add does, - // rejecting it too and leaving nothing on screen to prove the - // clear's rejection with. - await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - dropped = await dropWorkflowScope(page, workflowId) - } - ) - }) + await agentConversation.rememberRecoveryGraph() + try { + await test.step('deliver the clear while workflow scope is unavailable', async () => { + await agentConversation.sendPrompt(0) + await agentConversation.replayResponse(0, { + beforeGraphOps: async (ops) => { + if (!isClear(ops)) return + await expect(agentConversation.vueNodes.nodes).not.toHaveCount( + 0 + ) + restoreScope = await agentConversation.dropWorkflowScope() + }, + waitForGraphOpsDelivery: isClear + }) + }) - if (dropped === undefined) - throw new Error('the clear frame never armed the scope drop') - const droppedScope = dropped + if (!restoreScope) + throw new Error('the clear frame never dropped workflow scope') + const restore = restoreScope - await test.step('rejection: the same graph keeps its stale node', async () => { - // Observing the same graph the whole way through (never switching - // to a different tab) is what proves the clear was actually - // rejected here, rather than trivially passing against an - // already-empty replacement canvas. - await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) - }) + await test.step('keep the same graph stale after the rejected clear', async () => { + await expect(agentConversation.vueNodes.nodes).not.toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe( + true + ) + }) - await test.step('autonomous recovery: scope returns, no new frame is sent', async () => { - await restoreWorkflowScope(page, workflowId, droppedScope) - // The self-driven retry clears the same canvas on its own; nothing - // sends another frame after this point. - await expect(agentConversation.vueNodes.nodes).toHaveCount(0) - }) + await test.step('clear without another frame when scope returns', async () => { + await restore() + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + expect(await agentConversation.isRecoveryGraphUnchanged()).toBe( + true + ) + }) + } finally { + await restoreScope?.() + } }) }) } diff --git a/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts b/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts index 40538e12b07..e7f4f13e765 100644 --- a/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts +++ b/browser_tests/tests/agent/agentTabSwitchRunSubmission.spec.ts @@ -39,22 +39,24 @@ test.describe( await agentConversation.sendPrompt() - await agentConversation.replayResponse(0, async () => { - await test.step('user switches workflows before the agent edits arrive', async () => { - await expect(tabs).toHaveCount(1) - await expect( - topbar.getTab(0).and(topbar.getActiveTab()) - ).toBeVisible() - const inputs = await ksamplerInputs() - expect(inputs.steps).toBe(20) - expect(inputs.cfg).toBe(7) - await topbar.newWorkflowButton.click() - await expect(tabs).toHaveCount(2) - await expect( - topbar.getTab(1).and(topbar.getActiveTab()) - ).toBeVisible() - await expect(agentConversation.vueNodes.nodes).toHaveCount(0) - }) + await agentConversation.replayResponse(0, { + beforeFirstGraphOps: async () => { + await test.step('user switches workflows before the agent edits arrive', async () => { + await expect(tabs).toHaveCount(1) + await expect( + topbar.getTab(0).and(topbar.getActiveTab()) + ).toBeVisible() + const inputs = await ksamplerInputs() + expect(inputs.steps).toBe(20) + expect(inputs.cfg).toBe(7) + await topbar.newWorkflowButton.click() + await expect(tabs).toHaveCount(2) + await expect( + topbar.getTab(1).and(topbar.getActiveTab()) + ).toBeVisible() + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + }) + } }) await test.step('agent finishes editing the background workflow', async () => { diff --git a/browser_tests/types/globals.d.ts b/browser_tests/types/globals.d.ts index a0a4d37cb8b..54f8c7f4ea9 100644 --- a/browser_tests/types/globals.d.ts +++ b/browser_tests/types/globals.d.ts @@ -4,6 +4,7 @@ import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode' import type { LiteGraphGlobal } from '@/lib/litegraph/src/LiteGraphGlobal' import type { ComfyApp } from '@/scripts/app' import type { useWorkspaceStore } from '@/stores/workspaceStore' +import type { App } from 'vue' /** * Helper type for accessing nodes by ID in browser tests. @@ -30,6 +31,10 @@ interface CapturedMessages { } declare global { + interface HTMLElement { + __vue_app__?: App + } + interface Window { app?: ComfyApp graph?: LGraph @@ -60,6 +65,7 @@ declare global { * @see browser_tests/tests/agent/agentHumanAddTabSwitch.spec.ts */ __tabSwitchLens?: TabSwitchLens + __agentRecoveryGraph?: LGraph } const app: ComfyApp | undefined diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index 1fa048b0204..80a176caea9 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -27,7 +27,6 @@ export class AgentCrdtProjection { private readonly getFollowerDoc: () => Y.Doc, intent?: LocalIntent ) { - // Wires the adapter's store-only retry callback to the live-graph sweep. this.adapter = new EcsFollowerAdapter(mutations, intent, (workflowId) => this.reconcileLiveGraph(workflowId) ) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts index 3069eb0387c..7fda437ea6b 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.integration.test.ts @@ -608,9 +608,9 @@ describe('EcsFollowerAdapter integration', () => { const batchCalls: unknown[] = [] const trackedMutations: GraphMutations = { ...mutations, - batch: (batchContext, define) => { + batchResult: (batchContext, define) => { batchCalls.push(batchContext) - return mutations.batch(batchContext, define) + return mutations.batchResult(batchContext, define) } } @@ -699,10 +699,10 @@ describe('EcsFollowerAdapter integration', () => { // still fire on the existing budget afterward. const throwingMutations: GraphMutations = { ...realMutations, - batch: (batchContext, define) => { + batchResult: (batchContext, define) => { batchCallCount += 1 if (batchCallCount === 2) throw failure - return realMutations.batch(batchContext, define) + return realMutations.batchResult(batchContext, define) } } @@ -770,9 +770,9 @@ describe('EcsFollowerAdapter integration', () => { const batchCalls: unknown[] = [] const trackedMutations: GraphMutations = { ...mutations, - batch: (batchContext, define) => { + batchResult: (batchContext, define) => { batchCalls.push(batchContext) - return mutations.batch(batchContext, define) + return mutations.batchResult(batchContext, define) } } const sweepFailure = new Error('live sweep failed') @@ -828,7 +828,7 @@ describe('EcsFollowerAdapter integration', () => { } }) - it('keeps retry provenance from the frame that started the rejection episode', () => { + it('attributes recovery to every frame folded into its authoritative snapshot', () => { vi.useFakeTimers() try { let scopeAvailable = true @@ -876,16 +876,11 @@ describe('EcsFollowerAdapter integration', () => { scope, [toNodeId(99)], expect.objectContaining({ - actor: 'agent:first-frame', - opId: 'first-op', - opIds: ['first-op'] + actor: 'agent-reconcile', + opId: 'later-op', + opIds: ['first-op', 'later-op'] }) ) - expect(deleteLayouts).not.toHaveBeenCalledWith( - scope, - [toNodeId(99)], - expect.objectContaining({ actor: 'agent:later-frame' }) - ) adapter.destroy() follower.destroy() @@ -909,9 +904,9 @@ describe('EcsFollowerAdapter integration', () => { const batchCalls: unknown[] = [] const trackedMutations: GraphMutations = { ...mutations, - batch: (batchContext, define) => { + batchResult: (batchContext, define) => { batchCalls.push(batchContext) - return mutations.batch(batchContext, define) + return mutations.batchResult(batchContext, define) } } const onReconcileRetryCommitted = vi.fn() @@ -1026,6 +1021,57 @@ describe('EcsFollowerAdapter integration', () => { } ) + it.for(['clearForReset', 'discardPending'] as const)( + '%s called by a throwing live sweep cannot be followed by another sweep', + (lifecycleAction) => { + vi.useFakeTimers() + try { + let scopeAvailable = false + const mutations = createGraphMutations({ + placement: inertPlacementPort, + getScope: () => (scopeAvailable ? scope : null), + layout: { createNode: vi.fn(), deleteNodes: vi.fn() } + }) + const host = mint({ nodes: [], links: [] }, catalog) + const follower = new FollowerDoc() + const onReconcileRetryCommitted = vi.fn(() => { + if (lifecycleAction === 'clearForReset') { + adapter.clearForReset('wf', { + source: 'agent-remote', + actor: 'reset', + opId: 'reset' + }) + } else { + adapter.discardPending('wf') + } + throw new Error('live sweep failed after cancellation') + }) + const adapter = new EcsFollowerAdapter( + mutations, + undefined, + onReconcileRetryCommitted + ) + adapter.bind('wf', follower) + const update = Y.encodeStateAsUpdate(host) + follower.applyRemoteUpdate(update) + + expect(adapter.applyFrame({ workflowId: 'wf', seq: 1, update })).toBe( + false + ) + scopeAvailable = true + vi.advanceTimersByTime(5_000) + + expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) + + adapter.unbind('wf') + follower.destroy() + host.destroy() + } finally { + vi.useRealTimers() + } + } + ) + it.for(['clearForReset', 'discardPending', 'unbind'] as const)( 'a second, later live-sweep failure cannot orphan an earlier sweep-retry timer past %s', (lifecycleAction) => { @@ -1060,11 +1106,6 @@ describe('EcsFollowerAdapter integration', () => { vi.advanceTimersByTime(200) expect(onReconcileRetryCommitted).toHaveBeenCalledTimes(1) - // A second, independent rejection episode starts and its own retry - // commits well before the first sweep-retry timer would fire; its - // sweep also throws. Without idempotent scheduling this overwrites - // `liveSweepRetryTimer`, orphaning the first timer so nothing can - // ever `clearTimeout` it again. scopeAvailable = false expect(adapter.applyFrame({ workflowId: 'wf', seq: 2, update })).toBe( false @@ -1181,9 +1222,9 @@ describe('EcsFollowerAdapter integration', () => { const batchCalls: unknown[] = [] const mutations: GraphMutations = { ...realMutations, - batch: (context, define) => { + batchResult: (context, define) => { batchCalls.push(context) - return realMutations.batch(context, define) + return realMutations.batchResult(context, define) } } const adapter = new EcsFollowerAdapter(mutations) @@ -1198,8 +1239,7 @@ describe('EcsFollowerAdapter integration', () => { // Retype node 1 to a type with no output, while the doc still carries // link 9 originating from its old output slot 0: `prepare()`'s stale // link revalidation rejects this deterministically (see the "FEC-4" - // test below). Scope stays available throughout, so this is exactly - // the case `hasScope()` must distinguish from a transient scope race. + // test below). Scope stays available throughout. const before = Y.encodeStateVector(host) const retypeOp = { op_id: 'retype', @@ -1252,12 +1292,12 @@ describe('EcsFollowerAdapter integration', () => { getScope: () => scope, layout: { createNode: vi.fn(), deleteNodes: vi.fn() } }) - const realBatch = mutations.batch.bind(mutations) + const realBatch = mutations.batchResult.bind(mutations) const failure = new Error('projection failed') let throwNextBatch = true const throwingMutations: GraphMutations = { ...mutations, - batch: (context, define) => { + batchResult: (context, define) => { if (throwNextBatch) { throwNextBatch = false throw failure @@ -2179,12 +2219,13 @@ describe('EcsFollowerAdapter integration', () => { clearSemanticGraph: () => undefined } return { - batch: (_context, define) => { + batch: () => true, + batchResult: (_context, define) => { events.push(`${workflowId}:start`) define(noopBatch) if (workflowId === 'wf-a') adapter.applyFrame(frameB) events.push(`${workflowId}:end`) - return true + return { kind: 'committed' } }, addNode: () => true, setWidget: () => true, diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 2ee938e8d09..088eb3927c5 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -6,6 +6,7 @@ import { import * as Y from 'yjs' import type { + GraphMutationBatchResult, GraphMutations, SemanticLinkPayload, SemanticNodePayload @@ -362,6 +363,20 @@ function frameContext(update: DocUpdate): RemoteMutationContext { } } +function mergeRecoveryFrame(first: DocUpdate, latest: DocUpdate): DocUpdate { + const opIds = [...new Set([...(first.opIds ?? []), ...(latest.opIds ?? [])])] + return { + ...latest, + actor: + first.actor === latest.actor + ? first.actor + : first.actor || latest.actor + ? 'agent-reconcile' + : undefined, + ...(opIds.length > 0 ? { opIds } : { opIds: undefined }) + } +} + interface TargetSession { readonly workflowId: string readonly follower: FollowerDoc @@ -381,31 +396,25 @@ interface TargetSession { onLinksChanged: (event: Y.YMapEvent) => void reconcileNextFrame: boolean applying: boolean - /** Pending retry of a batch rejected while workflow scope is unavailable. */ - reconcileRetryTimer: ReturnType | null - reconcileRetryAttempt: number - /** - * The frame whose batch was most recently rejected, resubmitted verbatim - * (same actor/opIds/seq) by the retry timer instead of a synthetic frame: - * the reconcile branch never reads `update.update`, only `frameContext`, - * so replaying the original frame is both simpler and correctly attributed. - */ - lastRejectedFrame: DocUpdate | null - /** - * True while a retry timer's own resubmission is being applied, so the - * post-batch bookkeeping can tell "a fresh frame was rejected" (which may - * start a new retry episode) apart from "the retry's own attempt was - * rejected again" (which must not reset the retry budget). - */ - retryInFlight: boolean - /** - * Pending retry of the live-graph sweep after a committed retry batch's - * `onReconcileRetryCommitted` callback threw. The store mutation already - * committed by the time this fires, so only the sweep itself is retried. - */ - liveSweepRetryTimer: ReturnType | null + batchRecovery: BatchRecoveryState + liveSweepRecovery: LiveSweepRecoveryState } +type BatchRecoveryState = + | { kind: 'idle' } + | { + kind: 'scheduled' + timer: ReturnType + attempt: number + frame: DocUpdate + } + | { kind: 'applying'; attempt: number; frame: DocUpdate } + +type LiveSweepRecoveryState = + | { kind: 'idle' } + | { kind: 'running' } + | { kind: 'scheduled'; timer: ReturnType } + /** * Projects each subscribed semantic document into its own ECS mutation stream. * Target sessions own their Yjs observers, pending effects, and apply queue; @@ -418,14 +427,6 @@ export class EcsFollowerAdapter { constructor( private readonly mutations: MutationsForTarget, private readonly intent: LocalIntent = NO_LOCAL_INTENT, - /** - * Called after a self-driven retry's batch commits. The adapter is - * store-only and has no notion of the live graph; this is how its owner - * (`AgentCrdtProjection`) plugs in the same `reconcileLiveGraph` sweep a - * normally-arriving frame gets via `applyAndReconcile`. Without it, a - * retry that commits still leaves stale live nodes on screen (and - * savable) since nothing ever re-materializes the canvas for it. - */ private readonly onReconcileRetryCommitted: ( workflowId: string ) => void = () => undefined @@ -476,9 +477,6 @@ export class EcsFollowerAdapter { const session = this.targets.get(workflowId) if (!session) return false this.discardSessionPending(session) - // A pending retry (or an armed `reconcileNextFrame`) would otherwise - // fire up to `RECONCILE_RETRY_INTERVAL_MS` later and re-project the - // pre-reset doc, resurrecting exactly the state this reset clears. this.clearReconcileRetry(session) this.clearLiveSweepRetry(session) session.reconcileNextFrame = false @@ -525,11 +523,8 @@ export class EcsFollowerAdapter { frameQueue: [], reconcileNextFrame: true, applying: false, - reconcileRetryTimer: null, - reconcileRetryAttempt: 0, - lastRejectedFrame: null, - retryInFlight: false, - liveSweepRetryTimer: null, + batchRecovery: { kind: 'idle' }, + liveSweepRecovery: { kind: 'idle' }, onNodesChanged: (_events): void => undefined, onLinksChanged: (_event): void => undefined } @@ -596,129 +591,120 @@ export class EcsFollowerAdapter { const linkId = resolveLinkMapKey(id) return linkId === null ? [] : [linkId] }) - const committed = session.mutations.batch(frameContext(update), (batch) => { - // A SubgraphNode host that is already live must never be rebuilt from - // its doc entry: `reconcileNode` (and delete + `addNode`) replaces the - // host's input list in place, which drops the `widgetId` / - // `_subgraphSlot` bindings its promoted widgets hang off, leaving the - // host with no widgets at all. Resync the host's scalar fields (title, - // mode, flags, properties, colors) and promoted values only; - // `readSemanticNode` has already keyed the values from the definition. - const isHost = (payload: SemanticNodePayload) => - definitions().has(payload.type) - const upsertNode = ( - payload: SemanticNodePayload, - mode: 'add' | 'reconcile' - ) => { - if (mode === 'add') batch.addNode(payload) - else if (isHost(payload)) batch.reconcileNodeFields(payload) - else batch.reconcileNode(payload) - } - - if (reconcile) { - const pendingDeletes = this.intent.pendingDeletes(session.workflowId) - const nodes = [...session.nodes.keys()] - .filter((id) => !pendingDeletes.has(id)) - .flatMap((id) => { - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - return payload ? [payload] : [] - }) - const links = excludeIncompatibleLinks( - [...session.links.keys()].flatMap((id) => { - const link = readSemanticLink( - doc, - id, - definitions(), - session.reportedErrors - ) - return link && - !pendingDeletes.has(String(link.originNodeId)) && - !pendingDeletes.has(String(link.targetNodeId)) - ? [link] - : [] - }), - session.reportedErrors - ) - batch.removeMissing( - nodes.map(({ id }) => toNodeId(id)), - links.map(({ id }) => id) - ) - for (const payload of nodes) upsertNode(payload, 'reconcile') - for (const link of links) batch.connect(link) - return - } + const result = session.mutations.batchResult( + frameContext(update), + (batch) => { + // A SubgraphNode host that is already live must never be rebuilt from + // its doc entry: `reconcileNode` (and delete + `addNode`) replaces the + // host's input list in place, which drops the `widgetId` / + // `_subgraphSlot` bindings its promoted widgets hang off, leaving the + // host with no widgets at all. Resync the host's scalar fields (title, + // mode, flags, properties, colors) and promoted values only; + // `readSemanticNode` has already keyed the values from the definition. + const isHost = (payload: SemanticNodePayload) => + definitions().has(payload.type) + const upsertNode = ( + payload: SemanticNodePayload, + mode: 'add' | 'reconcile' + ) => { + if (mode === 'add') batch.addNode(payload) + else if (isHost(payload)) batch.reconcileNodeFields(payload) + else batch.reconcileNode(payload) + } - batch.removeLinks(removedLinkIds) - const payloads = new Map( - [...nodeActions] - .filter(([, action]) => action !== 'delete') - .map( - ([id]) => - [ + if (reconcile) { + const pendingDeletes = this.intent.pendingDeletes(session.workflowId) + const nodes = [...session.nodes.keys()] + .filter((id) => !pendingDeletes.has(id)) + .flatMap((id) => { + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + return payload ? [payload] : [] + }) + const links = excludeIncompatibleLinks( + [...session.links.keys()].flatMap((id) => { + const link = readSemanticLink( + doc, id, - readSemanticNode(doc, id, definitions, session.reportedErrors) - ] as const + definitions(), + session.reportedErrors + ) + return link && + !pendingDeletes.has(String(link.originNodeId)) && + !pendingDeletes.has(String(link.targetNodeId)) + ? [link] + : [] + }), + session.reportedErrors ) - ) - for (const [id, action] of nodeActions) { - if (action === 'delete') { - batch.deleteNode(toNodeId(id)) - continue + batch.removeMissing( + nodes.map(({ id }) => toNodeId(id)), + links.map(({ id }) => id) + ) + for (const payload of nodes) upsertNode(payload, 'reconcile') + for (const link of links) batch.connect(link) + return } - const payload = payloads.get(id) - if (action === 'update' && !(payload && isHost(payload))) - batch.deleteNode(toNodeId(id)) - } - for (const [id, payload] of payloads) { - if (!payload) continue - upsertNode(payload, nodeActions.get(id) === 'add' ? 'add' : 'reconcile') - } - // A node whose widget storage was replaced wholesale, either the named - // `widgets` map or the positional `__widgets_opaque` array (cmp writes - // both in one transaction when a host's storage flips to opaque, and - // deletes the opaque array when it flips back), is re-read in full. - for (const id of new Set([ - ...replacedWidgetMaps, - ...replacedOpaqueWidgets - ])) { - if (nodeActions.has(id)) continue - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - if (payload) upsertNode(payload, 'reconcile') - } - // A node whose scalar fields were edited by key (title, mode, flags, - // properties, colors) is re-read so a live host resyncs those fields - // without rebuilding its promoted widgets or slots. - for (const id of changedNodeFields) { - if ( - nodeActions.has(id) || - replacedWidgetMaps.has(id) || - replacedOpaqueWidgets.has(id) - ) - continue - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors + + batch.removeLinks(removedLinkIds) + const payloads = new Map( + [...nodeActions] + .filter(([, action]) => action !== 'delete') + .map( + ([id]) => + [ + id, + readSemanticNode(doc, id, definitions, session.reportedErrors) + ] as const + ) ) - if (payload) upsertNode(payload, 'reconcile') - } - for (const [id, names] of changedWidgets) { - if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue - const node = session.nodes.get(id) - const widgets = node?.get('widgets') - if (!(widgets instanceof Y.Map)) continue - if ([...names].some((name) => !widgets.has(name))) { + for (const [id, action] of nodeActions) { + if (action === 'delete') { + batch.deleteNode(toNodeId(id)) + continue + } + const payload = payloads.get(id) + if (action === 'update' && !(payload && isHost(payload))) + batch.deleteNode(toNodeId(id)) + } + for (const [id, payload] of payloads) { + if (!payload) continue + upsertNode( + payload, + nodeActions.get(id) === 'add' ? 'add' : 'reconcile' + ) + } + // A node whose widget storage was replaced wholesale, either the named + // `widgets` map or the positional `__widgets_opaque` array (cmp writes + // both in one transaction when a host's storage flips to opaque, and + // deletes the opaque array when it flips back), is re-read in full. + for (const id of new Set([ + ...replacedWidgetMaps, + ...replacedOpaqueWidgets + ])) { + if (nodeActions.has(id)) continue + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) upsertNode(payload, 'reconcile') + } + // A node whose scalar fields were edited by key (title, mode, flags, + // properties, colors) is re-read so a live host resyncs those fields + // without rebuilding its promoted widgets or slots. + for (const id of changedNodeFields) { + if ( + nodeActions.has(id) || + replacedWidgetMaps.has(id) || + replacedOpaqueWidgets.has(id) + ) + continue const payload = readSemanticNode( doc, id, @@ -726,165 +712,167 @@ export class EcsFollowerAdapter { session.reportedErrors ) if (payload) upsertNode(payload, 'reconcile') - continue } - for (const name of names) { - batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) + for (const [id, names] of changedWidgets) { + if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue + const node = session.nodes.get(id) + const widgets = node?.get('widgets') + if (!(widgets instanceof Y.Map)) continue + if ([...names].some((name) => !widgets.has(name))) { + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) upsertNode(payload, 'reconcile') + continue + } + for (const name of names) { + batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) + } } + const incomingLinks = excludeIncompatibleLinks( + [...changedLinks.values()].filter( + (link): link is SemanticLinkPayload => link !== null + ), + session.reportedErrors + ) + for (const link of incomingLinks) batch.connect(link) } - const incomingLinks = excludeIncompatibleLinks( - [...changedLinks.values()].filter( - (link): link is SemanticLinkPayload => link !== null - ), - session.reportedErrors - ) - for (const link of incomingLinks) batch.connect(link) - }) + ) // The pending sets were snapshotted and cleared before `batch` ran, so a // rejected batch (no scope, or validation failure) has already lost the // incremental record of this frame. Arm a full reconcile for the next // frame so the dropped edits are re-read from the doc instead of falling // through to incremental handling that never revisits them. + const committed = result.kind === 'committed' session.reconcileNextFrame = !committed - this.handleReconcileOutcome(session, update, committed) + this.handleReconcileOutcome(session, update, result) return committed } private handleReconcileOutcome( session: TargetSession, update: DocUpdate, - committed: boolean + result: GraphMutationBatchResult ): void { - if (committed) { + if (result.kind === 'committed') { this.clearReconcileRetry(session) return } - // A rejection that arrives while no retry is already in flight starts a - // new rejection episode: reset the budget so an unrelated later - // rejection (or scope staying down longer than the previous episode's - // remaining budget) still gets the full retry window, instead of - // inheriting whatever was left over from an earlier, unrelated episode. - // It also captures this episode's frame once: a later rejection that - // arrives while a retry is already armed must not overwrite the frame - // the pending timer will resubmit, or the replayed commit gets - // attributed to whichever frame happened to arrive last instead of the - // one that actually started the episode. `retryInFlight` excludes the - // retry's own resubmission (already counted against the budget, and not - // a new episode) from either of these. - if (!session.retryInFlight && !session.reconcileRetryTimer) { - session.reconcileRetryAttempt = 0 - session.lastRejectedFrame = update + if (result.reason === 'validation') { + if (session.batchRecovery.kind === 'applying') + session.batchRecovery = { kind: 'idle' } + return } - // Read the rejection captured by the same `batch()` call so scope - // becoming available immediately afterward cannot hide the race. A - // `GraphMutations` that implements neither optional classifier must - // never be treated as a scope race: `hasScope` has to explicitly say - // `false` before a deterministic-looking rejection is retried. - const rejection = session.mutations.lastBatchRejection?.() - if ( - rejection === 'no-scope' || - (rejection === undefined && session.mutations.hasScope?.() === false) - ) { - this.scheduleReconcileRetry(session) + + if (session.batchRecovery.kind === 'scheduled') { + session.batchRecovery.frame = mergeRecoveryFrame( + session.batchRecovery.frame, + update + ) + return } + if (session.batchRecovery.kind === 'applying') { + this.scheduleReconcileRetry( + session, + session.batchRecovery.frame, + session.batchRecovery.attempt + ) + return + } + this.scheduleReconcileRetry(session, update, 0) } - /** - * Retry a scope-rejected batch quickly at first, then at a low steady rate - * until scope returns or the target is reset/unbound. This keeps recovery - * armed even when no later frame arrives. - */ - private scheduleReconcileRetry(session: TargetSession): void { - if (session.reconcileRetryTimer) return - session.reconcileRetryAttempt += 1 + private scheduleReconcileRetry( + session: TargetSession, + frame: DocUpdate, + previousAttempts: number + ): void { + const attempt = previousAttempts + 1 const delay = - session.reconcileRetryAttempt <= RECONCILE_FAST_RETRY_LIMIT + attempt <= RECONCILE_FAST_RETRY_LIMIT ? RECONCILE_RETRY_INTERVAL_MS : RECONCILE_SLOW_RETRY_INTERVAL_MS - session.reconcileRetryTimer = setTimeout(() => { - session.reconcileRetryTimer = null - if (this.targets.get(session.workflowId) !== session) return - if (!session.reconcileNextFrame) return - // Resubmit the actual rejected frame (same actor/opIds/seq) rather - // than a synthetic one: the reconcile branch below never reads - // `update.update`, only `frameContext(update)`, so this both keeps - // `seq` in its normal domain and attributes the replayed commit to - // whichever op actually got rejected instead of a generic 'replay'. - const retryFrame = session.lastRejectedFrame ?? { - workflowId: session.workflowId, - seq: 0, - update: new Uint8Array() - } - session.retryInFlight = true - let committed: boolean - try { - committed = this.applyFrame(retryFrame) - } catch (error) { - // `applyFrame` deliberately propagates a throw from `batch()` (see - // "replays authoritative state through real mutations after a batch - // throws"), but this call has no caller to propagate to: it runs - // off a timer. Left uncaught it would become an unhandled global - // error, `reconcileRetryTimer` is already null, and the throw skips - // the `scheduleReconcileRetry` call below, silently killing the - // retry chain. Report it and keep retrying on the existing budget - // instead. - session.retryInFlight = false - reportError(error instanceof Error ? error : new Error(String(error)), { - errorType: 'error_agent_reconcile_retry_threw', - context: { workflowId: session.workflowId } - }) - this.scheduleReconcileRetry(session) - return - } - session.retryInFlight = false - if (committed) this.runLiveGraphSweep(session) - }, delay) + const scheduled: Extract = { + kind: 'scheduled', + timer: setTimeout(() => { + if (this.targets.get(session.workflowId) !== session) return + if (session.batchRecovery !== scheduled) return + if (!session.reconcileNextFrame) return + const applying = { + kind: 'applying', + attempt: scheduled.attempt, + frame: scheduled.frame + } satisfies BatchRecoveryState + session.batchRecovery = applying + let committed: boolean + try { + committed = this.applyFrame(applying.frame) + } catch (error) { + reportError( + error instanceof Error ? error : new Error(String(error)), + { + errorType: 'error_agent_reconcile_retry_threw', + context: { workflowId: session.workflowId } + } + ) + if (session.batchRecovery === applying) + this.scheduleReconcileRetry( + session, + applying.frame, + applying.attempt + ) + return + } + if (committed) this.runLiveGraphSweep(session) + }, delay), + attempt, + frame + } + session.batchRecovery = scheduled } - /** - * Sweeps the live graph for a retry batch that has already committed. - * `onReconcileRetryCommitted` can throw (its contract reaches extension - * `onRemoved` hooks), and by this point the store mutation is done and the - * retry state already cleared — replaying `applyFrame` would resubmit a - * batch that already succeeded. So a throw here only reschedules the sweep - * itself, keeping recovery armed until the live graph actually converges. - */ private runLiveGraphSweep(session: TargetSession): void { + if (session.liveSweepRecovery.kind === 'scheduled') + clearTimeout(session.liveSweepRecovery.timer) + const running = { kind: 'running' } satisfies LiveSweepRecoveryState + session.liveSweepRecovery = running try { this.onReconcileRetryCommitted(session.workflowId) + if (session.liveSweepRecovery === running) + session.liveSweepRecovery = { kind: 'idle' } } catch (error) { reportError(error instanceof Error ? error : new Error(String(error)), { errorType: 'error_agent_reconcile_live_sweep_threw', context: { workflowId: session.workflowId } }) - // A timer from an earlier failed sweep may already be armed for this - // session (e.g. a second, independent rejection episode committed and - // failed its own sweep before the first sweep's retry fired). - // Overwriting `liveSweepRetryTimer` here would orphan that earlier - // timer: it stays scheduled but nothing can ever `clearTimeout` it - // again, so it fires later even after `unbind`/`clearForReset`/ - // `discardPending` believed they had cancelled recovery. Scheduling is - // therefore idempotent: only arm a new timer when none is pending. - if (session.liveSweepRetryTimer) return - session.liveSweepRetryTimer = setTimeout(() => { - session.liveSweepRetryTimer = null - if (this.targets.get(session.workflowId) !== session) return - this.runLiveGraphSweep(session) - }, RECONCILE_SLOW_RETRY_INTERVAL_MS) + if (session.liveSweepRecovery !== running) return + const scheduled: Extract = + { + kind: 'scheduled', + timer: setTimeout(() => { + if (this.targets.get(session.workflowId) !== session) return + if (session.liveSweepRecovery !== scheduled) return + this.runLiveGraphSweep(session) + }, RECONCILE_SLOW_RETRY_INTERVAL_MS) + } + session.liveSweepRecovery = scheduled } } private clearReconcileRetry(session: TargetSession): void { - if (session.reconcileRetryTimer) clearTimeout(session.reconcileRetryTimer) - session.reconcileRetryTimer = null - session.reconcileRetryAttempt = 0 - session.lastRejectedFrame = null + if (session.batchRecovery.kind === 'scheduled') + clearTimeout(session.batchRecovery.timer) + session.batchRecovery = { kind: 'idle' } } private clearLiveSweepRetry(session: TargetSession): void { - if (session.liveSweepRetryTimer) clearTimeout(session.liveSweepRetryTimer) - session.liveSweepRetryTimer = null + if (session.liveSweepRecovery.kind === 'scheduled') + clearTimeout(session.liveSweepRecovery.timer) + session.liveSweepRecovery = { kind: 'idle' } } private discardSessionPending(session: TargetSession): void { diff --git a/src/workbench/extensions/agent/crdt/followerSubscription.test.ts b/src/workbench/extensions/agent/crdt/followerSubscription.test.ts index 4150b1c4428..7b01aa9aa5e 100644 --- a/src/workbench/extensions/agent/crdt/followerSubscription.test.ts +++ b/src/workbench/extensions/agent/crdt/followerSubscription.test.ts @@ -29,7 +29,7 @@ import type { DocFrameTransport, DocOp, DocUpdate } from './docFrameClient' import { DocFrameClient, encodeBase64 } from './docFrameClient' import { EcsFollowerAdapter } from './ecsFollowerAdapter' import { FollowerDoc } from './followerDoc' -import type { GraphMutations } from './graphMutations' +import type { GraphMutationBatchResult, GraphMutations } from './graphMutations' import { LayoutFollowerBridge } from './layoutFollowerBridge' import { FollowerSchemaError, assertReadableSchema } from './schemaGuard' @@ -127,6 +127,12 @@ describe('follower commit boundary', () => { }) const mutations = { batch: vi.fn(() => false), + batchResult: vi.fn( + (): GraphMutationBatchResult => ({ + kind: 'rejected', + reason: 'validation' + }) + ), addNode: vi.fn(() => false), setWidget: vi.fn(() => false), connect: vi.fn(() => false), diff --git a/src/workbench/extensions/agent/crdt/graphMutations.ts b/src/workbench/extensions/agent/crdt/graphMutations.ts index 8ad911107a1..fc2c06df58e 100644 --- a/src/workbench/extensions/agent/crdt/graphMutations.ts +++ b/src/workbench/extensions/agent/crdt/graphMutations.ts @@ -176,23 +176,19 @@ export interface GraphMutationBatch { clearSemanticGraph(): void } +export type GraphMutationBatchResult = + | { readonly kind: 'committed' } + | { readonly kind: 'rejected'; readonly reason: 'no-scope' | 'validation' } + export interface GraphMutations { batch( context: RemoteMutationContext, define: (batch: GraphMutationBatch) => void ): boolean - /** Reason the most recent synchronous `batch` call returned `false`. */ - lastBatchRejection?(): 'no-scope' | 'validation' | null - /** - * Cheap, side-effect-free check for whether `batch` would currently be - * rejected for lack of scope. Callers that retry a rejected `batch` (e.g. - * `EcsFollowerAdapter`'s self-driven reconcile retry) use this to tell a - * transient scope race, worth retrying, apart from a deterministic - * `prepare()` validation rejection that retrying can never fix. Optional - * so a test double that never rejects for scope reasons can omit it; a - * caller that only cares about `batch`'s boolean result is unaffected. - */ - hasScope?(): boolean + batchResult( + context: RemoteMutationContext, + define: (batch: GraphMutationBatch) => void + ): GraphMutationBatchResult addNode(payload: SemanticNodePayload, context: RemoteMutationContext): boolean setWidget( nodeId: NodeId, @@ -1277,7 +1273,6 @@ function createAutogrowMemory() { * provenance on every participating store action. */ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { - let lastBatchRejection: 'no-scope' | 'validation' | null = null const nodeStore = useNodeDataStore() const linkStore = useLinkStore() const linkPresentationStore = useLinkPresentationStore() @@ -2266,67 +2261,65 @@ export function createGraphMutations(deps: GraphMutationsDeps): GraphMutations { } } + const runBatch = ( + context: RemoteMutationContext, + define: (batch: GraphMutationBatch) => void + ): GraphMutationBatchResult => { + const scope = deps.getScope() + if (!scope) return { kind: 'rejected', reason: 'no-scope' } + const queued: QueuedMutation[] = [] + define({ + addNode(payload) { + queued.push({ kind: 'addNode', payload }) + }, + reconcileNode(payload) { + queued.push({ kind: 'reconcileNode', payload }) + }, + reconcileNodeFields(payload) { + queued.push({ kind: 'reconcileNodeFields', payload }) + }, + setWidget(nodeId, name, value) { + queued.push({ kind: 'setWidget', nodeId, name, value }) + }, + connect(link) { + queued.push({ kind: 'connect', link }) + }, + removeMissing(retainedNodeIds, retainedLinkIds) { + queued.push({ + kind: 'removeMissing', + retainedNodeIds, + retainedLinkIds + }) + }, + removeLinks(linkIds) { + queued.push({ kind: 'removeLinks', linkIds }) + }, + deleteNode(nodeId, removedLinkIds = []) { + queued.push({ kind: 'deleteNode', nodeId, removedLinkIds }) + }, + clearSemanticGraph() { + queued.push({ kind: 'clearSemanticGraph' }) + } + }) + const existingIds = nodeStore + .getGraphNodesFor(scope.rootGraphId, scope.owningGraphId) + .map((node) => node.id) + const memory = autogrowMemory.draft() + const prepared = prepare(memory, scope, queued) + if (typeof prepared === 'string') { + fail(prepared) + return { kind: 'rejected', reason: 'validation' } + } + offsetInsertedBatch(scope, existingIds, prepared) + commit(scope, prepared, context, memory) + return { kind: 'committed' } + } + const graphMutations: GraphMutations = { - lastBatchRejection() { - return lastBatchRejection - }, - hasScope() { - return deps.getScope() !== null - }, batch(context, define) { - lastBatchRejection = null - const scope = deps.getScope() - if (!scope) { - lastBatchRejection = 'no-scope' - return false - } - const queued: QueuedMutation[] = [] - define({ - addNode(payload) { - queued.push({ kind: 'addNode', payload }) - }, - reconcileNode(payload) { - queued.push({ kind: 'reconcileNode', payload }) - }, - reconcileNodeFields(payload) { - queued.push({ kind: 'reconcileNodeFields', payload }) - }, - setWidget(nodeId, name, value) { - queued.push({ kind: 'setWidget', nodeId, name, value }) - }, - connect(link) { - queued.push({ kind: 'connect', link }) - }, - removeMissing(retainedNodeIds, retainedLinkIds) { - queued.push({ - kind: 'removeMissing', - retainedNodeIds, - retainedLinkIds - }) - }, - removeLinks(linkIds) { - queued.push({ kind: 'removeLinks', linkIds }) - }, - deleteNode(nodeId, removedLinkIds = []) { - queued.push({ kind: 'deleteNode', nodeId, removedLinkIds }) - }, - clearSemanticGraph() { - queued.push({ kind: 'clearSemanticGraph' }) - } - }) - const existingIds = nodeStore - .getGraphNodesFor(scope.rootGraphId, scope.owningGraphId) - .map((node) => node.id) - const memory = autogrowMemory.draft() - const prepared = prepare(memory, scope, queued) - if (typeof prepared === 'string') { - lastBatchRejection = 'validation' - return fail(prepared) - } - offsetInsertedBatch(scope, existingIds, prepared) - commit(scope, prepared, context, memory) - return true + return runBatch(context, define).kind === 'committed' }, + batchResult: runBatch, addNode(payload, context) { return graphMutations.batch(context, (batch) => batch.addNode(payload)) }, From ad6f36ba037f4b47bffaa50d59ffb161c0b67f2f Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 25 Sep 2026 22:12:19 -0700 Subject: [PATCH 17/18] refactor(agent): split follower and harness branches for fallow Addresses https://github.com/Comfy-Org/ComfyUI_frontend/actions/runs/36111034979/job/107994301063 --- .../fixtures/agentConversationFixture.ts | 48 ++- .../agent/crdt/ecsFollowerAdapter.ts | 366 +++++++++++------- 2 files changed, 254 insertions(+), 160 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index b27f1cf2760..7b68e234daf 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -1008,7 +1008,16 @@ export class AgentConversationHarness { private async runWorkflowScopeAction( action: WorkflowScopeAction ): Promise { - return await this.page.evaluate((action) => { + if (action.kind === 'drop') + return await this.dropWorkflowScopeInPage(action) + await this.restoreWorkflowScopeInPage(action) + return undefined + } + + private async dropWorkflowScopeInPage( + action: Extract + ): Promise { + return await this.page.evaluate((workflowId) => { const isBinding = (value: unknown): value is WorkflowBindingActions => typeof value === 'object' && value !== null && @@ -1030,24 +1039,37 @@ export class AgentConversationHarness { throw new Error('agentWorkflowTabBinding store is not mounted') if (!workflows) throw new Error('workflow store is not mounted') - if (action.kind === 'drop') { - const tabPath = binding.tabPathFor(action.workflowId) - if (!tabPath) - throw new Error(`no bound tab path for workflow ${action.workflowId}`) - const tab = workflows.getWorkflowByPath(tabPath) - if (!tab) throw new Error(`no open tab at ${tabPath}`) - const graphId = tab.activeState?.id - if (!graphId) throw new Error(`workflow at ${tabPath} has no graph id`) - delete tab.activeState?.id - return { tabPath, graphId } - } + const tabPath = binding.tabPathFor(workflowId) + if (!tabPath) + throw new Error(`no bound tab path for workflow ${workflowId}`) + const tab = workflows.getWorkflowByPath(tabPath) + if (!tab) throw new Error(`no open tab at ${tabPath}`) + const graphId = tab.activeState?.id + if (!graphId) throw new Error(`workflow at ${tabPath} has no graph id`) + delete tab.activeState?.id + return { tabPath, graphId } + }, action.workflowId) + } + private async restoreWorkflowScopeInPage( + action: Extract + ): Promise { + await this.page.evaluate((action) => { + const isWorkflowLookup = (value: unknown): value is WorkflowLookup => + typeof value === 'object' && + value !== null && + 'getWorkflowByPath' in value && + typeof value.getWorkflowByPath === 'function' + const mount = document.getElementById('vue-app') + const pinia = mount?.__vue_app__?.config.globalProperties.$pinia + if (!pinia) throw new Error('Pinia is not mounted') + const workflows = [...pinia._s.values()].find(isWorkflowLookup) + if (!workflows) throw new Error('workflow store is not mounted') const tab = workflows.getWorkflowByPath(action.tabPath) if (!tab) throw new Error(`no open tab at ${action.tabPath}`) if (!tab.activeState) throw new Error(`workflow at ${action.tabPath} has no active state`) tab.activeState.id = action.graphId - return undefined }, action) } diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 088eb3927c5..218d904c471 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -6,6 +6,7 @@ import { import * as Y from 'yjs' import type { + GraphMutationBatch, GraphMutationBatchResult, GraphMutations, SemanticLinkPayload, @@ -415,6 +416,21 @@ type LiveSweepRecoveryState = | { kind: 'running' } | { kind: 'scheduled'; timer: ReturnType } +type BatchChanges = { + batch: GraphMutationBatch + session: TargetSession + doc: Y.Doc + definitions: () => SubgraphDefinitionIndex + reconcile: boolean + removedLinkIds: number[] + nodeActions: Map + replacedWidgetMaps: Set + replacedOpaqueWidgets: Set + changedNodeFields: Set + changedWidgets: Map> + changedLinks: Map +} + /** * Projects each subscribed semantic document into its own ECS mutation stream. * Target sessions own their Yjs observers, pending effects, and apply queue; @@ -593,153 +609,21 @@ export class EcsFollowerAdapter { }) const result = session.mutations.batchResult( frameContext(update), - (batch) => { - // A SubgraphNode host that is already live must never be rebuilt from - // its doc entry: `reconcileNode` (and delete + `addNode`) replaces the - // host's input list in place, which drops the `widgetId` / - // `_subgraphSlot` bindings its promoted widgets hang off, leaving the - // host with no widgets at all. Resync the host's scalar fields (title, - // mode, flags, properties, colors) and promoted values only; - // `readSemanticNode` has already keyed the values from the definition. - const isHost = (payload: SemanticNodePayload) => - definitions().has(payload.type) - const upsertNode = ( - payload: SemanticNodePayload, - mode: 'add' | 'reconcile' - ) => { - if (mode === 'add') batch.addNode(payload) - else if (isHost(payload)) batch.reconcileNodeFields(payload) - else batch.reconcileNode(payload) - } - - if (reconcile) { - const pendingDeletes = this.intent.pendingDeletes(session.workflowId) - const nodes = [...session.nodes.keys()] - .filter((id) => !pendingDeletes.has(id)) - .flatMap((id) => { - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - return payload ? [payload] : [] - }) - const links = excludeIncompatibleLinks( - [...session.links.keys()].flatMap((id) => { - const link = readSemanticLink( - doc, - id, - definitions(), - session.reportedErrors - ) - return link && - !pendingDeletes.has(String(link.originNodeId)) && - !pendingDeletes.has(String(link.targetNodeId)) - ? [link] - : [] - }), - session.reportedErrors - ) - batch.removeMissing( - nodes.map(({ id }) => toNodeId(id)), - links.map(({ id }) => id) - ) - for (const payload of nodes) upsertNode(payload, 'reconcile') - for (const link of links) batch.connect(link) - return - } - - batch.removeLinks(removedLinkIds) - const payloads = new Map( - [...nodeActions] - .filter(([, action]) => action !== 'delete') - .map( - ([id]) => - [ - id, - readSemanticNode(doc, id, definitions, session.reportedErrors) - ] as const - ) - ) - for (const [id, action] of nodeActions) { - if (action === 'delete') { - batch.deleteNode(toNodeId(id)) - continue - } - const payload = payloads.get(id) - if (action === 'update' && !(payload && isHost(payload))) - batch.deleteNode(toNodeId(id)) - } - for (const [id, payload] of payloads) { - if (!payload) continue - upsertNode( - payload, - nodeActions.get(id) === 'add' ? 'add' : 'reconcile' - ) - } - // A node whose widget storage was replaced wholesale, either the named - // `widgets` map or the positional `__widgets_opaque` array (cmp writes - // both in one transaction when a host's storage flips to opaque, and - // deletes the opaque array when it flips back), is re-read in full. - for (const id of new Set([ - ...replacedWidgetMaps, - ...replacedOpaqueWidgets - ])) { - if (nodeActions.has(id)) continue - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - if (payload) upsertNode(payload, 'reconcile') - } - // A node whose scalar fields were edited by key (title, mode, flags, - // properties, colors) is re-read so a live host resyncs those fields - // without rebuilding its promoted widgets or slots. - for (const id of changedNodeFields) { - if ( - nodeActions.has(id) || - replacedWidgetMaps.has(id) || - replacedOpaqueWidgets.has(id) - ) - continue - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - if (payload) upsertNode(payload, 'reconcile') - } - for (const [id, names] of changedWidgets) { - if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue - const node = session.nodes.get(id) - const widgets = node?.get('widgets') - if (!(widgets instanceof Y.Map)) continue - if ([...names].some((name) => !widgets.has(name))) { - const payload = readSemanticNode( - doc, - id, - definitions, - session.reportedErrors - ) - if (payload) upsertNode(payload, 'reconcile') - continue - } - for (const name of names) { - batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) - } - } - const incomingLinks = excludeIncompatibleLinks( - [...changedLinks.values()].filter( - (link): link is SemanticLinkPayload => link !== null - ), - session.reportedErrors - ) - for (const link of incomingLinks) batch.connect(link) - } + (batch) => + this.applyBatchChanges({ + batch, + session, + doc, + definitions, + reconcile, + removedLinkIds, + nodeActions, + replacedWidgetMaps, + replacedOpaqueWidgets, + changedNodeFields, + changedWidgets, + changedLinks + }) ) // The pending sets were snapshotted and cleared before `batch` ran, so a @@ -753,6 +637,194 @@ export class EcsFollowerAdapter { return committed } + private applyBatchChanges(args: BatchChanges): void { + if (args.reconcile) this.applyFullReconcile(args) + else this.applyIncrementalChanges(args) + } + + private applyFullReconcile({ + batch, + session, + doc, + definitions + }: Pick): void { + const pendingDeletes = this.intent.pendingDeletes(session.workflowId) + const nodes = [...session.nodes.keys()] + .filter((id) => !pendingDeletes.has(id)) + .flatMap((id) => { + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + return payload ? [payload] : [] + }) + const links = excludeIncompatibleLinks( + [...session.links.keys()].flatMap((id) => { + const link = readSemanticLink( + doc, + id, + definitions(), + session.reportedErrors + ) + return link && + !pendingDeletes.has(String(link.originNodeId)) && + !pendingDeletes.has(String(link.targetNodeId)) + ? [link] + : [] + }), + session.reportedErrors + ) + batch.removeMissing( + nodes.map(({ id }) => toNodeId(id)), + links.map(({ id }) => id) + ) + for (const payload of nodes) + this.upsertNode(batch, payload, definitions, 'reconcile') + for (const link of links) batch.connect(link) + } + + private applyIncrementalChanges(args: BatchChanges): void { + const { batch, session, doc, definitions, nodeActions } = args + batch.removeLinks(args.removedLinkIds) + const payloads = new Map( + [...nodeActions] + .filter(([, action]) => action !== 'delete') + .map( + ([id]) => + [ + id, + readSemanticNode(doc, id, definitions, session.reportedErrors) + ] as const + ) + ) + this.applyNodeActions(batch, definitions, nodeActions, payloads) + this.upsertChangedNodes(batch, definitions, nodeActions, payloads) + this.applyChangedNodeValues(args) + const incomingLinks = excludeIncompatibleLinks( + [...args.changedLinks.values()].filter( + (link): link is SemanticLinkPayload => link !== null + ), + session.reportedErrors + ) + for (const link of incomingLinks) batch.connect(link) + } + + private applyNodeActions( + batch: GraphMutationBatch, + definitions: () => SubgraphDefinitionIndex, + nodeActions: Map, + payloads: Map + ): void { + for (const [id, action] of nodeActions) { + if (action === 'delete') { + batch.deleteNode(toNodeId(id)) + continue + } + const payload = payloads.get(id) + if (action === 'update' && !(payload && definitions().has(payload.type))) + batch.deleteNode(toNodeId(id)) + } + } + + private upsertChangedNodes( + batch: GraphMutationBatch, + definitions: () => SubgraphDefinitionIndex, + nodeActions: Map, + payloads: Map + ): void { + for (const [id, payload] of payloads) { + if (payload) + this.upsertNode( + batch, + payload, + definitions, + nodeActions.get(id) === 'add' ? 'add' : 'reconcile' + ) + } + } + + private applyChangedNodeValues(args: BatchChanges): void { + const { + batch, + session, + doc, + definitions, + nodeActions, + replacedWidgetMaps, + replacedOpaqueWidgets + } = args + for (const id of new Set([ + ...replacedWidgetMaps, + ...replacedOpaqueWidgets + ])) { + if (nodeActions.has(id)) continue + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + } + for (const id of args.changedNodeFields) { + if ( + nodeActions.has(id) || + replacedWidgetMaps.has(id) || + replacedOpaqueWidgets.has(id) + ) + continue + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + } + this.applyChangedWidgets(args) + } + + private applyChangedWidgets(args: BatchChanges): void { + const { + batch, + session, + doc, + definitions, + nodeActions, + replacedWidgetMaps + } = args + for (const [id, names] of args.changedWidgets) { + if (nodeActions.has(id) || replacedWidgetMaps.has(id)) continue + const widgets = session.nodes.get(id)?.get('widgets') + if (!(widgets instanceof Y.Map)) continue + if ([...names].some((name) => !widgets.has(name))) { + const payload = readSemanticNode( + doc, + id, + definitions, + session.reportedErrors + ) + if (payload) this.upsertNode(batch, payload, definitions, 'reconcile') + continue + } + for (const name of names) + batch.setWidget(toNodeId(id), name, plain(widgets.get(name))) + } + } + + private upsertNode( + batch: GraphMutationBatch, + payload: SemanticNodePayload, + definitions: () => SubgraphDefinitionIndex, + mode: 'add' | 'reconcile' + ): void { + if (mode === 'add') batch.addNode(payload) + else if (definitions().has(payload.type)) batch.reconcileNodeFields(payload) + else batch.reconcileNode(payload) + } + private handleReconcileOutcome( session: TargetSession, update: DocUpdate, From b6a464cc537a97db3effdcf9b84e8fa33ef4ae7f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 26 Sep 2026 22:15:19 -0700 Subject: [PATCH 18/18] test(agent): adapt reconcile hook to replay options --- .../agent/agentCrdtReconcileOverwrite.spec.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts b/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts index 1d5006eac04..251be326f47 100644 --- a/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts +++ b/browser_tests/tests/agent/agentCrdtReconcileOverwrite.spec.ts @@ -388,13 +388,15 @@ test.describe( await agentConversation.sendPrompt(0) - await agentConversation.replayResponse(0, async () => { - await test.step('the user completes and leaves a widget edit', async () => { - await textField.fill(localValue) - await textField.press('Tab') - await expect(textField).not.toBeFocused() - await expect(textField).toHaveValue(localValue) - }) + await agentConversation.replayResponse(0, { + beforeFirstGraphOps: async () => { + await test.step('the user completes and leaves a widget edit', async () => { + await textField.fill(localValue) + await textField.press('Tab') + await expect(textField).not.toBeFocused() + await expect(textField).toHaveValue(localValue) + }) + } }) await agentConversation.waitForTurnComplete()