From f9cdcf3b32d6cc5eb365b0ba4a33bec622f8e212 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 24 Oct 2026 11:41:30 +0000 Subject: [PATCH] fix(agent): remove canvas groups a remote clear drops from the document (#21572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Asking the agent to clear the workflow removed every node but left the canvas groups drawn, while the agent replied that the canvas was empty (Windows QA report in #comfy-agent-user-feedback, tracked as BE-22005 and the older PM-831). The host's `clear` op already empties the document's `meta.groups` (multiplayer schema ยง6); the follower never observed `meta`, so the group removal never reached the canvas. - `DocChangeCollector` now observes `meta` and records each group id that the replaced `groups` value held and the current one does not. - `LiveGraphApplier` removes those groups with `graph.remove(group)` inside the existing remote write scope (provenance and before/after-change bracket unchanged). No reconcile pass and no store writes. - Groups the document never named (for example one drawn after the document was minted) are left alone, matching `clear_canvas`'s "nodes the user added that are not in it are kept". - The conversation harness gains a `seedPassthrough` option so a recorded case can start with top-level passthrough keys such as `groups`; the recorded responses are untouched. Supersedes the pre-refactor attempt in #17497, which targeted `ecsFollowerAdapter`/`graphMutations` (deleted by #18700). ## Verification - New black-box case in `browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts` (`with groups on the canvas`) replays the recorded `agent-rec-clear-workflow` turn over a seed with two groups. **Fails without the `src/` change** (`["Output group", "Prompt group"]` still on the canvas after the clear) and **passes with it**. The other three cases in the file stay green. - `vitest run src/workbench/extensions/agent/crdt/`: 72 files, 1169 passed, 1 expected fail. - `pnpm typecheck`, `pnpm typecheck:browser`, oxfmt, oxlint, eslint on changed files: clean. Not covered here: the follower still does not draw groups the document *adds* (`insert_workflow` with `groups`), and a group drawn after mint never reaches the document because no group op is minted. Both are recorded on BE-22005. (cherry picked from commit 48880f80a404e9b84a62a089a83e11885ad03fa1) --- .../fixtures/agentConversationFixture.ts | 12 ++++- .../agent/agentDeleteAndClearWorkflow.spec.ts | 52 +++++++++++++++++++ .../agentCrdtProjection.integration.test.ts | 33 +++++++++++- .../agent/crdt/agentCrdtProjection.ts | 8 ++- .../agent/crdt/docChangeCollector.ts | 31 ++++++++++- .../extensions/agent/crdt/liveGraphApplier.ts | 18 +++++++ 6 files changed, 148 insertions(+), 6 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 352780149fc..cc5caf2cd68 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -1419,6 +1419,7 @@ interface ConversationFixtures { bootAssets: ListAssetsResponse // Opt in only for cases that assert telemetry through hostTelemetryFixture. telemetryEnabled: boolean + seedPassthrough: Omit, 'nodes' | 'links'> comfyPage: ComfyPage agentConversation: AgentConversationHarness } @@ -1433,6 +1434,7 @@ export const agentConversationTest = agentTest.extend({ humanOpsHost: ['hold', { option: true }], bootAssets: [{ assets: [], total: 0, has_more: false }, { option: true }], telemetryEnabled: [false, { option: true }], + seedPassthrough: [{}, { option: true }], comfyPage: async ({ page, request }, use) => { await use(new ComfyPage(page, request)) }, @@ -1454,7 +1456,8 @@ export const agentConversationTest = agentTest.extend({ extraNodeDefs, humanOpsHost, bootAssets, - telemetryEnabled + telemetryEnabled, + seedPassthrough }, use, testInfo @@ -1466,10 +1469,15 @@ export const agentConversationTest = agentTest.extend({ throw new Error( `a conversation replay is judged on Vue nodes; tag the test ${VUE_NODES_TAG}` ) + const conversation = loadAgentConversation(conversationCase) + conversation.workflow.seed = { + ...conversation.workflow.seed, + ...seedPassthrough + } const harness = new AgentConversationHarness( page, comfyPage, - loadAgentConversation(conversationCase), + conversation, replayTiming, conversationCase, extraNodeDefs, diff --git a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts index ec58c6d13a1..60cf90f8c30 100644 --- a/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentDeleteAndClearWorkflow.spec.ts @@ -51,6 +51,58 @@ test.describe( expect(await agentConversation.isRecoveryGraphUnchanged()).toBe(true) }) + test.describe('with groups on the canvas', () => { + test.use({ + seedPassthrough: { + groups: [ + { + id: 7, + title: 'Prompt group', + bounding: [0, 0, 420, 260], + color: '#3f789e', + font_size: 24, + flags: {} + }, + { + id: 9, + title: 'Output group', + bounding: [520, 0, 320, 220], + color: '#b58b2a', + font_size: 24, + flags: {} + } + ] + } + }) + + test('removes the groups along with the nodes', async ({ + agentConversation, + page + }) => { + test.setTimeout(90_000) + const groupTitles = () => + page.evaluate(() => + window.app!.graph.groups.map((group) => group.title).sort() + ) + + await expect + .poll(groupTitles) + .toEqual(['Output group', 'Prompt group']) + await agentConversation.runTurns({ + beforeGraphOps: async (ops) => { + if (isClear(ops)) + expect(await groupTitles()).toEqual([ + 'Output group', + 'Prompt group' + ]) + } + }) + + await expect(agentConversation.vueNodes.nodes).toHaveCount(0) + await expect.poll(groupTitles).toEqual([]) + }) + }) + // PM-1500 / PM-1504: a clear that landed while the bound tab's graph id // was unresolvable used to be rejected and left stale nodes on screen. test('clears the canvas on the clear frame even when the workflow scope is dropped', async ({ diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index 339bd00e3bf..deea359bf4e 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -1,6 +1,6 @@ import { mint, nodesMap } from '@comfyorg/comfy-multi-player' import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' -import { beforeEach, describe, expect, it, onTestFinished } from 'vitest' +import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' import * as Y from 'yjs' import { assert } from '@/base/assert' @@ -367,4 +367,35 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { expect(source.title).toBe('Reminted') expect(graph.links.size).toBe(0) }) + + it('keeps only the groups the document holds again when retrying', () => { + const { graph } = buildLiveGraph() + const restored = new LiteGraph.LGraphGroup('Prompt group') + const dropped = new LiteGraph.LGraphGroup('Output group') + graph.add(restored) + graph.add(dropped) + const { host, hostEdit, projection, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + const [restoredGroup] = host.getMap('meta').get('groups') as unknown[] + + function fail(): never { + throw new Error('remove failed') + } + vi.spyOn(graph, 'remove') + .mockImplementationOnce(fail) + .mockImplementationOnce(fail) + hostEdit(() => host.getMap('meta').set('groups', [])) + expect(projection.hasPendingFailure(WORKFLOW_ID)).toBe(true) + expect(graph.groups).toEqual([restored, dropped]) + + hostEdit(() => host.getMap('meta').set('groups', [restoredGroup])) + expect(projection.retryFailed(WORKFLOW_ID)).toMatchObject({ + failureCount: 0 + }) + expect(graph.groups).toEqual([restored]) + expect(projection.hasPendingFailure(WORKFLOW_ID)).toBe(false) + }) }) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index cb2a2eaa716..d8c00eb21f2 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -6,7 +6,7 @@ import type { LGraph } from '@/lib/litegraph/src/litegraph' import type { NodeId } from '@/types/nodeId' import { recordDevEvent } from './devPanelLog' -import { DocChangeCollector } from './docChangeCollector' +import { DocChangeCollector, groupIds } from './docChangeCollector' import type { DocUpdate } from './docFrameClient' import type { FollowerDoc } from './followerDoc' import type { GraphOperation } from './graphOperations' @@ -75,7 +75,11 @@ function changesAgainstDoc(doc: Y.Doc, changes: FrameChanges): FrameChanges { if (!docNodes.has(id)) nodes.set(id, 'delete') else nodes.set(id, change === 'delete' ? 'update' : change) } - return { ...changes, nodes } + const docGroups = groupIds(doc.getMap('meta').get('groups')) + const removedGroups = new Set( + [...(changes.removedGroups ?? [])].filter((id) => !docGroups.has(id)) + ) + return { ...changes, nodes, removedGroups } } function pendingNodeDelta(target: BoundTarget, changes: FrameChanges) { diff --git a/src/workbench/extensions/agent/crdt/docChangeCollector.ts b/src/workbench/extensions/agent/crdt/docChangeCollector.ts index ecd8588a1aa..17d894036c0 100644 --- a/src/workbench/extensions/agent/crdt/docChangeCollector.ts +++ b/src/workbench/extensions/agent/crdt/docChangeCollector.ts @@ -27,17 +27,21 @@ export class DocChangeCollector { Map | 'all'> >() private readonly interiorDefinitions = new Set() + private readonly removedGroups = new Set() private readonly nodesMap: Y.Map> private readonly linksMap: Y.Map private readonly definitionsMap: Y.Map + private readonly metaMap: Y.Map constructor(doc: Y.Doc) { this.nodesMap = nodesMap(doc) this.linksMap = linksMap(doc) this.definitionsMap = definitionsObserverRoot(doc) + this.metaMap = doc.getMap('meta') this.nodesMap.observeDeep(this.onNodesChanged) this.linksMap.observe(this.onLinksChanged) this.definitionsMap.observeDeep(this.onDefinitionsChanged) + this.metaMap.observe(this.onMetaChanged) } /** Returns everything collected so far without clearing it. */ @@ -53,7 +57,8 @@ export class DocChangeCollector { new Map(nodes) ]) ), - interiorDefinitions: new Set(this.interiorDefinitions) + interiorDefinitions: new Set(this.interiorDefinitions), + removedGroups: new Set(this.removedGroups) } } @@ -71,12 +76,14 @@ export class DocChangeCollector { this.links.clear() this.interiors.clear() this.interiorDefinitions.clear() + this.removedGroups.clear() } destroy(): void { this.nodesMap.unobserveDeep(this.onNodesChanged) this.linksMap.unobserve(this.onLinksChanged) this.definitionsMap.unobserveDeep(this.onDefinitionsChanged) + this.metaMap.unobserve(this.onMetaChanged) this.discard() } @@ -198,4 +205,26 @@ export class DocChangeCollector { private readonly onLinksChanged = (event: Y.YMapEvent): void => { for (const id of event.keysChanged) this.links.add(id) } + + private readonly onMetaChanged = (event: Y.YMapEvent): void => { + const change = event.changes.keys.get('groups') + if (!change) return + const current = groupIds(this.metaMap.get('groups')) + for (const id of groupIds(change.oldValue)) + if (!current.has(id)) this.removedGroups.add(id) + for (const id of current) this.removedGroups.delete(id) + } +} + +export function groupIds(groups: unknown): Set { + const ids = new Set() + if (!Array.isArray(groups)) return ids + for (const group of groups) { + const id: unknown = + typeof group === 'object' && group !== null + ? (group as { id?: unknown }).id + : undefined + if (typeof id === 'number' || typeof id === 'string') ids.add(String(id)) + } + return ids } diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index 76b8f5d9952..1020f742826 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -73,6 +73,9 @@ export interface FrameChanges { interiors?: InteriorChanges interiorDefinitions?: ReadonlySet + + /** Root group ids the document's `meta.groups` held and no longer does. */ + removedGroups?: ReadonlySet } export interface RemoteApplyContext { @@ -640,6 +643,8 @@ export class LiveGraphApplier { }) } + this.deleteGroups(graph, changes.removedGroups, context) + for (const id of changes.resyncNodes) { if (touchedNodes.has(id)) continue this.try(context, () => this.syncFields(graph, doc, id)) @@ -805,6 +810,19 @@ export class LiveGraphApplier { if (node) graph.remove(node) } + private deleteGroups( + graph: LGraph, + ids: ReadonlySet | undefined, + context: RemoteApplyContext + ): void { + for (const id of ids ?? []) { + this.try(context, () => { + const group = graph.groups.find((group) => String(group.id) === id) + if (group) graph.remove(group) + }) + } + } + private upsertNode( graph: LGraph, doc: Y.Doc,