From 92df04a761eef9284d5f2d48be8712e5df4e5af0 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Tue, 3 Nov 2026 18:19:54 +0000 Subject: [PATCH] fix(agent): observe meta.groups in DocChangeCollector and project groups (#21749) ## Summary Part of FE-3387, implements FE-3583; supersedes #21572. `DocChangeCollector` observed `nodes`, `links` and `definitions` but never `meta`, so a remote `clear` left groups on the canvas (BE-22005) and groups carried by `insert_workflow` never appeared. The collector now observes all four roots and emits one change set per frame. - `DocChangeCollector` observes `meta` and adds a `groups` entry to the change set: each root group id whose presence in `meta.groups` changed. Other `meta` keys (including `extra`) are ignored; the package never rewrites `extra` after mint, and replaying it would move the local viewport. - `LiveGraphApplier` reads `meta.groups` at apply time and, per changed id, adds or removes the canvas group with `graph.add`/`graph.remove`, which emit the layout store's `createGroup`/`deleteGroup` operations (ADR CRDT-LAYOUT-0003). No reconcile pass. Groups the document never named, such as one drawn after mint, are left alone. Inserted groups are offset together with the nodes of the same batch. - Because the applier reads the document rather than the frame's delta, a failed frame retries to the document's current groups without the `changesAgainstDoc` patch #21572 needed. - `definitions` was already observed; registration through one `createSubgraphs` batch stays with FE-3401. Gap, needs a package/backend decision: the op vocabulary has no op for groups or for colour/shape/properties. Human group and appearance edits cannot reach the document, so the follower only mirrors host-authored group changes (`clear`, `insert_workflow`). Group geometry/appearance changes inside an existing `meta.groups` entry are not re-applied. No frontend-side ops are added here. ## Verification - `vitest run src/workbench/extensions/agent/crdt/`: 72 files, 1204 passed, 1 expected fail. - New tests fail on main and pass here: collector-level `meta.groups` add/remove diff; real host `clear` op removes both doc groups and keeps a post-mint group (#21572's scenario); #21572's retry case; real `insert_workflow` creates a group placed with its node. - oxfmt, oxlint, eslint on changed files and `pnpm typecheck:app` clean. --------- Co-authored-by: Christian Byrne --- packages/graph-model/src/workflowSchema.ts | 2 +- .../agentCrdtProjection.integration.test.ts | 160 +++++++++++++++++- .../agent/crdt/agentCrdtProjection.ts | 8 +- .../agent/crdt/docChangeCollector.test.ts | 21 +++ .../agent/crdt/docChangeCollector.ts | 31 +--- .../agent/crdt/liveGraphApplier.test.ts | 29 ++++ .../extensions/agent/crdt/liveGraphApplier.ts | 95 ++++++++--- 7 files changed, 290 insertions(+), 56 deletions(-) diff --git a/packages/graph-model/src/workflowSchema.ts b/packages/graph-model/src/workflowSchema.ts index 019c2e7a982..8e99197d5d5 100644 --- a/packages/graph-model/src/workflowSchema.ts +++ b/packages/graph-model/src/workflowSchema.ts @@ -342,7 +342,7 @@ const zSubgraphInstance = z }) .passthrough() -const zGroup = z +export const zGroup = z .object({ id: z.number().optional(), title: z.string(), diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index be18ce58874..2699d154338 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -1,5 +1,9 @@ -import { mint, nodesMap } from '@comfyorg/comfy-multi-player' -import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import { applyOps, mint, nodesMap } from '@comfyorg/comfy-multi-player' +import type { + Op, + WidgetCatalog, + WorkflowJSON +} from '@comfyorg/comfy-multi-player' import { beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' import * as Y from 'yjs' @@ -46,6 +50,18 @@ const CATALOG: WidgetCatalog = { types: { TestSource: { widget_order: ['steps', 'seed'] } } } +const HOST_OP: Pick = { + actor: 'agent:test:turn-1', + base_version: 1, + stamp: [1, 'agent:test:turn-1'] +} + +function applyHostOp(host: Y.Doc, op: Op) { + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) +} + type FrameClassification = Pick & Partial> @@ -368,6 +384,33 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { expect(graph.links.size).toBe(0) }) + it('removes the groups a host clear drops and keeps a group the document never held', () => { + const { graph } = buildLiveGraph() + const prompt = new LiteGraph.LGraphGroup('Prompt group') + const output = new LiteGraph.LGraphGroup('Output group') + graph.add(prompt) + graph.add(output) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + const drawnAfterMint = new LiteGraph.LGraphGroup('Drawn after mint') + graph.add(drawnAfterMint) + + hostEdit(() => + applyHostOp(host, { + ...HOST_OP, + op_id: 'c1ea'.padEnd(32, '0'), + op: 'clear', + removed_nodes: [1, 2] + }) + ) + + expect(graph._nodes).toEqual([]) + expect(graph.groups).toEqual([drawnAfterMint]) + }) + it('keeps only the groups the document holds again when retrying', () => { const { graph } = buildLiveGraph() const restored = new LiteGraph.LGraphGroup('Prompt group') @@ -379,7 +422,9 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { structuredClone(graph.serialize()) ) onTestFinished(destroy) - const [restoredGroup] = host.getMap('meta').get('groups') as unknown[] + const mintedGroups = host.getMap('meta').get('groups') + assert(Array.isArray(mintedGroups), 'mint wrote meta.groups') + const [restoredGroup] = mintedGroups function fail(): never { throw new Error('remove failed') @@ -398,4 +443,113 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { expect(graph.groups).toEqual([restored]) expect(projection.hasPendingFailure(WORKFLOW_ID)).toBe(false) }) + + it('creates the groups an inserted workflow carries, placed with its nodes', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a5e'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: 1, + type: 'TestSource', + pos: [20, 30], + size: [200, 100], + widgets_values: [20, 7] + } + ], + groups: [ + { id: 1, title: 'Inserted group', bounding: [0, 10, 300, 200] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const inserted = graph._nodes.find( + (node) => node !== source && node !== note + ) + const group = graph.groups.find(({ title }) => title === 'Inserted group') + assert(inserted && group, 'the insertion reached the canvas') + expect([...inserted.pos]).not.toEqual([20, 30]) + expect([ + group.pos[0] - inserted.pos[0], + group.pos[1] - inserted.pos[1] + ]).toEqual([-20, -20]) + expect([...group.size]).toEqual([300, 200]) + }) + it('places an inserted workflow that carries only groups', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a5f'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { id: 1, title: 'Inserted group', bounding: [0, 10, 300, 200] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const group = graph.groups.find(({ title }) => title === 'Inserted group') + assert(group, 'the inserted group reached the canvas') + expect([...group.pos]).toEqual([680, 20]) + }) + it('places a group-only insertion clear of the groups already on the canvas', () => { + const graph = new LGraph() + const existing = new LiteGraph.LGraphGroup('Existing group') + existing.pos = [0, 10] + existing.size = [300, 200] + graph.add(existing) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a61'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { id: 1, title: 'Inserted group', bounding: [0, 10, 300, 200] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const group = graph.groups.find(({ title }) => title === 'Inserted group') + assert(group, 'the inserted group reached the canvas') + expect([...group.pos]).toEqual([380, 10]) + }) }) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index a50910e0a44..4e85ff458d7 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -6,7 +6,7 @@ import type { LGraph } from '@/lib/litegraph/src/litegraph' import type { NodeId } from '@comfyorg/graph-model/nodeId' import { recordDevEvent } from './devPanelLog' -import { DocChangeCollector, groupIds } from './docChangeCollector' +import { DocChangeCollector } from './docChangeCollector' import type { DocUpdate } from './docFrameClient' import type { FollowerDoc } from './followerDoc' import type { GraphOperation } from './graphOperations' @@ -75,11 +75,7 @@ function changesAgainstDoc(doc: Y.Doc, changes: FrameChanges): FrameChanges { if (!docNodes.has(id)) nodes.set(id, 'delete') else nodes.set(id, change === 'delete' ? 'update' : change) } - const docGroups = groupIds(doc.getMap('meta').get('groups')) - const removedGroups = new Set( - [...(changes.removedGroups ?? [])].filter((id) => !docGroups.has(id)) - ) - return { ...changes, nodes, removedGroups } + return { ...changes, nodes } } function pendingNodeDelta(target: BoundTarget, changes: FrameChanges) { diff --git a/src/workbench/extensions/agent/crdt/docChangeCollector.test.ts b/src/workbench/extensions/agent/crdt/docChangeCollector.test.ts index 612e6c79b79..b37e6663141 100644 --- a/src/workbench/extensions/agent/crdt/docChangeCollector.test.ts +++ b/src/workbench/extensions/agent/crdt/docChangeCollector.test.ts @@ -164,6 +164,27 @@ describe('DocChangeCollector', () => { expect(changes.nodes.size).toBe(0) }) + it('records the group ids meta.groups gains or loses, ignoring other meta keys', () => { + const { doc, collector } = setup() + const group = (id: number) => ({ + id, + title: `Group ${id}`, + bounding: [0, 0, 200, 100] + }) + const meta = doc.getMap('meta') + doc.transact(() => meta.set('groups', [group(7), group(9)])) + expect(collector.take().groups).toEqual(new Set(['7', '9'])) + + doc.transact(() => meta.set('last_node_id', 12)) + expect(collector.take().groups).toEqual(new Set()) + + doc.transact(() => meta.set('groups', [group(9), group(11)])) + expect(collector.take().groups).toEqual(new Set(['7', '11'])) + + doc.transact(() => meta.set('groups', [])) + expect(collector.take().groups).toEqual(new Set(['9', '11'])) + }) + it('records link keys, clears on take, and stops observing after destroy', () => { const { doc, collector } = setup() diff --git a/src/workbench/extensions/agent/crdt/docChangeCollector.ts b/src/workbench/extensions/agent/crdt/docChangeCollector.ts index f6c8ccde6a3..0ccd57cb6fc 100644 --- a/src/workbench/extensions/agent/crdt/docChangeCollector.ts +++ b/src/workbench/extensions/agent/crdt/docChangeCollector.ts @@ -7,7 +7,7 @@ import * as Y from 'yjs' import { definitionsObserverRoot } from './agentSubgraphDefinitions' import type { FrameChanges, NodeChange } from './liveGraphApplier' -import { SYNCED_NODE_FIELDS } from './liveGraphApplier' +import { readDocGroups, SYNCED_NODE_FIELDS } from './liveGraphApplier' /** * Collects what the Y observers on one follower document report between two @@ -27,7 +27,7 @@ export class DocChangeCollector { Map | 'all'> >() private readonly interiorDefinitions = new Set() - private readonly removedGroups = new Set() + private readonly groups = new Set() private readonly nodesMap: Y.Map> private readonly linksMap: Y.Map private readonly definitionsMap: Y.Map @@ -37,7 +37,7 @@ export class DocChangeCollector { this.nodesMap = nodesMap(doc) this.linksMap = linksMap(doc) this.definitionsMap = definitionsObserverRoot(doc) - this.metaMap = doc.getMap('meta') + this.metaMap = doc.getMap('meta') this.nodesMap.observeDeep(this.onNodesChanged) this.linksMap.observe(this.onLinksChanged) this.definitionsMap.observeDeep(this.onDefinitionsChanged) @@ -60,7 +60,7 @@ export class DocChangeCollector { ]) ), interiorDefinitions: new Set(this.interiorDefinitions), - removedGroups: new Set(this.removedGroups) + groups: new Set(this.groups) } } @@ -78,7 +78,7 @@ export class DocChangeCollector { this.links.clear() this.interiors.clear() this.interiorDefinitions.clear() - this.removedGroups.clear() + this.groups.clear() } destroy(): void { @@ -223,22 +223,9 @@ export class DocChangeCollector { private readonly onMetaChanged = (event: Y.YMapEvent): void => { const change = event.changes.keys.get('groups') if (!change) return - const current = groupIds(this.metaMap.get('groups')) - for (const id of groupIds(change.oldValue)) - if (!current.has(id)) this.removedGroups.add(id) - for (const id of current) this.removedGroups.delete(id) + const before = readDocGroups(change.oldValue) + const after = readDocGroups(this.metaMap.get('groups')) + for (const id of before.keys()) if (!after.has(id)) this.groups.add(id) + for (const id of after.keys()) if (!before.has(id)) this.groups.add(id) } } - -export function groupIds(groups: unknown): Set { - const ids = new Set() - if (!Array.isArray(groups)) return ids - for (const group of groups) { - const id: unknown = - typeof group === 'object' && group !== null - ? (group as { id?: unknown }).id - : undefined - if (typeof id === 'number' || typeof id === 'string') ids.add(String(id)) - } - return ids -} diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts index 679be333122..d3afdf52e0b 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts @@ -2796,6 +2796,35 @@ describe('LiveGraphApplier change bracket', () => { } ) + it('drops the groups a replacement document no longer holds', () => { + const { graph, applier, applyCollected } = setup({ + nodes: [], + links: [], + groups: [ + { id: 5, title: 'Dropped', bounding: [0, 0, 100, 100] }, + { id: 6, title: 'Kept', bounding: [200, 0, 100, 100] } + ] + }) + applyCollected() + const replacement = followedDoc( + { + nodes: [], + links: [], + groups: [{ id: 6, title: 'Kept', bounding: [200, 0, 100, 100] }] + }, + CATALOG + ) + + applier.applyChanges( + replacement.doc, + replacement.collector.take(), + CONTEXT, + 'replace' + ) + + expect(graph.groups.map((group) => group.title)).toEqual(['Kept']) + }) + it('brackets a catch-up frame and a lineage replacement once each, and never without a graph', () => { const { canvas, events } = recordingCanvas() const { graph, doc, collector, applier } = setup(seed, { diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index 97bfbcafe8e..f254ce451de 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -22,19 +22,24 @@ import type { INodeInputSlot } from '@comfyorg/graph-model/slots' import { LGraphCanvas } from '@/lib/litegraph/src/LGraphCanvas' import { withGraphIntentSource } from '@/lib/litegraph/src/graphIntents' import { detachSerialisedLinks } from '@/lib/litegraph/src/linkDeduplication' -import { LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { + LGraphGroup, + LGraphNode, + LiteGraph +} from '@/lib/litegraph/src/litegraph' import type { LGraph } from '@/lib/litegraph/src/litegraph' import { topologicalSortSubgraphs } from '@/lib/litegraph/src/subgraph/subgraphDeduplication' import type { ExportedSubgraph, ISerialisableNodeInput, ISerialisableNodeOutput, + ISerialisedGroup, ISerialisedNode } from '@comfyorg/graph-model/serialisation' import type { IBaseWidget } from '@comfyorg/graph-model/widgets' import { isWidgetValue } from '@comfyorg/graph-model/widgets' import { reportError } from '@comfyorg/telemetry/reportError' -import { zComfyNode } from '@comfyorg/graph-model/workflowSchema' +import { zComfyNode, zGroup } from '@comfyorg/graph-model/workflowSchema' import { BaseWidget } from '@/lib/litegraph/src/widgets/BaseWidget' import { inputSpecTree } from '@/core/graph/nodeDef/inputSpecInterpreter' import { isUuidShapedSubgraphId } from '@/schemas/subgraphIdSchema' @@ -78,8 +83,8 @@ export interface FrameChanges { interiorDefinitions?: ReadonlySet - /** Root group ids the document's `meta.groups` held and no longer does. */ - removedGroups?: ReadonlySet + /** Root group ids that `meta.groups` gained or lost. */ + groups?: ReadonlySet } export interface RemoteApplyContext { @@ -220,6 +225,19 @@ const zDocNodeFields = zComfyNode [OPAQUE_WIDGETS_KEY]: zWidgetRecord.optional() }) +const zDocGroup = zGroup.extend({ id: z.number() }) + +/** The `meta.groups` records that configure a canvas group, keyed by id. */ +export function readDocGroups(groups: unknown): Map { + const parsed = new Map() + if (!Array.isArray(groups)) return parsed + for (const record of groups) { + const group = zDocGroup.safeParse(structuredClone(record)) + if (group.success) parsed.set(String(group.data.id), group.data) + } + return parsed +} + /** * Only the actor's kind is reported: the segments after it in * `agent::` and `human::` identify a user. @@ -764,6 +782,13 @@ function positionalWidgetValues( }) } +function addGroup(graph: LGraph, serialised: ISerialisedGroup): LGraphGroup { + const group = new LGraphGroup() + group.configure(serialised) + graph.add(group) + return group +} + function missingNode(docNode: DocNode): LGraphNode { const node = new LGraphNode( docNode.serialised.title || docNode.type || 'Missing Node', @@ -846,8 +871,6 @@ export class LiveGraphApplier { }) } - this.deleteGroups(graph, changes.removedGroups, context) - for (const [id, keys] of changes.resyncNodes) { if (touchedNodes.has(id)) continue this.try(context, () => this.syncFields(graph, doc, id, keys)) @@ -861,8 +884,14 @@ export class LiveGraphApplier { this.applyInteriors(graph, doc, changes, context, mode) this.applyLinks(graph, doc, [...changes.links], context) + const createdGroups = this.applyGroups( + graph, + doc, + changes.groups, + context + ) if (mode === 'merge') - this.placeBatch(graph, created, { + this.placeBatch(graph, created, createdGroups, { moveCollisions: context.liveAgentTurn === true }) return { @@ -892,6 +921,13 @@ export class LiveGraphApplier { if (docLinks.has(String(id))) continue this.try(context, () => graph.removeLink(id)) } + const docGroups = readDocGroups(doc.getMap('meta').get('groups')) + const absentGroups = graph.groups.filter( + (group) => !docGroups.has(String(group.id)) + ) + for (const group of absentGroups) { + this.try(context, () => graph.remove(group)) + } } private write(graph: LGraph, context: RemoteApplyContext, fn: () => T): T { @@ -1011,17 +1047,24 @@ export class LiveGraphApplier { if (node) graph.remove(node) } - private deleteGroups( + private applyGroups( graph: LGraph, + doc: Y.Doc, ids: ReadonlySet | undefined, context: RemoteApplyContext - ): void { - for (const id of ids ?? []) { + ): LGraphGroup[] { + if (!ids?.size) return [] + const docGroups = readDocGroups(doc.getMap('meta').get('groups')) + const created: LGraphGroup[] = [] + for (const id of ids) { this.try(context, () => { - const group = graph.groups.find((group) => String(group.id) === id) - if (group) graph.remove(group) + const live = graph.groups.find((group) => String(group.id) === id) + const docGroup = docGroups.get(id) + if (live && !docGroup) graph.remove(live) + else if (!live && docGroup) created.push(addGroup(graph, docGroup)) }) } + return created } private upsertNode( @@ -1769,28 +1812,32 @@ export class LiveGraphApplier { private placeBatch( graph: LGraph, created: readonly NodeId[], + createdGroups: readonly LGraphGroup[], { moveCollisions }: { moveCollisions: boolean } ): void { - if (created.length === 0) return const createdSet = new Set(created) - const rect = (node: LGraphNode): PlacementRect => ({ - x: node.pos[0], - y: node.pos[1], - width: node.size[0], - height: node.size[1] + const rect = (item: LGraphNode | LGraphGroup): PlacementRect => ({ + x: item.pos[0], + y: item.pos[1], + width: item.size[0], + height: item.size[1] }) - const incoming = graph._nodes.filter((node) => createdSet.has(node.id)) + const incoming = [ + ...graph._nodes.filter((node) => createdSet.has(node.id)), + ...createdGroups + ] const offset = placementOffset({ - existing: graph._nodes - .filter((node) => !createdSet.has(node.id)) - .map(rect), + existing: [ + ...graph._nodes.filter((node) => !createdSet.has(node.id)), + ...graph.groups.filter((group) => !createdGroups.includes(group)) + ].map(rect), viewport: this.deps.viewportBounds?.() ?? null, incoming: incoming.map(rect), moveCollisions }) if (!offset) return - for (const node of incoming) { - node.pos = [node.pos[0] + offset.dx, node.pos[1] + offset.dy] + for (const item of incoming) { + item.pos = [item.pos[0] + offset.dx, item.pos[1] + offset.dy] } } }