diff --git a/browser_tests/fixtures/data/agent/agentStageFrames.ts b/browser_tests/fixtures/data/agent/agentStageFrames.ts index 18e7485cdf9..9e74610803f 100644 --- a/browser_tests/fixtures/data/agent/agentStageFrames.ts +++ b/browser_tests/fixtures/data/agent/agentStageFrames.ts @@ -64,3 +64,26 @@ export const STAGE_FRAMES_CATALOG: WidgetCatalog = { export const STAGE_FRAMES_INSERT_OPS: RecordedGraphOperation[] = [ { op: 'insert_workflow', workflow: STAGE_FRAMES_TEMPLATE } ] + +export const EXISTING_FRAME_TITLE = 'Existing node frame' + +/** A workflow that carries only a group drawn around the existing node. */ +export const EXISTING_FRAME_INSERT_OPS: RecordedGraphOperation[] = [ + { + op: 'insert_workflow', + workflow: { + nodes: [], + links: [], + groups: [ + { + id: 1, + title: EXISTING_FRAME_TITLE, + bounding: [-20, -60, 360, 280], + color: '#3f789e', + font_size: 24, + flags: {} + } + ] + } + } +] diff --git a/browser_tests/tests/agent/agentStageFramesFromInsertedWorkflow.spec.ts b/browser_tests/tests/agent/agentStageFramesFromInsertedWorkflow.spec.ts index 769d2be0373..0c36b698503 100644 --- a/browser_tests/tests/agent/agentStageFramesFromInsertedWorkflow.spec.ts +++ b/browser_tests/tests/agent/agentStageFramesFromInsertedWorkflow.spec.ts @@ -15,6 +15,8 @@ import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' import { Topbar } from '@e2e/fixtures/components/Topbar' import { + EXISTING_FRAME_INSERT_OPS, + EXISTING_FRAME_TITLE, STAGE_FRAME_TITLES, STAGE_FRAMES_CATALOG, STAGE_FRAMES_EXISTING_NODES, @@ -195,5 +197,34 @@ test.describe( ) expect(savedGeometry).toEqual(live.geometry) }) + + test('keeps a group drawn around existing nodes on those nodes', async ({ + page + }) => { + socket.send(JSON.stringify(host.apply(EXISTING_FRAME_INSERT_OPS))) + + await expect + .poll(() => + page.evaluate(() => + window.app!.graph.groups.map((group) => { + group.recomputeInsideNodes() + return { + title: group.title, + bounding: [...group._bounding], + members: group.nodes.map((node) => + String(node.widgets?.[0]?.value ?? '') + ) + } + }) + ) + ) + .toEqual([ + { + title: EXISTING_FRAME_TITLE, + bounding: [-20, -60, 360, 280], + members: ['Existing node'] + } + ]) + }) } ) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index 2699d154338..57bae5bc35e 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -1,4 +1,5 @@ import { applyOps, mint, nodesMap } from '@comfyorg/comfy-multi-player' +import { fromPartial } from '@total-typescript/shoehorn' import type { Op, WidgetCatalog, @@ -9,6 +10,7 @@ import * as Y from 'yjs' import { assert } from '@/base/assert' import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import type { LGraphCanvas } from '@/lib/litegraph/src/litegraph' import type { ISerialisedGraph } from '@comfyorg/graph-model/serialisation' import { useNodeDataStore } from '@/stores/nodeDataStore' import { useWidgetValueStore } from '@/stores/widgetValueStore' @@ -68,7 +70,10 @@ type FrameClassification = Pick & function toWorkflowJson({ nodes, ...rest }: ISerialisedGraph): WorkflowJSON { return { ...rest, - nodes: nodes.map(({ flags, ...node }) => ({ ...node, flags: { ...flags } })) + nodes: nodes.map(({ flags, ...node }) => ({ + ...node, + flags: { ...flags } + })) } } @@ -97,6 +102,14 @@ function buildLiveGraph() { return { graph, source, note } } +/** A canvas context whose text measures {@link width} px wide. */ +function measuringContext(width: number) { + return fromPartial({ + font: '', + measureText: () => fromPartial({ width }) + }) +} + function snapshot(graph: LGraph) { const scope = graphScopeOf(graph) const widgetStore = useWidgetValueStore() @@ -132,10 +145,14 @@ function snapshot(graph: LGraph) { } } -function bindAndCatchUp(graph: LGraph, saved: ISerialisedGraph) { +function bindAndCatchUp( + graph: LGraph, + saved: ISerialisedGraph, + deps: ConstructorParameters[1] = {} +) { const host = mint(toWorkflowJson(saved), CATALOG) const follower = new FollowerDoc() - const projection = new AgentCrdtProjection(() => graph) + const projection = new AgentCrdtProjection(() => graph, deps) projection.bind(WORKFLOW_ID, follower) let seq = 0 const deliver = (update: Uint8Array, classification: FrameClassification) => { @@ -448,6 +465,8 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { const { graph, source, note } = buildLiveGraph() source.setSize([200, 100]) note.setSize([200, 100]) + source.updateArea() + note.updateArea() const { host, hostEdit, destroy } = bindAndCatchUp( graph, structuredClone(graph.serialize()) @@ -490,10 +509,12 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { ]).toEqual([-20, -20]) expect([...group.size]).toEqual([300, 200]) }) - it('places an inserted workflow that carries only groups', () => { + it('keeps a group-only insertion on the existing nodes it frames', () => { const { graph, source, note } = buildLiveGraph() source.setSize([200, 100]) note.setSize([200, 100]) + source.updateArea() + note.updateArea() const { host, hostEdit, destroy } = bindAndCatchUp( graph, structuredClone(graph.serialize()) @@ -516,6 +537,346 @@ describe('AgentCrdtProjection catch-up over a live graph', () => { { actor: HOST_OP.actor, catchUp: false } ) + const group = graph.groups.find(({ title }) => title === 'Inserted group') + assert(group, 'the inserted group reached the canvas') + expect([...group.pos]).toEqual([0, 10]) + expect([...group.size]).toEqual([300, 200]) + expect([...source.pos]).toEqual([10, 20]) + }) + it('keeps a group around existing nodes in place when the same insertion adds a node', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a62'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: 1, + type: 'TestSource', + pos: [20, 30], + size: [200, 100], + widgets_values: [20, 7] + } + ], + groups: [ + { id: 1, title: 'Around note', bounding: [380, -20, 240, 160] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const inserted = graph._nodes.find( + (node) => node !== source && node !== note + ) + const group = graph.groups.find(({ title }) => title === 'Around note') + assert(inserted && group, 'the insertion reached the canvas') + expect([...inserted.pos]).not.toEqual([20, 30]) + expect([...group.pos]).toEqual([380, -20]) + expect([...note.pos]).toEqual([400, 20]) + }) + it('keeps every group of a group-only insertion together when one frames existing nodes', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a63'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { id: 1, title: 'Around source', bounding: [0, -20, 240, 160] }, + { id: 2, title: 'Next stage', bounding: [250, 0, 100, 300] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const positions = Object.fromEntries( + graph.groups.map((group) => [group.title, [...group.pos]]) + ) + expect(positions).toEqual({ + 'Around source': [0, -20], + 'Next stage': [250, 0] + }) + }) + it('places a group with the new node it frames by its rendered title', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a65'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: 1, + type: 'TestSource', + pos: [20, 30], + size: [200, 100], + widgets_values: [20, 7] + } + ], + groups: [ + { id: 1, title: 'Around both', bounding: [0, -20, 300, 95] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const inserted = graph._nodes.find( + (node) => node !== source && node !== note + ) + const group = graph.groups.find(({ title }) => title === 'Around both') + assert(inserted && group, 'the insertion reached the canvas') + expect([...inserted.pos]).not.toEqual([20, 30]) + expect([ + group.pos[0] - inserted.pos[0], + group.pos[1] - inserted.pos[1] + ]).toEqual([-20, -50]) + }) + it('keeps a group clear of a new collapsed node it only overlaps unrendered', () => { + LiteGraph.vueNodesMode = true + onTestFinished(() => { + LiteGraph.vueNodesMode = false + }) + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.pos = [1000, 20] + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a66'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: 1, + type: 'TestSource', + pos: [20, 30], + size: [200, 100], + flags: { collapsed: true }, + widgets_values: [20, 7] + } + ], + groups: [ + { id: 1, title: 'Around source', bounding: [100, -20, 300, 200] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const inserted = graph._nodes.find( + (node) => node !== source && node !== note + ) + const group = graph.groups.find(({ title }) => title === 'Around source') + assert(inserted && group, 'the insertion reached the canvas') + expect([...inserted.pos]).not.toEqual([20, 30]) + expect([...group.pos]).toEqual([100, -20]) + }) + it('places a group with a new collapsed node it frames by its rendered title width', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.pos = [1000, 20] + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const ctx = measuringContext(400) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()), + { + getCanvas: () => + fromPartial({ + ctx, + emitBeforeChange: () => {}, + emitAfterChange: () => {} + }) + } + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a67'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: 1, + type: 'TestSource', + pos: [20, 30], + size: [200, 100], + flags: { collapsed: true }, + widgets_values: [20, 7] + } + ], + groups: [ + { id: 1, title: 'Around both', bounding: [100, -20, 300, 200] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const inserted = graph._nodes.find( + (node) => node !== source && node !== note + ) + const group = graph.groups.find(({ title }) => title === 'Around both') + assert(inserted && group, 'the insertion reached the canvas') + expect([...inserted.pos]).not.toEqual([20, 30]) + expect([ + group.pos[0] - inserted.pos[0], + group.pos[1] - inserted.pos[1] + ]).toEqual([80, -50]) + }) + it('keeps a group-only insertion on existing nodes that have not been drawn yet', () => { + const { graph, source, note } = buildLiveGraph() + source.pos = [1000, 500] + source.setSize([200, 100]) + note.setSize([200, 100]) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a68'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { id: 1, title: 'Around source', bounding: [990, 460, 240, 160] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const group = graph.groups.find(({ title }) => title === 'Around source') + assert(group, 'the inserted group reached the canvas') + expect([...group.pos]).toEqual([990, 460]) + }) + it('keeps a group on a collapsed node it frames as rendered', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([600, 100]) + note.pos = [1000, 20] + source.collapse(true) + const context = measuringContext(400) + source.updateArea(context) + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a64'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { + id: 1, + title: 'Around collapsed', + bounding: [150, -20, 200, 60] + } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + + const group = graph.groups.find(({ title }) => title === 'Around collapsed') + assert(group, 'the inserted group reached the canvas') + expect([...group.pos]).toEqual([150, -20]) + }) + it('places a group-only insertion that frames no node clear of the canvas content', () => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + source.updateArea() + note.updateArea() + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + + hostEdit( + () => + applyHostOp(host, { + ...HOST_OP, + op_id: '1a60'.padEnd(32, '0'), + op: 'insert_workflow', + workflow: { + nodes: [], + groups: [ + { id: 1, title: 'Inserted group', bounding: [150, 0, 100, 300] } + ] + } + }), + { actor: HOST_OP.actor, catchUp: false } + ) + const group = graph.groups.find(({ title }) => title === 'Inserted group') assert(group, 'the inserted group reached the canvas') expect([...group.pos]).toEqual([680, 20]) diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index 3aec6a00e65..a6969b2c195 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -28,6 +28,7 @@ import { LiteGraph } from '@/lib/litegraph/src/litegraph' import type { LGraph } from '@/lib/litegraph/src/litegraph' +import { containsCentre } from '@/lib/litegraph/src/measure' import { topologicalSortSubgraphs } from '@/lib/litegraph/src/subgraph/subgraphDeduplication' import type { ExportedSubgraph, @@ -62,7 +63,11 @@ import { readSubgraphDefinitions, readSubgraphInteriorWidgets } from './agentSubgraphDefinitions' -import type { Bounds as PlacementRect } from '@comfyorg/shared-frontend-utils/geometry' +import type { + Bounds as PlacementRect, + ReadOnlyRect, + Rect +} from '@comfyorg/shared-frontend-utils/geometry' import { placementOffset } from './batchPlacement' export type NodeChange = 'add' | 'update' | 'delete' @@ -2047,14 +2052,27 @@ export class LiveGraphApplier { width: item.size[0], height: item.size[1] }) - const incoming = [ - ...graph._nodes.filter((node) => createdSet.has(node.id)), - ...createdGroups - ] + const createdNodes = graph._nodes.filter((node) => createdSet.has(node.id)) + const existingNodes = graph._nodes.filter( + (node) => !createdSet.has(node.id) + ) + const ctx = this.deps.getCanvas?.()?.ctx ?? undefined + const existingBounds = existingNodes.map((node) => + isRendered(node) ? node.boundingRect : pendingBounds(node, ctx) + ) + const createdBounds = createdNodes.map((node) => pendingBounds(node, ctx)) + const framesOnlyExisting = (group: LGraphGroup) => + framesAny(group, existingBounds) && !framesAny(group, createdBounds) + const kept = createdGroups.filter(framesOnlyExisting) + const placedGroups = + kept.length > 0 && createdNodes.length === 0 + ? [] + : createdGroups.filter((group) => !kept.includes(group)) + const incoming = [...createdNodes, ...placedGroups] const offset = placementOffset({ existing: [ - ...graph._nodes.filter((node) => !createdSet.has(node.id)), - ...graph.groups.filter((group) => !createdGroups.includes(group)) + ...existingNodes, + ...graph.groups.filter((group) => !placedGroups.includes(group)) ].map(rect), viewport: this.deps.viewportBounds?.() ?? null, incoming: incoming.map(rect), @@ -2067,6 +2085,37 @@ export class LiveGraphApplier { } } +/** True once a frame has measured the node; until then its bounds are zero. */ +function isRendered(node: LGraphNode): boolean { + const [, , width, height] = node.boundingRect + return width > 0 || height > 0 +} + +/** + * The bounds the next frame will give a node that has not been drawn yet, + * measured the way LiteGraph does it, including a collapsed node's title width. + * `measure()` caches the collapsed width the node renders at; that cache is + * left for the next frame to write, because without a canvas context it would + * be written as zero. + */ +function pendingBounds( + node: LGraphNode, + ctx: CanvasRenderingContext2D | undefined +): ReadOnlyRect { + const out: Rect = [0, 0, 0, 0] + const collapsedWidth = node._collapsed_width + node.measure(out, ctx) + node._collapsed_width = collapsedWidth + return out +} + +function framesAny( + group: LGraphGroup, + bounds: readonly ReadOnlyRect[] +): boolean { + return bounds.some((rect) => containsCentre(group._bounding, rect)) +} + function isLinkPresent( graph: LGraph, origin: LGraphNode,