diff --git a/browser_tests/tests/agent/agentInsertWorkflowCanvasResult.spec.ts b/browser_tests/tests/agent/agentInsertWorkflowCanvasResult.spec.ts new file mode 100644 index 00000000000..cfa23ebff99 --- /dev/null +++ b/browser_tests/tests/agent/agentInsertWorkflowCanvasResult.spec.ts @@ -0,0 +1,317 @@ +import { expect, mergeTests } from '@playwright/test' +import type { Op, WidgetCatalog } from '@comfyorg/comfy-multi-player' + +import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' + +import subgraphAsset from '@e2e/assets/subgraphs/agent-subgraph-with-two-promoted-widgets.json' with { type: 'json' } +import { + agentTest, + bootAgentApp, + mockAgentTurnApi, + mockWorkflowPersistence +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const test = mergeTests(agentTest, webSocketFixture) + +// This spec closes the gap `agentSubgraphFollower.spec.ts` and +// `agentTemplatePlacement.spec.ts` leave open: both drive the real CRDT +// follower onto a real canvas, but neither one exercises the `insert_workflow` +// op itself -- one seeds its document directly from a workflow JSON and +// layers plain `add_node`/`connect` ops on top, the other only ever sends +// `add_node` batches. This test sends a single `insert_workflow` op, the way +// the cloud agent's `insert_workflow` tool call actually lands mid-turn, and +// checks the one representative case the unit-level `agentInsertWorkflow.*` +// suites (`src/workbench/extensions/agent/crdt/`) cannot: that a widget +// value, a cross-node link, and a subgraph all read back correctly on the +// live, user-visible canvas afterward -- not just in the projected graph +// snapshot a Vitest test inspects. + +const WORKFLOW_ID = 'a2f6c8b0-0000-4000-8000-000000000010' +const MESSAGE_ID = 'a2f6c8b0-0000-4000-8000-000000000011' +const THREAD_ID = 'a2f6c8b0-0000-4000-8000-000000000012' + +// Reuses the exact subgraph definition (two promoted widgets, `text` and +// `seed`) already proven to materialize correctly in +// `agentSubgraphFollower.spec.ts` -- there it arrives baked into the CRDT +// doc's initial mint; here it arrives as part of a live `insert_workflow` +// tool call instead. +const SUBGRAPH_DEFINITION = subgraphAsset.definitions.subgraphs[0] +const PROMPT_TEXT = 'a photo of a pier' +const SEED_VALUE = 918273 + +// Ids as they appear INSIDE the insert_workflow payload only. The applier +// mints fresh ids for everything an insert_workflow op carries (comfy-multi- +// player's `remapInsertedWorkflowIds`), so these never appear on the live +// canvas -- the test reads the materialized ids back off `window.app!.graph` +// instead of assuming they survive. +const SOURCE_NODE_ID = 20 +const HOST_NODE_ID = 11 +const CLIP_LINK_ID = 21 + +const nodeDefs: Record = { + AgentClipSource: { + name: 'AgentClipSource', + display_name: 'Agent Clip Source', + description: '', + category: 'testing', + python_module: 'testing', + output_node: false, + input: { required: {} }, + output: ['CLIP'], + output_is_list: [false], + output_name: ['CLIP'] + }, + CLIPTextEncode: { + name: 'CLIPTextEncode', + display_name: 'CLIP Text Encode', + description: '', + category: 'conditioning', + python_module: 'nodes', + output_node: false, + input: { + required: { + clip: ['CLIP', { forceInput: true }], + text: ['STRING', { default: '', multiline: true }] + } + }, + output: ['CONDITIONING'], + output_is_list: [false], + output_name: ['CONDITIONING'] + }, + KSampler: { + name: 'KSampler', + display_name: 'KSampler', + description: '', + category: 'sampling', + python_module: 'nodes', + output_node: false, + input: { + required: { + model: ['MODEL', { forceInput: true }], + positive: ['CONDITIONING', { forceInput: true }], + negative: ['CONDITIONING', { forceInput: true }], + latent_image: ['LATENT', { forceInput: true }], + seed: ['INT', { default: 0, min: 0, max: 2147483647 }], + control_after_generate: [ + ['fixed', 'increment', 'decrement', 'randomize'], + { default: 'randomize' } + ], + steps: ['INT', { default: 20, min: 1, max: 10000 }], + cfg: ['FLOAT', { default: 8, min: 0, max: 100 }], + sampler_name: [['euler'], { default: 'euler' }], + scheduler: [['simple'], { default: 'simple' }], + denoise: ['FLOAT', { default: 1, min: 0, max: 1 }] + } + }, + output: ['LATENT'], + output_is_list: [false], + output_name: ['LATENT'] + } +} + +const catalog: WidgetCatalog = { + types: { + AgentClipSource: { widget_order: [] }, + CLIPTextEncode: { widget_order: ['text'] }, + KSampler: { + widget_order: [ + 'seed', + 'control_after_generate', + 'steps', + 'cfg', + 'sampler_name', + 'scheduler', + 'denoise' + ] + } + } +} + +function insertWorkflowOps() { + return [ + { + op_id: 'agent-insert-workflow-canvas-result-op-000'.padEnd(32, '0'), + actor: 'agent:e2e', + base_version: 1, + stamp: [1, 'agent:e2e'], + op: 'insert_workflow', + workflow: { + nodes: [ + { + id: SOURCE_NODE_ID, + type: 'AgentClipSource', + pos: [40, 300], + size: [180, 80], + inputs: [], + outputs: [{ name: 'CLIP', type: 'CLIP', links: [CLIP_LINK_ID] }] + }, + { + id: HOST_NODE_ID, + type: SUBGRAPH_DEFINITION.id, + pos: [400, 300], + size: [210, 168], + inputs: [ + { name: 'clip', type: 'CLIP', link: CLIP_LINK_ID }, + { name: 'model', type: 'MODEL', link: null }, + { name: 'positive', type: 'CONDITIONING', link: null }, + { name: 'negative', type: 'CONDITIONING', link: null }, + { name: 'latent_image', type: 'LATENT', link: null }, + { + name: 'seed', + type: 'INT', + widget: { name: 'seed' }, + link: null + } + ], + outputs: [], + widgets_values: [PROMPT_TEXT, SEED_VALUE] + } + ], + links: [[CLIP_LINK_ID, SOURCE_NODE_ID, 0, HOST_NODE_ID, 0, 'CLIP']], + definitions: { subgraphs: [SUBGRAPH_DEFINITION] } + } + } + ] satisfies Op[] +} + +test.describe( + 'Agent insert_workflow canvas materialization', + { tag: ['@cloud', '@agent', '@canvas', '@node', '@widget', '@subgraph'] }, + () => { + test.use({ connectWebSocketToServer: false }) + + test('renders a promoted widget, a cross-node link, and a subgraph from one insert_workflow tool call', async ({ + page, + getWebSocket + }, testInfo) => { + test.setTimeout(60_000) + await page.setViewportSize({ width: 1920, height: 1280 }) + await page.addInitScript(() => { + localStorage.setItem('Comfy.Agent.CrdtFollower', 'true') + }) + await bootAgentApp(page, true, { + settings: { 'Comfy.VueNodes.Enabled': true }, + objectInfo: nodeDefs, + beforeNavigate: async (page) => { + await mockAgentTurnApi(page, { + message_id: MESSAGE_ID, + thread_id: THREAD_ID, + workflow_id: WORKFLOW_ID + }) + await mockWorkflowPersistence(page, WORKFLOW_ID) + } + }) + const socket = await getWebSocket() + const outboundFrames: string[] = [] + socket.onMessage((message) => outboundFrames.push(String(message))) + + // Drive the turn through the real, public Agent panel -- not a raw + // CRDT call -- exactly as a user would trigger the tool. + const agentPanel = new AgentPanel(page) + await agentPanel.open() + await agentPanel.selectWorkflow() + await agentPanel.sendMessage( + 'Insert the CLIP-conditioned sampler subgraph onto this canvas' + ) + + await expect + .poll(() => outboundFrames, { timeout: 15_000 }) + .toContainEqual( + expect.stringMatching( + new RegExp( + `doc_subscribe.*${WORKFLOW_ID}|${WORKFLOW_ID}.*doc_subscribe` + ) + ) + ) + + // The fake host: an empty document, matching the follower's own empty + // starting state, so the canvas is genuinely blank before the tool runs. + const host = new HostDoc(WORKFLOW_ID, { nodes: [], links: [] }, catalog) + for (const frame of host.initialSync()) socket.send(JSON.stringify(frame)) + + await expect + .poll(() => page.evaluate(() => window.app!.graph.nodes.length)) + .toBe(0) + + // The tool call executes: `insert_workflow` lands as a live doc_update, + // the same delta shape a real turn's CRDT broadcast carries. + const applied = host.apply(insertWorkflowOps()) + socket.send(JSON.stringify(applied)) + + await expect + .poll(() => + page.evaluate(() => { + const graph = window.app!.graph + const host = graph.nodes.find( + (node) => + typeof node.type === 'string' && graph.subgraphs.has(node.type) + ) + return host?.widgets?.length ?? 0 + }) + ) + .toBe(2) + + const canvasState = await page.evaluate(() => { + const graph = window.app!.graph + const host = graph.nodes.find( + (node) => + typeof node.type === 'string' && graph.subgraphs.has(node.type) + )! + const source = graph.nodes.find( + (node) => node.type === 'AgentClipSource' + )! + const linkId = source.outputs[0]?.links?.[0] + const link = + linkId != null + ? [...graph.links.values()].find( + (candidate) => candidate.id === linkId + ) + : undefined + return { + hostId: String(host.id), + sourceId: String(source.id), + widgets: host.widgets?.map(({ name, value }) => ({ name, value })), + linkOriginId: link ? String(link.origin_id) : null, + linkTargetId: link ? String(link.target_id) : null, + linkType: link?.type ?? null + } + }) + + // Widget: the promoted `text`/`seed` values from the inserted subgraph + // read back correctly on the live node. + expect(canvasState.widgets).toEqual([ + { name: 'text', value: PROMPT_TEXT }, + { name: 'seed', value: SEED_VALUE } + ]) + // Link: the freshly-inserted source connects to the freshly-inserted + // subgraph instance's promoted `clip` input, not to nothing / itself. + expect(canvasState.linkOriginId).toBe(canvasState.sourceId) + expect(canvasState.linkTargetId).toBe(canvasState.hostId) + expect(canvasState.linkType).toBe('CLIP') + + await page + .getByRole('button', { name: 'Fit View (.)', exact: true }) + .click() + + // Same widget values, now asserted against the rendered Vue node DOM -- + // the user-visible surface Christian's review asked for, not just the + // graph model. + const hostNode = new VueNodeHelpers(page).getNodeLocator( + canvasState.hostId + ) + await expect(hostNode.getByRole('textbox')).toHaveValue(PROMPT_TEXT) + await expect( + hostNode.getByLabel('seed', { exact: true }).getByRole('spinbutton') + ).toHaveValue(String(SEED_VALUE)) + + await testInfo.attach('insert-workflow-result.png', { + body: await page.screenshot(), + contentType: 'image/png' + }) + }) + } +) diff --git a/src/workbench/extensions/agent/crdt/agentInsertWorkflow.linkTopology.test.ts b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.linkTopology.test.ts new file mode 100644 index 00000000000..f53974e4576 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.linkTopology.test.ts @@ -0,0 +1,554 @@ +/** + * Extended link-materialization coverage for `insert_workflow`, targeting + * the PM-1562 bug class (order-independent numeric link-id derivation) with + * topologies `agentInsertWorkflowLinks.regression.test.ts` does not cover: + * fan-out, several independent chains landing in one batch, links crossing a + * freshly-inserted subgraph's boundary (both directions), a fully-interior + * subgraph link, and node-id collisions that must not cross-wire links. + */ +import { applyOps, mint } from '@comfyorg/comfy-multi-player' +import type { + InsertWorkflowOp, + WidgetCatalog +} from '@comfyorg/comfy-multi-player' +import { beforeEach, describe, expect, it, onTestFinished } from 'vitest' +import * as Y from 'yjs' + +import { + LGraph, + LGraphNode, + LiteGraph, + SubgraphNode +} from '@/lib/litegraph/src/litegraph' +import { + createTestSubgraph, + createTestSubgraphNode, + enableSubgraphNodeCreation +} from '@/lib/litegraph/src/subgraph/__fixtures__/subgraphHelpers' +import { graphScopeOf } from '@/types/graphScopeId' +import { toNodeId } from '@/types/nodeId' + +import { AgentCrdtProjection } from './agentCrdtProjection' +import { inertPlacementPort } from './__fixtures__/inertPlacementPort' +import { FollowerDoc } from './followerDoc' +import { createGraphMutations } from './graphMutations' + +class TestSource extends LGraphNode { + static override title = 'Test Source' + constructor() { + super('Test Source') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +class TestPassthrough extends LGraphNode { + static override title = 'Test Passthrough' + constructor() { + super('Test Passthrough') + this.addInput('image', 'IMAGE') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +class TestSink extends LGraphNode { + static override title = 'Test Sink' + constructor() { + super('Test Sink') + this.addInput('images', 'IMAGE') + this.serialize_widgets = true + } +} + +/** Two image inputs feeding one output, for fan-in topologies. */ +class TestCombine extends LGraphNode { + static override title = 'Test Combine' + constructor() { + super('Test Combine') + this.addInput('image_a', 'IMAGE') + this.addInput('image_b', 'IMAGE') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +const CATALOG: WidgetCatalog = { + types: { + TestSource: { widget_order: [] }, + TestPassthrough: { widget_order: [] }, + TestSink: { widget_order: [] }, + TestCombine: { widget_order: [] } + } +} + +let opSeq = 0 +function insertOp( + workflow: InsertWorkflowOp['workflow'], + opId?: string +): InsertWorkflowOp { + opSeq += 1 + return { + op_id: (opId ?? `insert-link-topology-op-${opSeq}`).padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [1, 'agent:test'], + op: 'insert_workflow', + workflow + } +} + +function bindProjection(workflowId: string, graph: LGraph) { + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + createGraphMutations({ + getScope: () => graphScopeOf(graph), + layout: { createNode: () => {}, deleteNodes: () => {} }, + placement: inertPlacementPort + }), + () => graph, + () => follower.doc + ) + projection.bind(workflowId, follower) + onTestFinished(() => { + projection.destroy() + follower.destroy() + }) + + let seq = 0 + const deliver = (update: Uint8Array, opIds: string[]): boolean => { + follower.applyRemoteUpdate(update) + const committed = projection.applyFrame({ + workflowId, + seq: ++seq, + update, + actor: 'agent:test', + opIds + }) + projection.reconcileLiveGraph(workflowId) + return committed + } + return deliver +} + +function readInputOrigins(node: LGraphNode) { + return node.inputs.map((input, index) => ({ + name: input.name, + origin: node.getInputLink(index)?.origin_id ?? null + })) +} + +function findByType(graph: LGraph, type: string): LGraphNode { + const node = graph._nodes.find((candidate) => candidate.type === type) + if (!node) throw new Error(`no live node of type ${type} was materialized`) + return node +} + +function findAllByType(graph: LGraph, type: string): LGraphNode[] { + return graph._nodes.filter((candidate) => candidate.type === type) +} + +beforeEach(() => { + LiteGraph.registerNodeType('TestSource', TestSource) + LiteGraph.registerNodeType('TestPassthrough', TestPassthrough) + LiteGraph.registerNodeType('TestSink', TestSink) + LiteGraph.registerNodeType('TestCombine', TestCombine) +}) + +describe('insert_workflow materializes varied link topologies', () => { + it('wires a single output fanning out to two independent downstream inputs', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const workflow = { + nodes: [ + { + id: 1, + type: 'TestSource', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [10, 11] }] + }, + { + id: 2, + type: 'TestPassthrough', + inputs: [{ name: 'image', type: 'IMAGE', link: 10 }] + }, + { + id: 3, + type: 'TestSink', + inputs: [{ name: 'images', type: 'IMAGE', link: 11 }] + } + ], + links: [ + [10, 1, 0, 2, 0, 'IMAGE'], + [11, 1, 0, 3, 0, 'IMAGE'] + ] + } + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-fan-out', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const source = findByType(graph, 'TestSource') + const passthrough = findByType(graph, 'TestPassthrough') + const sink = findByType(graph, 'TestSink') + expect(readInputOrigins(passthrough)).toEqual([ + { name: 'image', origin: source.id } + ]) + expect(readInputOrigins(sink)).toEqual([ + { name: 'images', origin: source.id } + ]) + }) + + it('wires three independent chains inserted in one batch without cross-wiring', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const chain = (base: number) => ({ + nodes: [ + { + id: base, + type: 'TestSource', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [base + 100] }] + }, + { + id: base + 1, + type: 'TestSink', + inputs: [{ name: 'images', type: 'IMAGE', link: base + 100 }] + } + ], + links: [[base + 100, base, 0, base + 1, 0, 'IMAGE']] + }) + const workflow = { + nodes: [...chain(1).nodes, ...chain(10).nodes, ...chain(20).nodes], + links: [...chain(1).links, ...chain(10).links, ...chain(20).links] + } + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-parallel-chains', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const sinks = findAllByType(graph, 'TestSink') + expect(sinks).toHaveLength(3) + const sources = findAllByType(graph, 'TestSource') + expect(sources).toHaveLength(3) + // Every sink's origin must be exactly one of the three sources, and no + // two sinks may share the same origin (which would mean a chain crossed + // into another). + const origins = sinks.map((sink) => readInputOrigins(sink)[0]?.origin) + expect(new Set(origins).size).toBe(3) + for (const origin of origins) { + expect(sources.map((s) => s.id)).toContainEqual(origin) + } + }) + + it('wires a link from an external node into a freshly-inserted subgraph instance promoted input', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'images', type: 'IMAGE' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('TestSink')! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + + const instanceHost = createTestSubgraphNode(subgraph, { id: 2 }) + blueprintGraph.add(instanceHost) + const source = LiteGraph.createNode('TestSource')! + source.id = toNodeId(1) + blueprintGraph.add(source) + source.connect(0, instanceHost, 0) + + const workflow = JSON.parse( + JSON.stringify(blueprintGraph.serialize()) + ) as InsertWorkflowOp['workflow'] + + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-into-subgraph', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const liveSource = findByType(graph, 'TestSource') + const instance = graph._nodes.find( + (node): node is SubgraphNode => node instanceof SubgraphNode + ) + expect(instance).toBeDefined() + if (!instance) return + const links = graph.serialize().links + expect( + links.some( + ([, originId, , targetId, targetSlot]) => + originId === liveSource.id && + targetId === instance.id && + targetSlot === 0 + ) + ).toBe(true) + }) + + it('renders a fully-interior link between two nodes inside a freshly-inserted subgraph, via insert_workflow', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + // Build the blueprint from a real litegraph graph (as the paste path + // does), so its serialized shape is exactly what insert_workflow's + // applier remaps: a subgraph definition whose interior link has no + // promoted host slot at all. + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ rootGraph: blueprintGraph }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const source = LiteGraph.createNode('TestSource')! + source.id = toNodeId(1) + subgraph.add(source) + const sink = LiteGraph.createNode('TestSink')! + sink.id = toNodeId(2) + subgraph.add(sink) + source.connect(0, sink, 0) + + const host = createTestSubgraphNode(subgraph, { id: 3 }) + blueprintGraph.add(host) + const workflow = JSON.parse( + JSON.stringify(blueprintGraph.serialize()) + ) as InsertWorkflowOp['workflow'] + + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const op = insertOp(workflow) + expect(applyOps(hostDoc, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-interior-link', graph) + expect(deliver(Y.encodeStateAsUpdate(hostDoc), [op.op_id])).toBe(true) + + const instance = graph._nodes.find( + (node): node is SubgraphNode => node instanceof SubgraphNode + ) + expect(instance).toBeDefined() + if (!instance) return + + const interiorSource = instance.subgraph._nodes.find( + (n) => n.type === 'TestSource' + ) + const interiorSink = instance.subgraph._nodes.find( + (n) => n.type === 'TestSink' + ) + expect(interiorSource).toBeDefined() + expect(interiorSink).toBeDefined() + if (!interiorSource || !interiorSink) return + expect(readInputOrigins(interiorSink)).toEqual([ + { name: 'images', origin: interiorSource.id } + ]) + }) + + it('does not cross-wire links when two inserts in one batch collide on node ids', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + // Both inserts number their source and sink identically (1, 2); only the + // op_id disambiguates the derived live ids. If link resolution ever + // collapsed ids down to their trailing numeric segment, node 1's output + // in op A could end up wired to node 2's input in op B. + const workflow = { + nodes: [ + { + id: 1, + type: 'TestSource', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [99] }] + }, + { + id: 2, + type: 'TestSink', + inputs: [{ name: 'images', type: 'IMAGE', link: 99 }] + } + ], + links: [[99, 1, 0, 2, 0, 'IMAGE']] + } + const opA = insertOp(workflow, 'insert-collide-a') + const opB = insertOp(workflow, 'insert-collide-b') + expect(applyOps(host, [opA, opB], CATALOG).outcomes).toEqual([ + { op_id: opA.op_id, outcome: 'applied' }, + { op_id: opB.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-batch-collision', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [opA.op_id, opB.op_id])).toBe( + true + ) + + const sources = findAllByType(graph, 'TestSource') + const sinks = findAllByType(graph, 'TestSink') + expect(sources).toHaveLength(2) + expect(sinks).toHaveLength(2) + + for (const sink of sinks) { + const origin = readInputOrigins(sink)[0]?.origin + // Each sink's origin must be A source that exists, and every sink's + // origin is unique to that sink's own op (no shared origin). + expect(sources.map((s) => s.id)).toContainEqual(origin) + } + const origins = sinks.map((sink) => readInputOrigins(sink)[0]?.origin) + expect(new Set(origins).size).toBe(2) + }) + + it('wires a link from a freshly-inserted subgraph instance promoted output to an external node', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + outputs: [{ name: 'IMAGE', type: 'IMAGE' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('TestSource')! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.outputNode.slots[0].connect(interior.outputs[0], interior) + + const instanceHost = createTestSubgraphNode(subgraph, { id: 2 }) + blueprintGraph.add(instanceHost) + const sink = LiteGraph.createNode('TestSink')! + sink.id = toNodeId(1) + blueprintGraph.add(sink) + instanceHost.connect(0, sink, 0) + + const workflow = JSON.parse( + JSON.stringify(blueprintGraph.serialize()) + ) as InsertWorkflowOp['workflow'] + + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-out-of-subgraph', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const liveSink = findByType(graph, 'TestSink') + const instance = graph._nodes.find( + (node): node is SubgraphNode => node instanceof SubgraphNode + ) + expect(instance).toBeDefined() + if (!instance) return + expect(readInputOrigins(liveSink)).toEqual([ + { name: 'images', origin: instance.id } + ]) + }) + + it('wires a fan-in link where two independent sources feed one combine node', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const workflow = { + nodes: [ + { + id: 1, + type: 'TestSource', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [201] }] + }, + { + id: 2, + type: 'TestSource', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [202] }] + }, + { + id: 3, + type: 'TestCombine', + inputs: [ + { name: 'image_a', type: 'IMAGE', link: 201 }, + { name: 'image_b', type: 'IMAGE', link: 202 } + ] + } + ], + links: [ + [201, 1, 0, 3, 0, 'IMAGE'], + [202, 2, 0, 3, 1, 'IMAGE'] + ] + } + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-fan-in', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const [sourceA, sourceB] = findAllByType(graph, 'TestSource') + const combine = findByType(graph, 'TestCombine') + expect(readInputOrigins(combine)).toEqual([ + { name: 'image_a', origin: sourceA.id }, + { name: 'image_b', origin: sourceB.id } + ]) + }) + + it('wires a link inserted incrementally into an already-materialized chain', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const seedOp = insertOp( + { nodes: [{ id: 1, type: 'TestSource' }], links: [] }, + 'insert-incr-seed' + ) + expect(applyOps(host, [seedOp], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + const deliver = bindProjection('wf-incremental-topology', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [seedOp.op_id])).toBe(true) + expect(graph._nodes).toHaveLength(1) + + const before = Y.encodeStateVector(host) + const secondOp = insertOp( + { + nodes: [ + { + id: 2, + type: 'TestPassthrough', + inputs: [{ name: 'image', type: 'IMAGE', link: 300 }], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [301] }] + }, + { + id: 3, + type: 'TestSink', + inputs: [{ name: 'images', type: 'IMAGE', link: 301 }] + } + ], + links: [[301, 2, 0, 3, 0, 'IMAGE']] + }, + 'insert-incr-second' + ) + expect(applyOps(host, [secondOp], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + expect(deliver(Y.encodeStateAsUpdate(host, before), [secondOp.op_id])).toBe( + true + ) + + const passthrough = findByType(graph, 'TestPassthrough') + const sink = findByType(graph, 'TestSink') + expect(readInputOrigins(sink)).toEqual([ + { name: 'images', origin: passthrough.id } + ]) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentInsertWorkflow.subgraphs.test.ts b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.subgraphs.test.ts new file mode 100644 index 00000000000..b08aa1c710e --- /dev/null +++ b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.subgraphs.test.ts @@ -0,0 +1,573 @@ +/** + * Extended subgraph coverage for `insert_workflow`, complementing + * `agentSubgraphInsertWorkflow.regression.test.ts` (one promoted-widget + * blueprint paste). This file drives the same real-graph-blueprint pattern + * (build with litegraph, `serialize()`, feed to `insert_workflow`) across + * scenarios that file does not cover: multiple subgraph definitions in one + * op, a promoted widget surviving a later `set_widget`, multiple widget + * types promoted together, subgraph-instance id collisions, several + * instances of one definition in a single op, and coexistence with a + * pre-existing instance. + * + * NOTE: an interior node that is ITSELF a subgraph instance (a subgraph + * nested inside another subgraph, as opposed to two sibling definitions) + * does not currently materialize through this pipeline — its `type` still + * names the un-remapped blueprint definition id, which nothing registers on + * the live side, so it falls back to a plain, widget-less node. That gap is + * outside this PR's fix scope; it is called out in the PR description as a + * follow-up rather than silently asserted around. + */ +import { applyOps, mint } from '@comfyorg/comfy-multi-player' +import type { + InsertWorkflowOp, + Op, + WidgetCatalog +} from '@comfyorg/comfy-multi-player' +import { beforeEach, describe, expect, it, onTestFinished } from 'vitest' +import * as Y from 'yjs' + +import { + LGraph, + LGraphNode, + LiteGraph, + SubgraphNode +} from '@/lib/litegraph/src/litegraph' +import { + createTestSubgraph, + createTestSubgraphNode, + enableSubgraphNodeCreation +} from '@/lib/litegraph/src/subgraph/__fixtures__/subgraphHelpers' +import { useWidgetValueStore } from '@/stores/widgetValueStore' +import { graphScopeOf } from '@/types/graphScopeId' +import { toNodeId } from '@/types/nodeId' +import type { WidgetValue } from '@/types/simplifiedWidget' + +import { AgentCrdtProjection } from './agentCrdtProjection' +import { inertPlacementPort } from './__fixtures__/inertPlacementPort' +import { FollowerDoc } from './followerDoc' +import { createGraphMutations } from './graphMutations' + +class NumberWidgetNode extends LGraphNode { + constructor() { + super('number-widget') + const input = this.addInput('value', 'NUMBER') + input.widget = { name: 'value' } + this.addWidget('number', 'value', 1, () => {}) + } +} + +class ComboWidgetNode extends LGraphNode { + constructor() { + super('combo-widget') + const input = this.addInput('mode', 'STRING') + input.widget = { name: 'mode' } + this.addWidget('combo', 'mode', 'fixed', () => {}, { + values: ['fixed', 'randomize'] + }) + } +} + +class ToggleWidgetNode extends LGraphNode { + constructor() { + super('toggle-widget') + const input = this.addInput('enabled', 'BOOLEAN') + input.widget = { name: 'enabled' } + this.addWidget('toggle', 'enabled', false, () => {}) + } +} + +/** Two promoted widgets on a single interior node, different types. */ +class DualWidgetNode extends LGraphNode { + constructor() { + super('dual-widget') + const modeInput = this.addInput('mode', 'STRING') + modeInput.widget = { name: 'mode' } + this.addWidget('combo', 'mode', 'fixed', () => {}, { + values: ['fixed', 'randomize'] + }) + const stepsInput = this.addInput('steps', 'NUMBER') + stepsInput.widget = { name: 'steps' } + this.addWidget('number', 'steps', 20, () => {}) + } +} + +const CATALOG: WidgetCatalog = { + types: { + 'number-widget': { widget_order: ['value'] }, + 'combo-widget': { widget_order: ['mode'] }, + 'toggle-widget': { widget_order: ['enabled'] }, + 'dual-widget': { widget_order: ['mode', 'steps'] } + } +} + +beforeEach(() => { + LiteGraph.registerNodeType('number-widget', NumberWidgetNode) + LiteGraph.registerNodeType('combo-widget', ComboWidgetNode) + LiteGraph.registerNodeType('toggle-widget', ToggleWidgetNode) + LiteGraph.registerNodeType('dual-widget', DualWidgetNode) +}) + +let opSeq = 0 +function insertOp( + workflow: InsertWorkflowOp['workflow'], + opId?: string +): InsertWorkflowOp { + opSeq += 1 + return { + op_id: (opId ?? `insert-subgraph-op-${opSeq}`).padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [1, 'agent:test'], + op: 'insert_workflow', + workflow + } +} + +function setWidgetOp( + opId: string, + seq: number, + nodeId: string | number, + widget: string, + value: unknown, + valueIndex: number +): Op { + return { + op_id: opId.padEnd(32, '0'), + actor: 'agent:test', + base_version: seq, + stamp: [seq, 'agent:test'], + op: 'set_widget', + node_id: nodeId, + widget, + value, + promoted: { value_index: valueIndex, host_widgets_values: [value] } + } as unknown as Op +} + +function bindProjection(workflowId: string, graph: LGraph) { + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + createGraphMutations({ + getScope: () => graphScopeOf(graph), + layout: { createNode: () => {}, deleteNodes: () => {} }, + placement: inertPlacementPort + }), + () => graph, + () => follower.doc + ) + projection.bind(workflowId, follower) + onTestFinished(() => { + projection.destroy() + follower.destroy() + }) + + let seq = 0 + const deliver = (update: Uint8Array, opIds: string[]): boolean => { + follower.applyRemoteUpdate(update) + const committed = projection.applyFrame({ + workflowId, + seq: ++seq, + update, + actor: 'agent:test', + opIds + }) + projection.reconcileLiveGraph(workflowId) + return committed + } + return deliver +} + +function findSubgraphInstances(graph: LGraph): SubgraphNode[] { + return graph._nodes.filter( + (node): node is SubgraphNode => node instanceof SubgraphNode + ) +} + +function serializeBlueprint(graph: LGraph): InsertWorkflowOp['workflow'] { + return JSON.parse( + JSON.stringify(graph.serialize()) + ) as InsertWorkflowOp['workflow'] +} + +/** + * A subgraph host's `widgets_values` is a named record when it promotes + * named widgets, even though `LGraphNode`'s general type only declares the + * positional-array shape `ISerialisedNode` uses. Read it through this one + * narrow accessor rather than asserting at every call site. + */ +function promotedWidgetValues(node: SubgraphNode): Record { + return node.widgets_values as unknown as Record +} + +describe('insert_workflow materializes subgraphs correctly', () => { + it('materializes two distinct subgraph definitions carried by the same insert_workflow op', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + // Two SEPARATE, independently-instantiated definitions, built from one + // blueprint that declares them together (as a paste of two grouped + // selections would). This exercises insert_workflow's handling of a + // `workflow.definitions.subgraphs` list with more than one entry in a + // single op, each resolved to its own registration and materialized + // correctly. + const blueprintGraph = new LGraph() + const subgraphA = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'value', type: 'NUMBER' }] + }) + blueprintGraph.subgraphs.set(subgraphA.id, subgraphA) + const leafA = LiteGraph.createNode('number-widget')! + leafA.id = toNodeId(50) + subgraphA.add(leafA) + subgraphA.inputNode.slots[0].connect(leafA.inputs[0], leafA) + const hostA = createTestSubgraphNode(subgraphA, { id: 1 }) + hostA.widgets[0].value = 11 + blueprintGraph.add(hostA) + + const subgraphB = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'value', type: 'NUMBER' }] + }) + blueprintGraph.subgraphs.set(subgraphB.id, subgraphB) + const leafB = LiteGraph.createNode('number-widget')! + leafB.id = toNodeId(60) + subgraphB.add(leafB) + subgraphB.inputNode.slots[0].connect(leafB.inputs[0], leafB) + const hostB = createTestSubgraphNode(subgraphB, { id: 2 }) + hostB.widgets[0].value = 22 + blueprintGraph.add(hostB) + + const workflow = serializeBlueprint(blueprintGraph) + expect(workflow.definitions?.subgraphs).toHaveLength(2) + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const op = insertOp(workflow) + expect(applyOps(hostDoc, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-nested-definitions', graph) + expect(deliver(Y.encodeStateAsUpdate(hostDoc), [op.op_id])).toBe(true) + + const instances = findSubgraphInstances(graph) + expect(instances).toHaveLength(2) + expect( + new Set(instances.map((i) => promotedWidgetValues(i).value)) + ).toEqual(new Set([11, 22])) + // Each instance resolved to its OWN definition, not a shared/confused one. + expect(instances[0].subgraph).not.toBe(instances[1].subgraph) + }) + + it('carries a promoted widget through insert_workflow and preserves it across a subsequent set_widget', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'value', type: 'NUMBER' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('number-widget')! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + const host = createTestSubgraphNode(subgraph, { id: 1 }) + blueprintGraph.add(host) + host.widgets[0].value = 5 + + const workflow = serializeBlueprint(blueprintGraph) + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const deliver = bindProjection('wf-promoted-survives-set', graph) + deliver(Y.encodeStateAsUpdate(hostDoc), []) + + const insert = insertOp(workflow) + let vector = Y.encodeStateVector(hostDoc) + expect(applyOps(hostDoc, [insert], CATALOG).outcomes).toEqual([ + { op_id: insert.op_id, outcome: 'applied' } + ]) + expect( + deliver(Y.encodeStateAsUpdate(hostDoc, vector), [insert.op_id]) + ).toBe(true) + + const [instance] = findSubgraphInstances(graph) + expect(instance).toBeDefined() + expect(instance.widgets[0]?.value).toBe(5) + const promotedWidgetId = instance.inputs[0]?.widgetId + expect(promotedWidgetId).toBeDefined() + + vector = Y.encodeStateVector(hostDoc) + const write = setWidgetOp( + 'insert-then-set-widget', + 2, + String(instance.id), + 'value', + 42, + 0 + ) + expect(applyOps(hostDoc, [write], CATALOG).outcomes).toEqual([ + { op_id: write.op_id, outcome: 'applied' } + ]) + expect(deliver(Y.encodeStateAsUpdate(hostDoc, vector), [write.op_id])).toBe( + true + ) + + expect(graph._nodes.find((n) => n.id === instance.id)).toBe(instance) + expect(instance.widgets[0]?.value).toBe(42) + if (promotedWidgetId) { + expect(useWidgetValueStore().getWidget(promotedWidgetId)?.value).toBe(42) + } + }) + + it('promotes two different widget types on the same host and renders both correctly', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [ + { name: 'mode', type: 'STRING' }, + { name: 'steps', type: 'NUMBER' } + ] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const comboInterior = LiteGraph.createNode('combo-widget')! + comboInterior.id = toNodeId(8) + subgraph.add(comboInterior) + subgraph.inputNode.slots[0].connect(comboInterior.inputs[0], comboInterior) + const numberInterior = LiteGraph.createNode('number-widget')! + numberInterior.id = toNodeId(9) + subgraph.add(numberInterior) + subgraph.inputNode.slots[1].connect( + numberInterior.inputs[0], + numberInterior + ) + const host = createTestSubgraphNode(subgraph, { id: 1 }) + blueprintGraph.add(host) + const modeWidget = host.widgets.find((w) => w.name === 'mode') + const stepsWidget = host.widgets.find((w) => w.name === 'steps') + if (!modeWidget || !stepsWidget) { + throw new Error('blueprint host missing an expected promoted widget') + } + modeWidget.value = 'randomize' + stepsWidget.value = 77 + + const workflow = serializeBlueprint(blueprintGraph) + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const op = insertOp(workflow) + expect(applyOps(hostDoc, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-dual-promoted', graph) + expect(deliver(Y.encodeStateAsUpdate(hostDoc), [op.op_id])).toBe(true) + + const [instance] = findSubgraphInstances(graph) + expect(instance).toBeDefined() + // Read through the host's keyed `widgets_values` rather than the live + // `widgets` array: both promoted values land here correctly. + expect(promotedWidgetValues(instance)).toEqual({ + mode: 'randomize', + steps: 77 + }) + }) + + it('two subgraph instances sharing an original host id, in the SAME graph, keep independent promoted values', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + function makeWorkflow(value: number) { + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'value', type: 'NUMBER' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('number-widget')! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + const host = createTestSubgraphNode(subgraph, { id: 9 }) + blueprintGraph.add(host) + host.widgets[0].value = value + return serializeBlueprint(blueprintGraph) + } + + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const deliver = bindProjection('wf-subgraph-collision', graph) + deliver(Y.encodeStateAsUpdate(hostDoc), []) + + const opA = insertOp(makeWorkflow(1), 'insert-subgraph-collision-a') + let vector = Y.encodeStateVector(hostDoc) + expect(applyOps(hostDoc, [opA], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + expect(deliver(Y.encodeStateAsUpdate(hostDoc, vector), [opA.op_id])).toBe( + true + ) + + const opB = insertOp(makeWorkflow(2), 'insert-subgraph-collision-b') + vector = Y.encodeStateVector(hostDoc) + expect(applyOps(hostDoc, [opB], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + expect(deliver(Y.encodeStateAsUpdate(hostDoc, vector), [opB.op_id])).toBe( + true + ) + + const instances = findSubgraphInstances(graph) + expect(instances).toHaveLength(2) + const [first, second] = instances + expect(first.id).not.toBe(second.id) + expect( + new Set([first.widgets[0]?.value, second.widgets[0]?.value]) + ).toEqual(new Set([1, 2])) + }) + + it('materializes several independent instances of the same definition inserted in one op', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'value', type: 'NUMBER' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('number-widget')! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + + const hostA = createTestSubgraphNode(subgraph, { id: 1 }) + hostA.widgets[0].value = 100 + blueprintGraph.add(hostA) + const hostB = createTestSubgraphNode(subgraph, { id: 2 }) + hostB.widgets[0].value = 200 + blueprintGraph.add(hostB) + + const workflow = serializeBlueprint(blueprintGraph) + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const op = insertOp(workflow) + expect(applyOps(hostDoc, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-multi-instance', graph) + expect(deliver(Y.encodeStateAsUpdate(hostDoc), [op.op_id])).toBe(true) + + const instances = findSubgraphInstances(graph) + expect(instances).toHaveLength(2) + expect( + new Set(instances.map((instance) => instance.widgets[0]?.value)) + ).toEqual(new Set([100, 200])) + }) + + it('inserts a subgraph instance without disturbing a pre-existing, unrelated instance', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + function makeWorkflow( + nodeType: string, + hostId: number, + value: WidgetValue + ) { + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'v', type: '*' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode(nodeType)! + interior.id = toNodeId(7) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + const host = createTestSubgraphNode(subgraph, { id: hostId }) + host.widgets[0].value = value + blueprintGraph.add(host) + return serializeBlueprint(blueprintGraph) + } + + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const deliver = bindProjection('wf-coexisting-subgraphs', graph) + deliver(Y.encodeStateAsUpdate(hostDoc), []) + + const firstOp = insertOp( + makeWorkflow('toggle-widget', 1, true), + 'insert-coexist-first' + ) + let vector = Y.encodeStateVector(hostDoc) + expect(applyOps(hostDoc, [firstOp], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + expect( + deliver(Y.encodeStateAsUpdate(hostDoc, vector), [firstOp.op_id]) + ).toBe(true) + const [firstInstance] = findSubgraphInstances(graph) + expect(promotedWidgetValues(firstInstance)).toEqual({ v: true }) + + const secondOp = insertOp( + makeWorkflow('combo-widget', 2, 'randomize'), + 'insert-coexist-second' + ) + vector = Y.encodeStateVector(hostDoc) + expect(applyOps(hostDoc, [secondOp], CATALOG).outcomes[0]?.outcome).toBe( + 'applied' + ) + expect( + deliver(Y.encodeStateAsUpdate(hostDoc, vector), [secondOp.op_id]) + ).toBe(true) + + const instances = findSubgraphInstances(graph) + expect(instances).toHaveLength(2) + // The first instance must still carry its original value untouched. + const stillFirst = instances.find((i) => i.id === firstInstance.id) + const other = instances.find((i) => i.id !== firstInstance.id) + expect(stillFirst && promotedWidgetValues(stillFirst)).toEqual({ + v: true + }) + expect(other && promotedWidgetValues(other)).toEqual({ v: 'randomize' }) + }) + + it('renders a promoted toggle widget on a subgraph instance', () => { + const graph = new LGraph() + onTestFinished(enableSubgraphNodeCreation(graph)) + + const blueprintGraph = new LGraph() + const subgraph = createTestSubgraph({ + rootGraph: blueprintGraph, + inputs: [{ name: 'enabled', type: 'BOOLEAN' }] + }) + blueprintGraph.subgraphs.set(subgraph.id, subgraph) + const interior = LiteGraph.createNode('toggle-widget')! + interior.id = toNodeId(4) + subgraph.add(interior) + subgraph.inputNode.slots[0].connect(interior.inputs[0], interior) + const host = createTestSubgraphNode(subgraph, { id: 1 }) + host.widgets[0].value = true + blueprintGraph.add(host) + + const workflow = serializeBlueprint(blueprintGraph) + const hostDoc = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => hostDoc.destroy()) + const op = insertOp(workflow) + expect(applyOps(hostDoc, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-toggle-promoted', graph) + expect(deliver(Y.encodeStateAsUpdate(hostDoc), [op.op_id])).toBe(true) + + const [instance] = findSubgraphInstances(graph) + expect(instance).toBeDefined() + expect(promotedWidgetValues(instance)).toEqual({ + enabled: true + }) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentInsertWorkflow.templates.test.ts b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.templates.test.ts new file mode 100644 index 00000000000..6f760d11a05 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.templates.test.ts @@ -0,0 +1,785 @@ +/** + * Coverage for `insert_workflow` against workflow shapes representative of + * the templates catalogue referenced in this repo's `AGENTS.md` (checkpoint + * loaders, img2img, ControlNet, LoRA, video). The actual catalogue is + * GCS-hosted (`gs://cloud-workflow-templates/...`) and this repo vendors no + * local fixture set for it, so these are literal graphs built to the same + * node/widget shapes as the well-known public templates, exercised through + * the same real pipeline every other file in this group drives + * (comfy-multi-player's `insert_workflow` applier -> `AgentCrdtProjection` + * -> `graphMutations` -> `agentNodeMaterializer`), asserting every node + * renders with correct widgets and every link lands. + */ +import { applyOps, mint } from '@comfyorg/comfy-multi-player' +import type { + InsertWorkflowOp, + WidgetCatalog +} from '@comfyorg/comfy-multi-player' +import { beforeEach, describe, expect, it, onTestFinished } from 'vitest' +import * as Y from 'yjs' + +import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { + stripGraphPrefix, + useWidgetValueStore +} from '@/stores/widgetValueStore' +import { graphScopeOf } from '@/types/graphScopeId' + +import { AgentCrdtProjection } from './agentCrdtProjection' +import { inertPlacementPort } from './__fixtures__/inertPlacementPort' +import { FollowerDoc } from './followerDoc' +import { createGraphMutations } from './graphMutations' + +class TestCheckpointLoader extends LGraphNode { + static override title = 'Test Checkpoint Loader' + constructor() { + super('Test Checkpoint Loader') + this.addOutput('MODEL', 'MODEL') + this.addOutput('CLIP', 'CLIP') + this.addOutput('VAE', 'VAE') + this.addWidget('combo', 'ckpt_name', 'sd_xl_base.safetensors', () => {}, { + values: ['sd_xl_base.safetensors', 'sd_xl_refiner.safetensors'] + }) + this.serialize_widgets = true + } +} + +class TestCLIPTextEncode extends LGraphNode { + static override title = 'Test CLIP Text Encode' + constructor() { + super('Test CLIP Text Encode') + this.addInput('clip', 'CLIP') + this.addOutput('CONDITIONING', 'CONDITIONING') + this.addCustomWidget({ + name: 'text', + type: 'customtext', + value: '', + options: { multiline: true }, + y: 0 + }) + this.serialize_widgets = true + } +} + +class TestKSampler extends LGraphNode { + static override title = 'Test KSampler' + constructor() { + super('Test KSampler') + this.addInput('model', 'MODEL') + this.addInput('positive', 'CONDITIONING') + this.addInput('negative', 'CONDITIONING') + this.addInput('latent_image', 'LATENT') + this.addOutput('LATENT', 'LATENT') + this.addWidget('number', 'seed', 0, () => {}) + this.addWidget('number', 'steps', 20, () => {}) + this.addWidget('number', 'cfg', 8, () => {}) + this.addWidget('combo', 'sampler_name', 'euler', () => {}, { + values: ['euler', 'dpmpp_2m'] + }) + this.addWidget('combo', 'scheduler', 'normal', () => {}, { + values: ['normal', 'karras'] + }) + this.addWidget('number', 'denoise', 1, () => {}) + this.serialize_widgets = true + } +} + +class TestEmptyLatentImage extends LGraphNode { + static override title = 'Test Empty Latent Image' + constructor() { + super('Test Empty Latent Image') + this.addOutput('LATENT', 'LATENT') + this.addWidget('number', 'width', 1024, () => {}) + this.addWidget('number', 'height', 1024, () => {}) + this.addWidget('number', 'batch_size', 1, () => {}) + this.serialize_widgets = true + } +} + +class TestVAEDecode extends LGraphNode { + static override title = 'Test VAE Decode' + constructor() { + super('Test VAE Decode') + this.addInput('samples', 'LATENT') + this.addInput('vae', 'VAE') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +class TestVAEEncode extends LGraphNode { + static override title = 'Test VAE Encode' + constructor() { + super('Test VAE Encode') + this.addInput('pixels', 'IMAGE') + this.addInput('vae', 'VAE') + this.addOutput('LATENT', 'LATENT') + this.serialize_widgets = true + } +} + +class TestLoadImage extends LGraphNode { + static override title = 'Test Load Image' + constructor() { + super('Test Load Image') + this.addOutput('IMAGE', 'IMAGE') + this.addWidget('combo', 'image', 'example.png', () => {}, { + values: ['example.png'] + }) + this.serialize_widgets = true + } +} + +class TestSaveImage extends LGraphNode { + static override title = 'Test Save Image' + constructor() { + super('Test Save Image') + this.addInput('images', 'IMAGE') + this.addWidget('text', 'filename_prefix', 'ComfyUI', () => {}) + this.serialize_widgets = true + } +} + +class TestSaveVideo extends LGraphNode { + static override title = 'Test Save Video' + constructor() { + super('Test Save Video') + this.addInput('images', 'IMAGE') + this.addWidget('combo', 'format', 'mp4', () => {}, { + values: ['mp4', 'webm'] + }) + this.addWidget('number', 'fps', 24, () => {}) + this.serialize_widgets = true + } +} + +class TestControlNetLoader extends LGraphNode { + static override title = 'Test ControlNet Loader' + constructor() { + super('Test ControlNet Loader') + this.addOutput('CONTROL_NET', 'CONTROL_NET') + this.addWidget('combo', 'control_net_name', 'canny.safetensors', () => {}, { + values: ['canny.safetensors', 'depth.safetensors'] + }) + this.serialize_widgets = true + } +} + +class TestControlNetApply extends LGraphNode { + static override title = 'Test ControlNet Apply' + constructor() { + super('Test ControlNet Apply') + this.addInput('conditioning', 'CONDITIONING') + this.addInput('control_net', 'CONTROL_NET') + this.addInput('image', 'IMAGE') + this.addOutput('CONDITIONING', 'CONDITIONING') + this.addWidget('number', 'strength', 1, () => {}) + this.serialize_widgets = true + } +} + +class TestLoraLoader extends LGraphNode { + static override title = 'Test LoRA Loader' + constructor() { + super('Test LoRA Loader') + this.addInput('model', 'MODEL') + this.addInput('clip', 'CLIP') + this.addOutput('MODEL', 'MODEL') + this.addOutput('CLIP', 'CLIP') + this.addWidget('combo', 'lora_name', 'add_detail.safetensors', () => {}, { + values: ['add_detail.safetensors', 'style_anime.safetensors'] + }) + this.addWidget('number', 'strength_model', 1, () => {}) + this.addWidget('number', 'strength_clip', 1, () => {}) + this.serialize_widgets = true + } +} + +const CATALOG: WidgetCatalog = { + types: { + 'Test Checkpoint Loader': { widget_order: ['ckpt_name'] }, + 'Test CLIP Text Encode': { widget_order: ['text'] }, + 'Test KSampler': { + widget_order: [ + 'seed', + 'steps', + 'cfg', + 'sampler_name', + 'scheduler', + 'denoise' + ] + }, + 'Test Empty Latent Image': { + widget_order: ['width', 'height', 'batch_size'] + }, + 'Test VAE Decode': { widget_order: [] }, + 'Test VAE Encode': { widget_order: [] }, + 'Test Load Image': { widget_order: ['image'] }, + 'Test Save Image': { widget_order: ['filename_prefix'] }, + 'Test Save Video': { widget_order: ['format', 'fps'] }, + 'Test ControlNet Loader': { widget_order: ['control_net_name'] }, + 'Test ControlNet Apply': { widget_order: ['strength'] }, + 'Test LoRA Loader': { + widget_order: ['lora_name', 'strength_model', 'strength_clip'] + } + } +} + +beforeEach(() => { + LiteGraph.registerNodeType('Test Checkpoint Loader', TestCheckpointLoader) + LiteGraph.registerNodeType('Test CLIP Text Encode', TestCLIPTextEncode) + LiteGraph.registerNodeType('Test KSampler', TestKSampler) + LiteGraph.registerNodeType('Test Empty Latent Image', TestEmptyLatentImage) + LiteGraph.registerNodeType('Test VAE Decode', TestVAEDecode) + LiteGraph.registerNodeType('Test VAE Encode', TestVAEEncode) + LiteGraph.registerNodeType('Test Load Image', TestLoadImage) + LiteGraph.registerNodeType('Test Save Image', TestSaveImage) + LiteGraph.registerNodeType('Test Save Video', TestSaveVideo) + LiteGraph.registerNodeType('Test ControlNet Loader', TestControlNetLoader) + LiteGraph.registerNodeType('Test ControlNet Apply', TestControlNetApply) + LiteGraph.registerNodeType('Test LoRA Loader', TestLoraLoader) +}) + +let opSeq = 0 +function insertOp( + workflow: InsertWorkflowOp['workflow'], + opId?: string +): InsertWorkflowOp { + opSeq += 1 + return { + op_id: (opId ?? `insert-template-op-${opSeq}`).padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [1, 'agent:test'], + op: 'insert_workflow', + workflow + } +} + +function bindProjection(workflowId: string, graph: LGraph) { + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + createGraphMutations({ + getScope: () => graphScopeOf(graph), + layout: { createNode: () => {}, deleteNodes: () => {} }, + placement: inertPlacementPort + }), + () => graph, + () => follower.doc + ) + projection.bind(workflowId, follower) + onTestFinished(() => { + projection.destroy() + follower.destroy() + }) + + let seq = 0 + const deliver = (update: Uint8Array, opIds: string[]): boolean => { + follower.applyRemoteUpdate(update) + const committed = projection.applyFrame({ + workflowId, + seq: ++seq, + update, + actor: 'agent:test', + opIds + }) + projection.reconcileLiveGraph(workflowId) + return committed + } + return deliver +} + +function findByType(graph: LGraph, type: string): LGraphNode { + const node = graph._nodes.find((candidate) => candidate.type === type) + if (!node) throw new Error(`no live node of type ${type} was materialized`) + return node +} + +function findAllByType(graph: LGraph, type: string): LGraphNode[] { + return graph._nodes.filter((candidate) => candidate.type === type) +} + +function readInputOrigins(node: LGraphNode) { + return node.inputs.map((input, index) => ({ + name: input.name, + origin: node.getInputLink(index)?.origin_id ?? null + })) +} + +function renderedWidgetValues(graphId: string, nodeId: string) { + const widgetValueStore = useWidgetValueStore() + const bareNodeId = stripGraphPrefix(nodeId) + if (!bareNodeId) return {} + const values: Record = {} + for (const id of widgetValueStore.getNodeWidgetIds(graphId, bareNodeId)) { + const widget = widgetValueStore.getWidget(id) + if (widget) values[widget.name] = widget.value + } + return values +} + +function insertAndDeliver( + graph: LGraph, + workflowId: string, + workflow: InsertWorkflowOp['workflow'] +) { + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const deliver = bindProjection(workflowId, graph) + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + return { host, deliver } +} + +describe('insert_workflow materializes representative workflow templates', () => { + it('a classic checkpoint text-to-image pipeline renders fully', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test Checkpoint Loader', + widgets_values: ['sd_xl_refiner.safetensors'], + outputs: [ + { name: 'MODEL', type: 'MODEL', links: [10] }, + { name: 'CLIP', type: 'CLIP', links: [11, 12] }, + { name: 'VAE', type: 'VAE', links: [13] } + ] + }, + { + id: 2, + type: 'Test CLIP Text Encode', + widgets_values: ['a glorious spooky duck'], + inputs: [{ name: 'clip', type: 'CLIP', link: 11 }], + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [20] }] + }, + { + id: 3, + type: 'Test CLIP Text Encode', + widgets_values: ['blurry, low quality'], + inputs: [{ name: 'clip', type: 'CLIP', link: 12 }], + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [21] }] + }, + { + id: 4, + type: 'Test Empty Latent Image', + widgets_values: [512, 768, 2], + outputs: [{ name: 'LATENT', type: 'LATENT', links: [22] }] + }, + { + id: 5, + type: 'Test KSampler', + widgets_values: [12345, 30, 6.5, 'dpmpp_2m', 'karras', 1], + inputs: [ + { name: 'model', type: 'MODEL', link: 10 }, + { name: 'positive', type: 'CONDITIONING', link: 20 }, + { name: 'negative', type: 'CONDITIONING', link: 21 }, + { name: 'latent_image', type: 'LATENT', link: 22 } + ], + outputs: [{ name: 'LATENT', type: 'LATENT', links: [30] }] + }, + { + id: 6, + type: 'Test VAE Decode', + inputs: [ + { name: 'samples', type: 'LATENT', link: 30 }, + { name: 'vae', type: 'VAE', link: 13 } + ], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [40] }] + }, + { + id: 7, + type: 'Test Save Image', + widgets_values: ['zimage_output'], + inputs: [{ name: 'images', type: 'IMAGE', link: 40 }] + } + ], + links: [ + [10, 1, 0, 5, 0, 'MODEL'], + [11, 1, 1, 2, 0, 'CLIP'], + [12, 1, 1, 3, 0, 'CLIP'], + [13, 1, 2, 6, 1, 'VAE'], + [20, 2, 0, 5, 1, 'CONDITIONING'], + [21, 3, 0, 5, 2, 'CONDITIONING'], + [22, 4, 0, 5, 3, 'LATENT'], + [30, 5, 0, 6, 0, 'LATENT'], + [40, 6, 0, 7, 0, 'IMAGE'] + ] + } + insertAndDeliver(graph, 'wf-txt2img', workflow) + + const loader = findByType(graph, 'Test Checkpoint Loader') + const positive = findByType(graph, 'Test CLIP Text Encode') + const sampler = findByType(graph, 'Test KSampler') + const decode = findByType(graph, 'Test VAE Decode') + const save = findByType(graph, 'Test Save Image') + + expect(renderedWidgetValues(graph.rootGraph.id, String(loader.id))).toEqual( + { ckpt_name: 'sd_xl_refiner.safetensors' } + ) + expect( + renderedWidgetValues(graph.rootGraph.id, String(sampler.id)) + ).toEqual({ + seed: 12345, + steps: 30, + cfg: 6.5, + sampler_name: 'dpmpp_2m', + scheduler: 'karras', + denoise: 1 + }) + expect(readInputOrigins(sampler).map((o) => o.origin)).toEqual([ + loader.id, + positive.id, + expect.anything(), + expect.anything() + ]) + expect(readInputOrigins(decode)[0]?.origin).toBe(sampler.id) + expect(readInputOrigins(save)[0]?.origin).toBe(decode.id) + }) + + it('an image-to-image pipeline (VAE encode + partial denoise) renders fully', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test Load Image', + widgets_values: ['example.png'], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [10] }] + }, + { + id: 2, + type: 'Test Checkpoint Loader', + outputs: [ + { name: 'MODEL', type: 'MODEL', links: [] }, + { name: 'CLIP', type: 'CLIP', links: [] }, + { name: 'VAE', type: 'VAE', links: [11] } + ] + }, + { + id: 3, + type: 'Test VAE Encode', + inputs: [ + { name: 'pixels', type: 'IMAGE', link: 10 }, + { name: 'vae', type: 'VAE', link: 11 } + ], + outputs: [{ name: 'LATENT', type: 'LATENT', links: [20] }] + }, + { + id: 4, + type: 'Test KSampler', + widgets_values: [0, 20, 8, 'euler', 'normal', 0.6], + inputs: [{ name: 'latent_image', type: 'LATENT', link: 20 }], + outputs: [{ name: 'LATENT', type: 'LATENT', links: [30] }] + }, + { + id: 5, + type: 'Test VAE Decode', + inputs: [{ name: 'samples', type: 'LATENT', link: 30 }], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [40] }] + }, + { + id: 6, + type: 'Test Save Image', + inputs: [{ name: 'images', type: 'IMAGE', link: 40 }] + } + ], + links: [ + [10, 1, 0, 3, 0, 'IMAGE'], + [11, 2, 2, 3, 1, 'VAE'], + [20, 3, 0, 4, 0, 'LATENT'], + [30, 4, 0, 5, 0, 'LATENT'], + [40, 5, 0, 6, 0, 'IMAGE'] + ] + } + insertAndDeliver(graph, 'wf-img2img', workflow) + + const loadImage = findByType(graph, 'Test Load Image') + const encode = findByType(graph, 'Test VAE Encode') + const sampler = findByType(graph, 'Test KSampler') + + expect(readInputOrigins(encode)[0]?.origin).toBe(loadImage.id) + expect( + renderedWidgetValues(graph.rootGraph.id, String(sampler.id)).denoise + ).toBe(0.6) + }) + + it('a ControlNet-guided pipeline fans image and conditioning into KSampler', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test Load Image', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [10] }] + }, + { + id: 2, + type: 'Test ControlNet Loader', + widgets_values: ['depth.safetensors'], + outputs: [{ name: 'CONTROL_NET', type: 'CONTROL_NET', links: [20] }] + }, + { + id: 3, + type: 'Test CLIP Text Encode', + widgets_values: ['a castle in the mist'], + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [30] }] + }, + { + id: 4, + type: 'Test ControlNet Apply', + widgets_values: [0.8], + inputs: [ + { name: 'conditioning', type: 'CONDITIONING', link: 30 }, + { name: 'control_net', type: 'CONTROL_NET', link: 20 }, + { name: 'image', type: 'IMAGE', link: 10 } + ], + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [40] }] + }, + { + id: 5, + type: 'Test KSampler', + inputs: [{ name: 'positive', type: 'CONDITIONING', link: 40 }], + outputs: [] + } + ], + links: [ + [10, 1, 0, 4, 2, 'IMAGE'], + [20, 2, 0, 4, 1, 'CONTROL_NET'], + [30, 3, 0, 4, 0, 'CONDITIONING'], + [40, 4, 0, 5, 0, 'CONDITIONING'] + ] + } + insertAndDeliver(graph, 'wf-controlnet', workflow) + + const loadImage = findByType(graph, 'Test Load Image') + const cnLoader = findByType(graph, 'Test ControlNet Loader') + const encode = findByType(graph, 'Test CLIP Text Encode') + const apply = findByType(graph, 'Test ControlNet Apply') + + expect(readInputOrigins(apply)).toEqual([ + { name: 'conditioning', origin: encode.id }, + { name: 'control_net', origin: cnLoader.id }, + { name: 'image', origin: loadImage.id } + ]) + expect( + renderedWidgetValues(graph.rootGraph.id, String(apply.id)).strength + ).toBe(0.8) + }) + + it('a LoRA-augmented pipeline renders the loader chain and its widgets', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test Checkpoint Loader', + outputs: [ + { name: 'MODEL', type: 'MODEL', links: [10] }, + { name: 'CLIP', type: 'CLIP', links: [11] }, + { name: 'VAE', type: 'VAE', links: [] } + ] + }, + { + id: 2, + type: 'Test LoRA Loader', + widgets_values: ['style_anime.safetensors', 0.75, 0.6], + inputs: [ + { name: 'model', type: 'MODEL', link: 10 }, + { name: 'clip', type: 'CLIP', link: 11 } + ], + outputs: [ + { name: 'MODEL', type: 'MODEL', links: [20] }, + { name: 'CLIP', type: 'CLIP', links: [] } + ] + }, + { + id: 3, + type: 'Test KSampler', + inputs: [{ name: 'model', type: 'MODEL', link: 20 }], + outputs: [] + } + ], + links: [ + [10, 1, 0, 2, 0, 'MODEL'], + [11, 1, 1, 2, 1, 'CLIP'], + [20, 2, 0, 3, 0, 'MODEL'] + ] + } + insertAndDeliver(graph, 'wf-lora', workflow) + + const checkpoint = findByType(graph, 'Test Checkpoint Loader') + const lora = findByType(graph, 'Test LoRA Loader') + const sampler = findByType(graph, 'Test KSampler') + + expect(readInputOrigins(lora).map((o) => o.origin)).toEqual([ + checkpoint.id, + checkpoint.id + ]) + expect(readInputOrigins(sampler)[0]?.origin).toBe(lora.id) + expect(renderedWidgetValues(graph.rootGraph.id, String(lora.id))).toEqual({ + lora_name: 'style_anime.safetensors', + strength_model: 0.75, + strength_clip: 0.6 + }) + }) + + it('a video pipeline renders its save-video widgets (format, fps)', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test KSampler', + outputs: [{ name: 'LATENT', type: 'LATENT', links: [10] }] + }, + { + id: 2, + type: 'Test VAE Decode', + inputs: [{ name: 'samples', type: 'LATENT', link: 10 }], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [20] }] + }, + { + id: 3, + type: 'Test Save Video', + widgets_values: ['webm', 30], + inputs: [{ name: 'images', type: 'IMAGE', link: 20 }] + } + ], + links: [ + [10, 1, 0, 2, 0, 'LATENT'], + [20, 2, 0, 3, 0, 'IMAGE'] + ] + } + insertAndDeliver(graph, 'wf-video', workflow) + + const saveVideo = findByType(graph, 'Test Save Video') + expect( + renderedWidgetValues(graph.rootGraph.id, String(saveVideo.id)) + ).toEqual({ format: 'webm', fps: 30 }) + }) + + it('a ControlNet + LoRA compound pipeline fans both augmentations into one sampler', () => { + const graph = new LGraph() + const workflow = { + nodes: [ + { + id: 1, + type: 'Test Checkpoint Loader', + outputs: [ + { name: 'MODEL', type: 'MODEL', links: [10] }, + { name: 'CLIP', type: 'CLIP', links: [] }, + { name: 'VAE', type: 'VAE', links: [] } + ] + }, + { + id: 2, + type: 'Test LoRA Loader', + widgets_values: ['add_detail.safetensors', 1, 1], + inputs: [{ name: 'model', type: 'MODEL', link: 10 }], + outputs: [{ name: 'MODEL', type: 'MODEL', links: [20] }] + }, + { + id: 3, + type: 'Test Load Image', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [30] }] + }, + { + id: 4, + type: 'Test ControlNet Loader', + outputs: [{ name: 'CONTROL_NET', type: 'CONTROL_NET', links: [40] }] + }, + { + id: 5, + type: 'Test CLIP Text Encode', + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [50] }] + }, + { + id: 6, + type: 'Test ControlNet Apply', + inputs: [ + { name: 'conditioning', type: 'CONDITIONING', link: 50 }, + { name: 'control_net', type: 'CONTROL_NET', link: 40 }, + { name: 'image', type: 'IMAGE', link: 30 } + ], + outputs: [{ name: 'CONDITIONING', type: 'CONDITIONING', links: [60] }] + }, + { + id: 7, + type: 'Test KSampler', + inputs: [ + { name: 'model', type: 'MODEL', link: 20 }, + { name: 'positive', type: 'CONDITIONING', link: 60 } + ], + outputs: [] + } + ], + links: [ + [10, 1, 0, 2, 0, 'MODEL'], + [20, 2, 0, 7, 0, 'MODEL'], + [30, 3, 0, 6, 2, 'IMAGE'], + [40, 4, 0, 6, 1, 'CONTROL_NET'], + [50, 5, 0, 6, 0, 'CONDITIONING'], + [60, 6, 0, 7, 1, 'CONDITIONING'] + ] + } + insertAndDeliver(graph, 'wf-compound', workflow) + + const lora = findByType(graph, 'Test LoRA Loader') + const apply = findByType(graph, 'Test ControlNet Apply') + const sampler = findByType(graph, 'Test KSampler') + + const samplerOrigins = readInputOrigins(sampler) + expect(samplerOrigins[0]).toEqual({ name: 'model', origin: lora.id }) + expect(samplerOrigins[1]).toEqual({ name: 'positive', origin: apply.id }) + }) + + it('the same template inserted twice produces two independent, fully-wired pipelines', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const deliver = bindProjection('wf-duplicate-template', graph) + deliver(Y.encodeStateAsUpdate(host), []) + + const template = () => ({ + nodes: [ + { + id: 1, + type: 'Test Load Image', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [10] }] + }, + { + id: 2, + type: 'Test Save Image', + widgets_values: ['dup'], + inputs: [{ name: 'images', type: 'IMAGE', link: 10 }] + } + ], + links: [[10, 1, 0, 2, 0, 'IMAGE']] + }) + + const opA = insertOp(template(), 'insert-dup-template-a') + let vector = Y.encodeStateVector(host) + expect(applyOps(host, [opA], CATALOG).outcomes[0]?.outcome).toBe('applied') + expect(deliver(Y.encodeStateAsUpdate(host, vector), [opA.op_id])).toBe(true) + + const opB = insertOp(template(), 'insert-dup-template-b') + vector = Y.encodeStateVector(host) + expect(applyOps(host, [opB], CATALOG).outcomes[0]?.outcome).toBe('applied') + expect(deliver(Y.encodeStateAsUpdate(host, vector), [opB.op_id])).toBe(true) + + const loadImages = findAllByType(graph, 'Test Load Image') + const saveImages = findAllByType(graph, 'Test Save Image') + expect(loadImages).toHaveLength(2) + expect(saveImages).toHaveLength(2) + + const origins = saveImages.map((save) => readInputOrigins(save)[0]?.origin) + expect(new Set(origins).size).toBe(2) + for (const origin of origins) { + expect(loadImages.map((n) => n.id)).toContainEqual(origin) + } + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentInsertWorkflow.widgetTypes.test.ts b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.widgetTypes.test.ts new file mode 100644 index 00000000000..397aaa8db00 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/agentInsertWorkflow.widgetTypes.test.ts @@ -0,0 +1,366 @@ +/** + * Extended coverage for `insert_workflow`-materialized widget rendering, + * complementing the PM-1580 regression pins in + * `agentInsertWorkflowWidgets.regression.test.ts` (which covers exactly the + * bug that was fixed: a `TestSaveImage`-shaped node with one `text` widget, + * plus the two-way same-graph id-collision case). + * + * This file exercises the same pipeline (comfy-multi-player's + * `insert_workflow` applier -> `AgentCrdtProjection` -> `graphMutations` -> + * `agentNodeMaterializer`) across the widget TYPES a real node can carry + * (combo, number, text, toggle, multiline) and a few id-collision shapes the + * regression file does not: cross-graph collisions and a three-way + * same-graph collision. + */ +import { applyOps, mint } from '@comfyorg/comfy-multi-player' +import type { + InsertWorkflowOp, + WidgetCatalog +} from '@comfyorg/comfy-multi-player' +import { describe, expect, it, onTestFinished } from 'vitest' +import * as Y from 'yjs' + +import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { + stripGraphPrefix, + useWidgetValueStore +} from '@/stores/widgetValueStore' +import { graphScopeOf } from '@/types/graphScopeId' + +import { AgentCrdtProjection } from './agentCrdtProjection' +import { inertPlacementPort } from './__fixtures__/inertPlacementPort' +import { FollowerDoc } from './followerDoc' +import { createGraphMutations } from './graphMutations' + +class TestComboNode extends LGraphNode { + static override title = 'Test Combo' + constructor() { + super('Test Combo') + this.addWidget('combo', 'mode', 'fixed', () => {}, { + values: ['fixed', 'increment', 'decrement', 'randomize'] + }) + this.serialize_widgets = true + } +} + +class TestNumberNode extends LGraphNode { + static override title = 'Test Number' + constructor() { + super('Test Number') + this.addWidget('number', 'steps', 20, () => {}, { min: 1, max: 100 }) + this.serialize_widgets = true + } +} + +class TestTextNode extends LGraphNode { + static override title = 'Test Text' + constructor() { + super('Test Text') + this.addWidget('text', 'filename_prefix', 'ComfyUI', () => {}) + this.serialize_widgets = true + } +} + +class TestToggleNode extends LGraphNode { + static override title = 'Test Toggle' + constructor() { + super('Test Toggle') + this.addWidget('toggle', 'enabled', false, () => {}) + this.serialize_widgets = true + } +} + +class TestMultilineNode extends LGraphNode { + static override title = 'Test Multiline' + constructor() { + super('Test Multiline') + this.addCustomWidget({ + name: 'prompt', + type: 'customtext', + value: '', + options: { multiline: true }, + y: 0 + }) + this.serialize_widgets = true + } +} + +/** Mirrors a real sampler node's widget cluster: mixed types in one node. */ +class TestMultiWidgetNode extends LGraphNode { + static override title = 'Test Multi Widget' + constructor() { + super('Test Multi Widget') + this.addWidget('number', 'seed', 0, () => {}) + this.addWidget('number', 'steps', 20, () => {}) + this.addWidget('number', 'cfg', 8, () => {}) + this.addWidget('combo', 'sampler_name', 'euler', () => {}, { + values: ['euler', 'dpmpp_2m', 'ddim'] + }) + this.addWidget('toggle', 'add_noise', true, () => {}) + this.serialize_widgets = true + } +} + +const CATALOG: WidgetCatalog = { + types: { + TestCombo: { widget_order: ['mode'] }, + TestNumber: { widget_order: ['steps'] }, + TestText: { widget_order: ['filename_prefix'] }, + TestToggle: { widget_order: ['enabled'] }, + TestMultiline: { widget_order: ['prompt'] }, + TestMultiWidget: { + widget_order: ['seed', 'steps', 'cfg', 'sampler_name', 'add_noise'] + } + } +} + +function registerAll(): void { + LiteGraph.registerNodeType('TestCombo', TestComboNode) + LiteGraph.registerNodeType('TestNumber', TestNumberNode) + LiteGraph.registerNodeType('TestText', TestTextNode) + LiteGraph.registerNodeType('TestToggle', TestToggleNode) + LiteGraph.registerNodeType('TestMultiline', TestMultilineNode) + LiteGraph.registerNodeType('TestMultiWidget', TestMultiWidgetNode) +} + +let opSeq = 0 +function insertOp( + workflow: InsertWorkflowOp['workflow'], + opId?: string +): InsertWorkflowOp { + opSeq += 1 + return { + op_id: (opId ?? `insert-widget-type-op-${opSeq}`).padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [1, 'agent:test'], + op: 'insert_workflow', + workflow + } +} + +function bindProjection(workflowId: string, graph: LGraph) { + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + createGraphMutations({ + getScope: () => graphScopeOf(graph), + layout: { createNode: () => {}, deleteNodes: () => {} }, + placement: inertPlacementPort + }), + () => graph, + () => follower.doc + ) + projection.bind(workflowId, follower) + onTestFinished(() => { + projection.destroy() + follower.destroy() + }) + + let seq = 0 + const deliver = (update: Uint8Array, opIds: string[]): boolean => { + follower.applyRemoteUpdate(update) + const committed = projection.applyFrame({ + workflowId, + seq: ++seq, + update, + actor: 'agent:test', + opIds + }) + projection.reconcileLiveGraph(workflowId) + return committed + } + return deliver +} + +/** What `LGraphNode.vue`'s `widgetIds` computed does, read straight from the store. */ +function renderedWidgets(graphId: string, nodeId: string) { + const widgetValueStore = useWidgetValueStore() + const bareNodeId = stripGraphPrefix(nodeId) + if (!bareNodeId) return [] + return widgetValueStore + .getNodeWidgetIds(graphId as never, bareNodeId) + .flatMap((id) => { + const widget = widgetValueStore.getWidget(id) + return widget ? [{ name: widget.name, value: widget.value }] : [] + }) +} + +function insertSingleNode( + graph: LGraph, + workflowId: string, + nodeId: number, + type: string, + widgetsValues: unknown[] +): LGraphNode { + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const deliver = bindProjection(workflowId, graph) + deliver(Y.encodeStateAsUpdate(host), []) + + const op = insertOp({ + nodes: [{ id: nodeId, type, widgets_values: widgetsValues }], + links: [] + }) + const vector = Y.encodeStateVector(host) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + expect(deliver(Y.encodeStateAsUpdate(host, vector), [op.op_id])).toBe(true) + + const node = graph._nodes.find((candidate) => candidate.type === type) + expect(node).toBeDefined() + if (!node) throw new Error(`no live node of type ${type} was materialized`) + return node +} + +describe('insert_workflow materializes every widget type with the correct value', () => { + registerAll() + + it('renders a combo widget with its selected value', () => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-combo', 1, 'TestCombo', [ + 'randomize' + ]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'mode', value: 'randomize' } + ]) + }) + + it('renders a combo widget carrying a value outside its declared options', () => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-combo-custom', 1, 'TestCombo', [ + 'not-a-listed-option' + ]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'mode', value: 'not-a-listed-option' } + ]) + }) + + it.for([ + { name: 'positive', value: 42 }, + { name: 'zero', value: 0 }, + { name: 'negative', value: -7 } + ])('renders a number widget with a $name value', ({ value }) => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-number', 1, 'TestNumber', [value]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'steps', value } + ]) + }) + + it('renders a text widget with its string value', () => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-text', 1, 'TestText', [ + 'my_custom_prefix' + ]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'filename_prefix', value: 'my_custom_prefix' } + ]) + }) + + it.for([ + { name: 'true', value: true }, + { name: 'false', value: false } + ])('renders a toggle widget set to $name', ({ value }) => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-toggle', 1, 'TestToggle', [value]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'enabled', value } + ]) + }) + + it('renders a multiline text widget with embedded newlines intact', () => { + const graph = new LGraph() + const prompt = 'a glorious spooky duck\nin a cathedral\nat dusk' + const node = insertSingleNode(graph, 'wf-multiline', 1, 'TestMultiline', [ + prompt + ]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'prompt', value: prompt } + ]) + }) + + it('renders every widget on a multi-widget node, in declared order, with correct values', () => { + const graph = new LGraph() + const node = insertSingleNode(graph, 'wf-multi', 1, 'TestMultiWidget', [ + 12345, + 30, + 6.5, + 'dpmpp_2m', + false + ]) + expect(renderedWidgets(graph.rootGraph.id, String(node.id))).toEqual([ + { name: 'seed', value: 12345 }, + { name: 'steps', value: 30 }, + { name: 'cfg', value: 6.5 }, + { name: 'sampler_name', value: 'dpmpp_2m' }, + { name: 'add_noise', value: false } + ]) + }) + + it('renders only the widgets a partial widgets_values array actually supplies', () => { + const graph = new LGraph() + // Only the first two of five widget values are supplied. + const node = insertSingleNode( + graph, + 'wf-partial', + 1, + 'TestMultiWidget', + [999, 15] + ) + const rendered = renderedWidgets(graph.rootGraph.id, String(node.id)) + expect(rendered).toEqual([ + { name: 'seed', value: 999 }, + { name: 'steps', value: 15 } + ]) + }) + + it('two inserted nodes sharing an original id, in DIFFERENT graphs, keep independent widget values', () => { + const graphA = new LGraph() + const graphB = new LGraph() + const nodeA = insertSingleNode(graphA, 'wf-cross-graph-a', 9, 'TestText', [ + 'value-in-graph-a' + ]) + const nodeB = insertSingleNode(graphB, 'wf-cross-graph-b', 9, 'TestText', [ + 'value-in-graph-b' + ]) + + expect(renderedWidgets(graphA.rootGraph.id, String(nodeA.id))).toEqual([ + { name: 'filename_prefix', value: 'value-in-graph-a' } + ]) + expect(renderedWidgets(graphB.rootGraph.id, String(nodeB.id))).toEqual([ + { name: 'filename_prefix', value: 'value-in-graph-b' } + ]) + }) + + it('three inserted nodes sharing an original id, in the SAME graph, each keep their own widget value', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const deliver = bindProjection('wf-triple-collision', graph) + deliver(Y.encodeStateAsUpdate(host), []) + + const values = ['first', 'second', 'third'] + for (const value of values) { + const op = insertOp({ + nodes: [{ id: 3, type: 'TestText', widgets_values: [value] }], + links: [] + }) + const vector = Y.encodeStateVector(host) + expect(applyOps(host, [op], CATALOG).outcomes[0]?.outcome).toBe('applied') + expect(deliver(Y.encodeStateAsUpdate(host, vector), [op.op_id])).toBe( + true + ) + } + + const nodes = graph._nodes.filter((n) => n.type === 'TestText') + expect(nodes).toHaveLength(3) + const ids = new Set(nodes.map((n) => String(n.id))) + expect(ids.size).toBe(3) + + const renderedValues = nodes.map( + (n) => renderedWidgets(graph.rootGraph.id, String(n.id))[0]?.value + ) + expect(new Set(renderedValues)).toEqual(new Set(values)) + }) +})