diff --git a/package.json b/package.json index 62eb6d0d810..b1e554391d0 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "dependencies": { "@alloc/quick-lru": "catalog:", "@atlaskit/pragmatic-drag-and-drop": "^1.3.1", - "@comfyorg/comfy-multi-player": "0.3.2", + "@comfyorg/comfy-multi-player": "catalog:", "@comfyorg/comfyui-desktop-bridge-types": "catalog:", "@comfyorg/comfyui-electron-types": "catalog:", "@comfyorg/design-system": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 080dbbc27e3..e00d3767cb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,6 +21,9 @@ catalogs: '@astrojs/vue': specifier: ^7.0.2 version: 7.0.2 + '@comfyorg/comfy-multi-player': + specifier: https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634 + version: 0.3.3 '@comfyorg/comfyui-desktop-bridge-types': specifier: 0.2.0 version: 0.2.0 @@ -476,8 +479,8 @@ importers: specifier: ^1.3.1 version: 1.8.1 '@comfyorg/comfy-multi-player': - specifier: 0.3.2 - version: 0.3.2 + specifier: 'catalog:' + version: https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634 '@comfyorg/comfyui-desktop-bridge-types': specifier: 'catalog:' version: 0.2.0 @@ -1605,8 +1608,9 @@ packages: resolution: {integrity: sha512-y7/yvZ2TPAnR9+jnc00klvNNLkJiXFFrQA/hlLCcxA9a2A4zQIOimyFQ9XfwYKiGD1fb5GY8vbKIIgO8d5Tb2A==} engines: {node: '>= 20.12.0'} - '@comfyorg/comfy-multi-player@0.3.2': - resolution: {integrity: sha512-xeNSPz0A9HvGemiDoA0W0QOWomeZGduG6dmAMRrScjq793AMVTpzVsr1gBxV0n1DLMm38ck7jKFIx2irMo9a5A==} + '@comfyorg/comfy-multi-player@https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634': + resolution: {gitHosted: true, integrity: sha512-YhGP8rTlRGZjSgZVAfvdJOWaKvR7dF6Oo6w+IicyK+4f7XqQk30yW5hUFoHaGLFJH6RGwDvSey0/BS2Iq2r/PA==, tarball: https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634} + version: 0.3.3 '@comfyorg/comfyui-desktop-bridge-types@0.2.0': resolution: {integrity: sha512-V5Wj5KXDzTDm5GJAa0xBISjmndNIY5P79pRHQbLkkqJxgp/GPb/QGvcI+xG4ogol0ODJ4j/UBnUmkzpSafw8Aw==} @@ -10109,7 +10113,7 @@ snapshots: fast-wrap-ansi: 0.2.2 sisteransi: 1.0.5 - '@comfyorg/comfy-multi-player@0.3.2': + '@comfyorg/comfy-multi-player@https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634': dependencies: yjs: 13.6.31 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2074ab02448..fea44f46f71 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -159,11 +159,12 @@ catalog: zod: ^3.23.8 zod-to-json-schema: ^3.24.1 zod-validation-error: ^3.3.0 + '@comfyorg/comfy-multi-player': https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634 cleanupUnusedCatalogs: true allowBuilds: - '@comfyorg/comfy-multi-player@https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/58e2333ce3bb6375fd2efbc1e9d7df17327f37e0': true + '@comfyorg/comfy-multi-player@https://codeload.github.com/Comfy-Org/comfy-multi-player/tar.gz/b475423a0f8b019bbac261ce218dfc016e9aa634': true '@firebase/util': false '@sentry/cli': true '@tailwindcss/oxide': true diff --git a/src/workbench/extensions/agent/crdt/agentInsertWorkflowLinks.regression.test.ts b/src/workbench/extensions/agent/crdt/agentInsertWorkflowLinks.regression.test.ts new file mode 100644 index 00000000000..edf64c7d0a6 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/agentInsertWorkflowLinks.regression.test.ts @@ -0,0 +1,481 @@ +import { applyOps, linksMap, mint, project } from '@comfyorg/comfy-multi-player' +import type { + DisconnectOp, + InsertWorkflowOp, + WidgetCatalog +} from '@comfyorg/comfy-multi-player' +import { + assert, + beforeEach, + describe, + expect, + it, + onTestFinished +} from 'vitest' +import * as Y from 'yjs' + +import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' +import { graphScopeOf } from '@/types/graphScopeId' + +import { AgentCrdtProjection } from './agentCrdtProjection' +import { inertPlacementPort } from './__fixtures__/inertPlacementPort' +import { FollowerDoc } from './followerDoc' +import { createGraphMutations } from './graphMutations' + +const WORKFLOW_ID = 'wf-plain-insert' + +class TestLoadImage extends LGraphNode { + static override title = 'Test Load Image' + constructor() { + super('Test Load Image') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +class TestReferenceNode extends LGraphNode { + static override title = 'Test Reference Node' + constructor() { + super('Test Reference Node') + this.addInput('image', 'IMAGE') + this.addOutput('IMAGE', 'IMAGE') + this.serialize_widgets = true + } +} + +class TestSaveVideo extends LGraphNode { + static override title = 'Test Save Video' + constructor() { + super('Test Save Video') + this.addInput('images', 'IMAGE') + this.serialize_widgets = true + } +} + +const CATALOG: WidgetCatalog = { + types: { + TestLoadImage: { widget_order: [] }, + TestReferenceNode: { widget_order: [] }, + TestSaveVideo: { widget_order: [] } + } +} + +function insertOp( + workflow: InsertWorkflowOp['workflow'], + opId = 'insert-plain-chain-op' +): InsertWorkflowOp { + return { + op_id: opId.padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [1, 'agent:test'], + op: 'insert_workflow', + workflow + } +} + +function bindProjection(workflowId: string, graph: LGraph) { + const follower = new FollowerDoc() + const projection = new AgentCrdtProjection( + createGraphMutations({ + getScope: () => graphScopeOf(graph), + layout: { createNode: () => {}, deleteNodes: () => {} }, + placement: inertPlacementPort + }), + () => graph, + () => follower.doc + ) + projection.bind(workflowId, follower) + onTestFinished(() => { + projection.destroy() + follower.destroy() + }) + + let seq = 0 + const deliver = (update: Uint8Array, opIds: string[]): boolean => { + follower.applyRemoteUpdate(update) + const committed = projection.applyFrame({ + workflowId, + seq: ++seq, + update, + actor: 'agent:test', + opIds + }) + projection.reconcileLiveGraph(workflowId) + return committed + } + return deliver +} + +function readInputOrigins(node: LGraphNode) { + return node.inputs.map((input, index) => ({ + name: input.name, + origin: node.getInputLink(index)?.origin_id ?? null + })) +} + +function docLinkIds(doc: Y.Doc): unknown[] { + return [...linksMap(doc).values()].map((link) => + link instanceof Y.Array ? link.get(0) : Array.isArray(link) ? link[0] : null + ) +} + +function liveLinkIds(graph: LGraph) { + return graph.serialize().links.map(([id]) => id) +} + +function expectSafeNumericLinkIds(doc: Y.Doc, count: number): void { + const ids = docLinkIds(doc) + expect(ids).toHaveLength(count) + expect( + ids.every( + (id) => typeof id === 'number' && id >= 0 && Number.isSafeInteger(id) + ) + ).toBe(true) +} + +function findByType(graph: LGraph, type: string): LGraphNode { + const node = graph._nodes.find((candidate) => candidate.type === type) + if (!node) throw new Error(`no live node of type ${type} was materialized`) + return node +} + +beforeEach(() => { + LiteGraph.registerNodeType('TestLoadImage', TestLoadImage) + LiteGraph.registerNodeType('TestReferenceNode', TestReferenceNode) + LiteGraph.registerNodeType('TestSaveVideo', TestSaveVideo) +}) + +describe('a plain insert_workflow batch materializes its links', () => { + it('wires links on the first full reconcile', () => { + const graph = new LGraph() + + const workflow = { + nodes: [ + { + id: 101, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [201] }] + }, + { + id: 102, + type: 'TestReferenceNode', + inputs: [{ name: 'image', type: 'IMAGE', link: 201 }], + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [202] }] + }, + { + id: 103, + type: 'TestSaveVideo', + inputs: [{ name: 'images', type: 'IMAGE', link: 202 }] + } + ], + links: [ + [201, 101, 0, 102, 0, 'IMAGE'], + [202, 102, 0, 103, 0, 'IMAGE'] + ] + } + + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + const op = insertOp(workflow) + expect(applyOps(host, [op], CATALOG).outcomes).toEqual([ + { op_id: op.op_id, outcome: 'applied' } + ]) + expectSafeNumericLinkIds(host, 2) + const expectedLinkIds = docLinkIds(host) + + const deliver = bindProjection(WORKFLOW_ID, graph) + expect(deliver(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + + const loadImage = findByType(graph, 'TestLoadImage') + const reference = findByType(graph, 'TestReferenceNode') + const saveVideo = findByType(graph, 'TestSaveVideo') + + expect(readInputOrigins(reference)).toEqual([ + { name: 'image', origin: loadImage.id } + ]) + expect(readInputOrigins(saveVideo)).toEqual([ + { name: 'images', origin: reference.id } + ]) + expect(liveLinkIds(graph)).toEqual(expectedLinkIds) + + const reloadedGraph = new LGraph() + const reload = bindProjection('wf-reloaded', reloadedGraph) + expect(reload(Y.encodeStateAsUpdate(host), [op.op_id])).toBe(true) + expect(liveLinkIds(reloadedGraph)).toEqual(expectedLinkIds) + }) + + it('wires a link inserted after the graph is already open, on the incremental path', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const seedOp = insertOp( + { nodes: [{ id: 999, type: 'TestSaveVideo' }] }, + 'insert-incremental-seed-op' + ) + expect(applyOps(host, [seedOp], CATALOG).outcomes).toEqual([ + { op_id: seedOp.op_id, outcome: 'applied' } + ]) + + const deliver = bindProjection('wf-incremental', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [seedOp.op_id])).toBe(true) + expect(graph._nodes).toHaveLength(1) + + const before = Y.encodeStateVector(host) + const secondOp = insertOp( + { + nodes: [ + { + id: 111, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [301] }] + }, + { + id: 112, + type: 'TestReferenceNode', + inputs: [{ name: 'image', type: 'IMAGE', link: 301 }] + } + ], + links: [[301, 111, 0, 112, 0, 'IMAGE']] + }, + 'insert-incremental-second-op' + ) + expect(applyOps(host, [secondOp], CATALOG).outcomes).toEqual([ + { op_id: secondOp.op_id, outcome: 'applied' } + ]) + expectSafeNumericLinkIds(host, 1) + const expectedLinkIds = docLinkIds(host) + expect(deliver(Y.encodeStateAsUpdate(host, before), [secondOp.op_id])).toBe( + true + ) + + const loadImage = findByType(graph, 'TestLoadImage') + const reference = findByType(graph, 'TestReferenceNode') + expect(readInputOrigins(reference)).toEqual([ + { name: 'image', origin: loadImage.id } + ]) + expect(liveLinkIds(graph)).toEqual(expectedLinkIds) + }) + + it('keeps formerly colliding insertions distinct from a persisted id', () => { + const graph = new LGraph() + const persistedLinkId = 13_181_811_759 + const host = mint( + { + nodes: [ + { + id: 1, + type: 'TestLoadImage', + outputs: [ + { name: 'IMAGE', type: 'IMAGE', links: [persistedLinkId] } + ] + }, + { + id: 2, + type: 'TestSaveVideo', + inputs: [{ name: 'images', type: 'IMAGE', link: persistedLinkId }] + } + ], + links: [[persistedLinkId, 1, 0, 2, 0, 'IMAGE']] + }, + CATALOG + ) + onTestFinished(() => host.destroy()) + const workflow = { + nodes: [ + { + id: 101, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [201] }] + }, + { + id: 102, + type: 'TestSaveVideo', + inputs: [{ name: 'images', type: 'IMAGE', link: 201 }] + } + ], + links: [[201, 101, 0, 102, 0, 'IMAGE']] + } + const firstOp = insertOp(workflow, 'd7b0c03fe2abc909a5c84a6e02d80cdd') + const secondOp = insertOp(workflow, '589889ae708156b3e33fdf39e3cfb931') + expect(applyOps(host, [firstOp, secondOp], CATALOG).outcomes).toEqual([ + { op_id: firstOp.op_id, outcome: 'applied' }, + { op_id: secondOp.op_id, outcome: 'applied' } + ]) + + expectSafeNumericLinkIds(host, 3) + const expectedLinkIds = docLinkIds(host) + expect(new Set(expectedLinkIds).size).toBe(3) + expect(expectedLinkIds).toContain(persistedLinkId) + const deliver = bindProjection('wf-colliding-links', graph) + expect( + deliver(Y.encodeStateAsUpdate(host), [firstOp.op_id, secondOp.op_id]) + ).toBe(true) + expect(new Set(liveLinkIds(graph))).toEqual(new Set(expectedLinkIds)) + expect( + graph.serialize().links.find(([id]) => id === persistedLinkId) + ).toEqual([persistedLinkId, 1, 0, 2, 0, 'IMAGE']) + }) + + it.for([ + { name: 'canonical numeric string', mapKey: '1', linkId: '1' }, + { name: 'negative string', mapKey: '-1', linkId: '-1' }, + { name: 'fractional string', mapKey: '1.5', linkId: '1.5' }, + { + name: 'unsafe integer string', + mapKey: '9007199254740992', + linkId: '9007199254740992' + }, + { name: 'negative number', mapKey: '-1', linkId: -1 }, + { name: 'fractional number', mapKey: '1.5', linkId: 1.5 }, + { + name: 'unsafe integer number', + mapKey: '9007199254740992', + linkId: 9_007_199_254_740_992 + }, + { name: 'NaN', mapKey: 'NaN', linkId: Number.NaN }, + { name: 'infinity', mapKey: 'Infinity', linkId: Number.POSITIVE_INFINITY }, + { name: 'missing tuple id', mapKey: '1', linkId: null } + ])('rejects a $name link id', ({ mapKey, linkId }) => { + const graph = new LGraph() + const host = mint( + { + nodes: [ + { + id: 1, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [linkId] }] + }, + { + id: 2, + type: 'TestSaveVideo', + inputs: [{ name: 'images', type: 'IMAGE', link: linkId }] + } + ], + links: [] + }, + CATALOG + ) + onTestFinished(() => host.destroy()) + linksMap(host).set(mapKey, [linkId, 1, 0, 2, 0, 'IMAGE']) + + const deliver = bindProjection('wf-malformed-link', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [])).toBe(true) + expect(graph._nodes).toHaveLength(2) + expect(graph.serialize().links).toHaveLength(0) + }) + + it('retires a live link when its map key and tuple id diverge', () => { + const graph = new LGraph() + const host = mint( + { + nodes: [ + { + id: 1, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [9] }] + }, + { + id: 2, + type: 'TestSaveVideo', + inputs: [{ name: 'images', type: 'IMAGE', link: 9 }] + } + ], + links: [[9, 1, 0, 2, 0, 'IMAGE']] + }, + CATALOG + ) + onTestFinished(() => host.destroy()) + const deliver = bindProjection('wf-divergent-link-id', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [])).toBe(true) + expect(liveLinkIds(graph)).toEqual([9]) + + const before = Y.encodeStateVector(host) + linksMap(host).set('9', [10, 1, 0, 2, 0, 'IMAGE']) + expect(deliver(Y.encodeStateAsUpdate(host, before), [])).toBe(true) + expect(liveLinkIds(graph)).toEqual([]) + }) + + it('removes then reinserts a numeric-id link', () => { + const graph = new LGraph() + const host = mint({ nodes: [], links: [] }, CATALOG) + onTestFinished(() => host.destroy()) + + const workflow = { + nodes: [ + { + id: 121, + type: 'TestLoadImage', + outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [401] }] + }, + { + id: 122, + type: 'TestReferenceNode', + inputs: [{ name: 'image', type: 'IMAGE', link: 401 }] + } + ], + links: [[401, 121, 0, 122, 0, 'IMAGE']] + } + + const firstOp = insertOp(workflow, 'insert-reinsertion-first-op') + expect(applyOps(host, [firstOp], CATALOG).outcomes).toEqual([ + { op_id: firstOp.op_id, outcome: 'applied' } + ]) + const firstLinkId = docLinkIds(host)[0] + expectSafeNumericLinkIds(host, 1) + assert(typeof firstLinkId === 'number') + + const deliver = bindProjection('wf-reinsertion', graph) + expect(deliver(Y.encodeStateAsUpdate(host), [firstOp.op_id])).toBe(true) + + const loadImage = findByType(graph, 'TestLoadImage') + const reference = findByType(graph, 'TestReferenceNode') + expect(readInputOrigins(reference)).toEqual([ + { name: 'image', origin: loadImage.id } + ]) + + const projected = project(host, CATALOG) + const loadNode = projected.nodes.find((n) => n.type === 'TestLoadImage') + const refNode = projected.nodes.find((n) => n.type === 'TestReferenceNode') + assert.exists(loadNode) + assert.exists(refNode) + const beforeDisconnect = Y.encodeStateVector(host) + const disconnect = { + op_id: 'insert-reinsertion-disconnect-op'.padEnd(32, '0'), + actor: 'agent:test', + base_version: 1, + stamp: [2, 'agent:test'], + op: 'disconnect', + link_id: firstLinkId, + to_node: refNode.id, + to_slot: 0 + } satisfies DisconnectOp + expect(applyOps(host, [disconnect], CATALOG).outcomes).toEqual([ + { op_id: disconnect.op_id, outcome: 'applied' } + ]) + expect( + deliver(Y.encodeStateAsUpdate(host, beforeDisconnect), [disconnect.op_id]) + ).toBe(true) + expect(graph._nodes).toHaveLength(2) + expect(docLinkIds(host)).toEqual([]) + expect(liveLinkIds(graph)).toEqual([]) + + const beforeReinsertion = Y.encodeStateVector(host) + const reinsertionOp = insertOp(workflow, 'insert-reinsertion-second-op') + expect(applyOps(host, [reinsertionOp], CATALOG).outcomes).toEqual([ + { op_id: reinsertionOp.op_id, outcome: 'applied' } + ]) + const reinsertedLinkId = docLinkIds(host)[0] + expectSafeNumericLinkIds(host, 1) + expect(reinsertedLinkId).not.toBe(firstLinkId) + expect( + deliver(Y.encodeStateAsUpdate(host, beforeReinsertion), [ + reinsertionOp.op_id + ]) + ).toBe(true) + + expect(graph._nodes).toHaveLength(4) + expect(liveLinkIds(graph)).toEqual([reinsertedLinkId]) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts index 7b6c2a08eea..86bb1eba4e2 100644 --- a/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts +++ b/src/workbench/extensions/agent/crdt/ecsFollowerAdapter.ts @@ -13,6 +13,7 @@ import type { import { isIncompatibleLinkType } from './graphMutations' import { reportError } from '@/platform/telemetry/reportError' import type { RemoteMutationContext } from '@/types/graphMutationContext' +import { parseLinkId } from '@/types/linkId' import { toNodeId } from '@/types/nodeId' import { readSubgraphDefinitions } from './agentSubgraphDefinitions' @@ -204,6 +205,17 @@ function reportInvalidHostTarget( ) } +function resolveLinkId(raw: unknown): number | null { + return typeof raw === 'number' && raw >= 0 && Number.isSafeInteger(raw) + ? raw + : null +} + +function resolveLinkMapKey(id: string): number | null { + const linkId = parseLinkId(id) + return linkId !== undefined && linkId >= 0 ? linkId : null +} + function readSemanticLink( doc: Y.Doc, id: string, @@ -213,11 +225,13 @@ function readSemanticLink( const raw = linksMap(doc).get(id) const tuple = raw instanceof Y.Array ? raw.toArray() : raw if (!Array.isArray(tuple) || tuple.length < 5) return null - const linkId = Number(tuple[0] ?? id) + const linkId = resolveLinkId(tuple[0]) + const mapLinkId = resolveLinkMapKey(id) const originSlot = Number(tuple[2]) const targetSlot = Number(tuple[4]) if ( - !Number.isInteger(linkId) || + linkId === null || + mapLinkId !== linkId || tuple[1] == null || tuple[3] == null || !Number.isInteger(originSlot) || @@ -496,9 +510,11 @@ export class EcsFollowerAdapter { : null ]) ) - const removedLinkIds = [...changedLinks].flatMap(([id, link]) => - link && !isIncompatibleLinkType(link) ? [] : [Number(id)] - ) + const removedLinkIds = [...changedLinks].flatMap(([id, link]) => { + if (link && !isIncompatibleLinkType(link)) return [] + const linkId = resolveLinkMapKey(id) + return linkId === null ? [] : [linkId] + }) const committed = session.mutations.batch(frameContext(update), (batch) => { // A SubgraphNode host that is already live must never be rebuilt from // its doc entry: `reconcileNode` (and delete + `addNode`) replaces the