diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index f717482e149..7d6727ea5a8 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -102,6 +102,30 @@ class TestPrompt extends LGraphNode { } } +class TestVideoLoader extends LGraphNode { + constructor() { + super('Test Video Loader') + this.addWidget('combo', 'video', 'clip.mp4', () => {}, { + values: ['clip.mp4'] + }) + this.addWidget('button', 'choose video to upload', 'image', () => {}) + let preview: Record = { hidden: false, params: {} } + this.addDOMWidget( + 'videopreview', + 'preview', + document.createElement('div'), + { + serialize: false, + getValue: () => preview, + setValue: (value: Record) => { + preview = value + } + } + ) + this.serialize_widgets = true + } +} + class TestDynamicGroup extends LGraphNode { constructor() { super('Test Dynamic Group') @@ -142,7 +166,8 @@ const CATALOG: WidgetCatalog = { TestSource: { widget_order: ['steps'] }, TestSink: { widget_order: [] }, TestAutogrowSink: { widget_order: [] }, - TestPrompt: { widget_order: ['text'] } + TestPrompt: { widget_order: ['text'] }, + TestVideoLoader: { widget_order: ['video'] } } } @@ -198,6 +223,7 @@ beforeEach(() => { LiteGraph.registerNodeType('TestAutogrowSink', TestAutogrowSink) LiteGraph.registerNodeType('TestNote', TestNote) LiteGraph.registerNodeType('TestPrompt', TestPrompt) + LiteGraph.registerNodeType('TestVideoLoader', TestVideoLoader) LiteGraph.registerNodeType('TestDynamicGroup', TestDynamicGroup) LiteGraph.registerNodeType('TestEmptyComboGroup', TestEmptyComboGroup) }) @@ -583,6 +609,60 @@ describe('attachDocOpMinter', () => { doc.destroy() }) + it('leaves a display-only DOM widget out of a catalogued add_node, and the applier takes it', async () => { + const loader = new TestVideoLoader() + graph.add(loader) + await afterFlush() + + expect(minted).toEqual([ + expect.objectContaining({ + op: 'add_node', + class_type: 'TestVideoLoader', + node: expect.objectContaining({ + widgets_values: { video: 'clip.mp4' } + }) + }) + ]) + const doc = mint({ nodes: [], links: [] }, CATALOG) + const { outcomes } = applyOps( + doc, + mintWireOps(minted, { actor: 'human:user:tab', baseVersion: 1 }), + CATALOG + ) + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'applied' })]) + doc.destroy() + }) + + it('keeps a DOM widget out of the prompt as a value when it backs a declared input', async () => { + const loader = new TestVideoLoader() + loader.addInput('videopreview', 'VIDEO_PREVIEW', { + widget: { name: 'videopreview' } + }) + graph.add(loader) + await afterFlush() + + expect(minted[0]).toMatchObject({ + op: 'add_node', + node: { + widgets_values: { + video: 'clip.mp4', + videopreview: { hidden: false, params: {} } + } + } + }) + const catalog: WidgetCatalog = { + types: { TestVideoLoader: { widget_order: ['video', 'videopreview'] } } + } + const doc = mint({ nodes: [], links: [] }, catalog) + const { outcomes } = applyOps( + doc, + mintWireOps(minted, { actor: 'human:user:tab', baseVersion: 1 }), + catalog + ) + expect(outcomes).toEqual([expect.objectContaining({ outcome: 'applied' })]) + doc.destroy() + }) + describe("link ops address the document's own input order", () => { /** A sink whose live input order (`mask`, `image`) is the reverse of the document's. */ function seedReorderedSink() { diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index 0da2954731d..e3654cc7ef2 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -150,7 +150,8 @@ function valueWidgetsOnly( const stored = rootGraphId ? widgetValueStore.getWidget(widgetId(rootGraphId, node.id, name)) : undefined - if (isValueWidget(widget, stored)) filtered[name] = value + if (isValueWidget(widget, stored) && !isDisplayOnlyDomWidget(node, widget)) + filtered[name] = value } return filtered } @@ -169,6 +170,18 @@ function isValueWidget( return type !== 'button' && serialize !== false } +function isDisplayOnlyDomWidget( + node: LGraphNode, + widget: IBaseWidget | undefined +): boolean { + if (!widget) return false + const rendersOwnElement = + ('element' in widget && Boolean(widget.element)) || + ('component' in widget && Boolean(widget.component)) + if (!rendersOwnElement || widget.options.serialize !== false) return false + return !node.inputs.some((input) => input.widget?.name === widget.name) +} + function nodeKey(graphId: string, nodeId: NodeId): string { return `${graphId}:${String(nodeId)}` } @@ -200,7 +213,10 @@ function isValueWidgetWrite( const widget = owner?.widgets?.find( (candidate) => candidate.name === event.name ) - return isValueWidget(widget, stored) + return ( + isValueWidget(widget, stored) && + !(owner && isDisplayOnlyDomWidget(owner, widget)) + ) } function promotedHostWrite( diff --git a/src/workbench/extensions/agent/crdt/restoreOpMinter.test.ts b/src/workbench/extensions/agent/crdt/restoreOpMinter.test.ts index 2f0cfd1f9da..bfc4cde55fd 100644 --- a/src/workbench/extensions/agent/crdt/restoreOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/restoreOpMinter.test.ts @@ -1,8 +1,11 @@ +import { applyOps, mint } from '@comfyorg/comfy-multi-player' +import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { LGraph, LGraphNode, LiteGraph } from '@/lib/litegraph/src/litegraph' import type { GraphOperation } from './graphOperations' +import { mintWireOps } from './opEnvelope' import { attachRestoreOpMinter, notifyRestoreMintersAfterGraphConfigure, @@ -20,6 +23,28 @@ class TestSource extends LGraphNode { } } +class TestPreviewSource extends LGraphNode { + constructor() { + super('Test Preview Source') + this.addWidget('text', 'image', 'a.png', () => {}) + let preview: Record = { hidden: false } + this.addDOMWidget( + 'videopreview', + 'preview', + document.createElement('div'), + { + serialize: false, + getValue: () => preview, + setValue: (value: Record) => { + preview = value + } + } + ) + this.addOutput('image', 'IMAGE') + this.serialize_widgets = true + } +} + class TestSink extends LGraphNode { constructor() { super('Test Sink') @@ -30,6 +55,7 @@ class TestSink extends LGraphNode { beforeEach(() => { LiteGraph.registerNodeType('TestSource', TestSource) LiteGraph.registerNodeType('TestSink', TestSink) + LiteGraph.registerNodeType('TestPreviewSource', TestPreviewSource) }) /** @@ -120,6 +146,39 @@ describe('attachRestoreOpMinter', () => { ]) }) + it('restores a deleted node with a display-only DOM widget in a batch the applier takes whole', () => { + const catalog: WidgetCatalog = { + types: { + TestSource: { widget_order: ['image'] }, + TestSink: { widget_order: [] }, + TestPreviewSource: { widget_order: ['image'] } + } + } + const preview = new TestPreviewSource() + graph.add(preview) + graph.remove(preview) + const doc = mint(graph.serialize() as unknown as WorkflowJSON, catalog) + const restored = new TestPreviewSource() + restored.id = preview.id + + restoreThrough(() => { + graph.add(restored) + restored.connect(0, sink, 0) + }) + + expect(minted[0]).toMatchObject({ op: 'add_node' }) + expect(minted[0]).toHaveProperty('node.widgets_values', { image: 'a.png' }) + const { outcomes } = applyOps( + doc, + mintWireOps(minted, { actor: 'human:user:tab', baseVersion: 1 }), + catalog + ) + expect(outcomes.map((outcome) => outcome.outcome)).toEqual( + minted.map(() => 'applied') + ) + doc.destroy() + }) + it('maps a restored link target from live slot order to document order', () => { sink.addInput('prompt', 'STRING') docInputs = ['prompt', 'image']