diff --git a/browser_tests/tests/agent/agentHumanNoteWidgetWrite.spec.ts b/browser_tests/tests/agent/agentHumanNoteWidgetWrite.spec.ts new file mode 100644 index 00000000000..c9c51a02847 --- /dev/null +++ b/browser_tests/tests/agent/agentHumanNoteWidgetWrite.spec.ts @@ -0,0 +1,164 @@ +import { expect, mergeTests } from '@playwright/test' +import type { Page, WebSocketRoute } from '@playwright/test' +import type { WidgetCatalog } from '@comfyorg/comfy-multi-player' + +import { + BLANK_WORKFLOW, + agentTest, + bootAgentApp, + loadIntoBootWorkflow, + mockAgentTurnApi, + mockWorkflowPersistence +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import type { WireOpEnvelope } from '@e2e/fixtures/agentWireFrame' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { subscribeAgentWorkflow } from '@e2e/fixtures/utils/subscribeAgentWorkflow' +import { nextFrame } from '@e2e/fixtures/utils/timing' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const test = mergeTests(agentTest, webSocketFixture) + +const WORKFLOW_ID = 'c7a21d40-0000-4000-8000-000000000050' +const MESSAGE_ID = 'c7a21d40-0000-4000-8000-000000000051' +const THREAD_ID = 'c7a21d40-0000-4000-8000-000000000052' + +const TYPED_NOTE = 'remember to swap the LoRA before the final render' + +const catalog: WidgetCatalog = { types: {} } + +interface OpsFrame { + type: string + data: { ops?: WireOpEnvelope[] } +} + +function docOpsFrames(frames: readonly string[]): OpsFrame[] { + return frames + .map((frame) => JSON.parse(frame) as OpsFrame) + .filter((frame) => frame.type === 'doc_ops') +} + +function relayToHost( + host: HostDoc, + socket: WebSocketRoute, + ops: WireOpEnvelope[] +): void { + const { result, update } = host.applyWire(ops) + socket.send(JSON.stringify(result)) + if (update !== null) socket.send(JSON.stringify(update)) +} + +async function bootWithHost( + page: Page, + getAgentSocket: () => Promise +) { + await page.setViewportSize({ width: 1920, height: 1280 }) + await page.addInitScript(() => { + localStorage.setItem('Comfy.Agent.CrdtFollower', 'true') + }) + await bootAgentApp(page, true, { + objectInfo: {}, + beforeNavigate: async (page) => { + await mockAgentTurnApi(page, { + message_id: MESSAGE_ID, + thread_id: THREAD_ID, + workflow_id: WORKFLOW_ID + }) + await mockWorkflowPersistence(page, WORKFLOW_ID) + } + }) + await loadIntoBootWorkflow(page, BLANK_WORKFLOW) + + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const socket = await getAgentSocket() + const outboundFrames = await subscribeAgentWorkflow({ + agentPanel, + socket, + workflowId: WORKFLOW_ID, + prompt: 'Leave the canvas to me' + }) + + const host = new HostDoc(WORKFLOW_ID, { nodes: [], links: [] }, catalog) + for (const frame of host.initialSync()) socket.send(JSON.stringify(frame)) + await expect + .poll(() => page.evaluate(() => window.app!.graph.nodes.length)) + .toBe(0) + + return { host, socket, outboundFrames, vueNodes: new VueNodeHelpers(page) } +} + +test.describe( + 'A human edit to a frontend-only node the catalog cannot describe', + { tag: ['@cloud', '@agent', '@canvas', '@node', '@widget', '@vue-nodes'] }, + () => { + test.use({ connectWebSocketToServer: false }) + test.describe.configure({ timeout: 90_000 }) + + test('keeps the text the user typed into a Note, with no rejection notice', async ({ + page, + toast, + getAgentSocket + }) => { + const { host, socket, outboundFrames, vueNodes } = await bootWithHost( + page, + getAgentSocket + ) + + const beforeAdd = outboundFrames.length + await page.evaluate(() => { + window.app!.graph.add(window.LiteGraph!.createNode('Note')) + }) + await expect + .poll(() => outboundFrames.slice(beforeAdd), { timeout: 15_000 }) + .toContainEqual(expect.stringContaining('"op":"add_node"')) + for (const frame of docOpsFrames(outboundFrames.slice(beforeAdd))) + relayToHost(host, socket, frame.data.ops ?? []) + await nextFrame(page) + + const noteId = await page.evaluate( + () => + window.app!.graph.nodes.find((node) => node.type === 'Note')?.id ?? + null + ) + expect(noteId).not.toBeNull() + + await page + .getByRole('button', { name: 'Fit View (.)', exact: true }) + .click() + const noteText = vueNodes + .getNodeLocator(String(noteId)) + .getByLabel('text', { exact: true }) + + const beforeEdit = outboundFrames.length + await noteText.fill(TYPED_NOTE) + await noteText.blur() + + await expect + .poll(() => outboundFrames.slice(beforeEdit), { timeout: 15_000 }) + .toContainEqual(expect.stringContaining('"op":"set_widget"')) + for (const frame of docOpsFrames(outboundFrames.slice(beforeEdit))) + relayToHost(host, socket, frame.data.ops ?? []) + await nextFrame(page) + await nextFrame(page) + + await expect(noteText).toHaveValue(TYPED_NOTE) + + const stored = host + .projection() + .nodes.find((node) => String(node.id) === String(noteId)) + expect(stored).toMatchObject({ + type: 'Note', + widgets_values: { text: TYPED_NOTE }, + widgets_values_form: { order: ['text'] } + }) + + await expect( + toast.toastErrors.filter({ + hasText: 'Widget edit was rejected and was not saved' + }) + ).toHaveCount(0) + }) + } +) diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index dd7aa3fb042..b317294929a 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -727,10 +727,7 @@ describe('attachDocOpMinter', () => { ).not.toHaveProperty('ghost') }) - // Note, PrimitiveNode, Get/Set nodes and blueprint hosts have no catalog - // entry: the applier rejects a name-keyed record for them and stores a - // positional array opaquely. - it('keeps add_node widgets_values positional for a frontend-only node, and the applier takes it', async () => { + it('declares a frontend-only node own widget form, so the applier takes a later widget write', async () => { const note = new TestNote() graph.add(note) await afterFlush() @@ -739,7 +736,10 @@ describe('attachDocOpMinter', () => { expect.objectContaining({ op: 'add_node', class_type: 'TestNote', - node: expect.objectContaining({ widgets_values: ['a note'] }) + node: expect.objectContaining({ + widgets_values: { text: 'a note' }, + widgets_values_form: { order: ['text'] } + }) }) ]) const doc = mint({ nodes: [], links: [] }, CATALOG) @@ -749,6 +749,22 @@ describe('attachDocOpMinter', () => { CATALOG ) expect(outcomes).toEqual([expect.objectContaining({ outcome: 'applied' })]) + + expect( + applyOutcomes( + doc, + [ + { + op: 'set_widget', + node_id: note.id, + widget: 'text', + value: 'edited' + } + ], + CATALOG, + 2 + ) + ).toEqual([expect.objectContaining({ outcome: 'applied' })]) doc.destroy() }) diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index 303ffa51b54..62f134057d7 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -191,7 +191,15 @@ interface PendingWidgetValue { type MissingSlotGrowth = NonNullable> -/** Returns the node's save-format snapshot for a semantic add operation. */ +/** + * Returns the node's save-format snapshot for a semantic add operation. + * + * Excludes only subgraph hosts, whose widget values are positional by design + * (`applyHostWidgets` indexes them against the live promoted inputs). Every + * other named-value node gets a declared `widgets_values_form` when one can + * be proven (schema A24); otherwise it keeps the positional snapshot rather + * than shipping an undeclared name-keyed record (KA-12). + */ export function wireNodeSnapshot(node: LGraphNode): WorkflowNode | null { let serialized: ISerialisedNode try { @@ -208,15 +216,12 @@ export function wireNodeSnapshot(node: LGraphNode): WorkflowNode | null { ...rest } = wireSerialized const snapshot = { ...rest, flags } satisfies WorkflowNode - if (!named || node.isVirtualNode) return snapshot + if (!named || node.isSubgraphNode()) return snapshot const values = valueWidgetsOnly(node, named) - // The form is proven against the POST-HOOK serialization, not derived from - // the live widget list alone: `serialize()` above already ran the extension - // serialize hook, which may have replaced either widget-values field. const form = provenWidgetValuesForm(node, values, serialized) - return form - ? { ...snapshot, widgets_values: values, widgets_values_form: form } - : { ...snapshot, widgets_values: values } + if (form) + return { ...snapshot, widgets_values: values, widgets_values_form: form } + return node.isVirtualNode ? snapshot : { ...snapshot, widgets_values: values } } function nodeKey(graphId: string, nodeId: NodeId): string { diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts index df2cc885f3f..48c98a45220 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts @@ -188,7 +188,7 @@ describe('a human-added node across a tab switch', () => { { name: 'a frontend-only (virtual) node', type: 'TestVirtual', - widgetsValues: ['x'] + widgetsValues: { name: 'x' } } ])( 'keeps $name and delivers a batch queued behind its in-flight add across a tab switch with an advancing host',