Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions browser_tests/tests/agent/agentHumanNoteWidgetWrite.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
import { expect, mergeTests } from '@playwright/test'
import type { Page, WebSocketRoute } from '@playwright/test'
import type { WidgetCatalog } from '@comfyorg/comfy-multi-player'

import {
BLANK_WORKFLOW,
agentTest,
bootAgentApp,
loadIntoBootWorkflow,
mockAgentTurnApi,
mockWorkflowPersistence
} from '@e2e/fixtures/agentPanelFixture'
import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc'
import type { WireOpEnvelope } from '@e2e/fixtures/agentWireFrame'
import { AgentPanel } from '@e2e/fixtures/components/AgentPanel'
import { subscribeAgentWorkflow } from '@e2e/fixtures/utils/subscribeAgentWorkflow'
import { nextFrame } from '@e2e/fixtures/utils/timing'
import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers'
import { webSocketFixture } from '@e2e/fixtures/ws'

const test = mergeTests(agentTest, webSocketFixture)

const WORKFLOW_ID = 'c7a21d40-0000-4000-8000-000000000050'
const MESSAGE_ID = 'c7a21d40-0000-4000-8000-000000000051'
const THREAD_ID = 'c7a21d40-0000-4000-8000-000000000052'

const TYPED_NOTE = 'remember to swap the LoRA before the final render'

const catalog: WidgetCatalog = { types: {} }

interface OpsFrame {
type: string
data: { ops?: WireOpEnvelope[] }
}
Comment thread
christian-byrne marked this conversation as resolved.
Resolved

function docOpsFrames(frames: readonly string[]): OpsFrame[] {
return frames
.map((frame) => JSON.parse(frame) as OpsFrame)
.filter((frame) => frame.type === 'doc_ops')
}
Comment thread
christian-byrne marked this conversation as resolved.
Resolved

function relayToHost(
host: HostDoc,
socket: WebSocketRoute,
ops: WireOpEnvelope[]
): void {
const { result, update } = host.applyWire(ops)
socket.send(JSON.stringify(result))
if (update !== null) socket.send(JSON.stringify(update))
}

async function bootWithHost(
page: Page,
getAgentSocket: () => Promise<WebSocketRoute>
) {
await page.setViewportSize({ width: 1920, height: 1280 })
await page.addInitScript(() => {
localStorage.setItem('Comfy.Agent.CrdtFollower', 'true')
})
await bootAgentApp(page, true, {
objectInfo: {},
beforeNavigate: async (page) => {
await mockAgentTurnApi(page, {
message_id: MESSAGE_ID,
thread_id: THREAD_ID,
workflow_id: WORKFLOW_ID
})
await mockWorkflowPersistence(page, WORKFLOW_ID)
}
})
await loadIntoBootWorkflow(page, BLANK_WORKFLOW)

const agentPanel = new AgentPanel(page)
await agentPanel.open()
const socket = await getAgentSocket()
const outboundFrames = await subscribeAgentWorkflow({
agentPanel,
socket,
workflowId: WORKFLOW_ID,
prompt: 'Leave the canvas to me'
})

const host = new HostDoc(WORKFLOW_ID, { nodes: [], links: [] }, catalog)
for (const frame of host.initialSync()) socket.send(JSON.stringify(frame))
await expect
.poll(() => page.evaluate(() => window.app!.graph.nodes.length))
.toBe(0)

return { host, socket, outboundFrames, vueNodes: new VueNodeHelpers(page) }
}

test.describe(
'A human edit to a frontend-only node the catalog cannot describe',
{ tag: ['@cloud', '@agent', '@canvas', '@node', '@widget', '@vue-nodes'] },
() => {
test.use({ connectWebSocketToServer: false })
test.describe.configure({ timeout: 90_000 })

test('keeps the text the user typed into a Note, with no rejection notice', async ({
page,
toast,
getAgentSocket
}) => {
const { host, socket, outboundFrames, vueNodes } = await bootWithHost(
page,
getAgentSocket
)

const beforeAdd = outboundFrames.length
await page.evaluate(() => {
window.app!.graph.add(window.LiteGraph!.createNode('Note'))
})
await expect
.poll(() => outboundFrames.slice(beforeAdd), { timeout: 15_000 })
.toContainEqual(expect.stringContaining('"op":"add_node"'))
for (const frame of docOpsFrames(outboundFrames.slice(beforeAdd)))
relayToHost(host, socket, frame.data.ops ?? [])
await nextFrame(page)

const noteId = await page.evaluate(
() =>
window.app!.graph.nodes.find((node) => node.type === 'Note')?.id ??
null
)
expect(noteId).not.toBeNull()

await page
.getByRole('button', { name: 'Fit View (.)', exact: true })
.click()
const noteText = vueNodes
.getNodeLocator(String(noteId))
.getByLabel('text', { exact: true })

const beforeEdit = outboundFrames.length
await noteText.fill(TYPED_NOTE)
await noteText.blur()

await expect
.poll(() => outboundFrames.slice(beforeEdit), { timeout: 15_000 })
.toContainEqual(expect.stringContaining('"op":"set_widget"'))
for (const frame of docOpsFrames(outboundFrames.slice(beforeEdit)))
relayToHost(host, socket, frame.data.ops ?? [])
await nextFrame(page)
await nextFrame(page)

await expect(noteText).toHaveValue(TYPED_NOTE)

const stored = host
.projection()
.nodes.find((node) => String(node.id) === String(noteId))
expect(stored).toMatchObject({
type: 'Note',
widgets_values: { text: TYPED_NOTE },
widgets_values_form: { order: ['text'] }
})

await expect(
toast.toastErrors.filter({
hasText: 'Widget edit was rejected and was not saved'
})
).toHaveCount(0)
})
}
)
26 changes: 21 additions & 5 deletions src/workbench/extensions/agent/crdt/docOpMinter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -727,10 +727,7 @@ describe('attachDocOpMinter', () => {
).not.toHaveProperty('ghost')
})

// Note, PrimitiveNode, Get/Set nodes and blueprint hosts have no catalog
// entry: the applier rejects a name-keyed record for them and stores a
// positional array opaquely.
it('keeps add_node widgets_values positional for a frontend-only node, and the applier takes it', async () => {
it('declares a frontend-only node own widget form, so the applier takes a later widget write', async () => {
const note = new TestNote()
graph.add(note)
await afterFlush()
Expand All @@ -739,7 +736,10 @@ describe('attachDocOpMinter', () => {
expect.objectContaining({
op: 'add_node',
class_type: 'TestNote',
node: expect.objectContaining({ widgets_values: ['a note'] })
node: expect.objectContaining({
widgets_values: { text: 'a note' },
widgets_values_form: { order: ['text'] }
})
})
])
const doc = mint({ nodes: [], links: [] }, CATALOG)
Expand All @@ -749,6 +749,22 @@ describe('attachDocOpMinter', () => {
CATALOG
)
expect(outcomes).toEqual([expect.objectContaining({ outcome: 'applied' })])

expect(
applyOutcomes(
doc,
[
{
op: 'set_widget',
node_id: note.id,
widget: 'text',
value: 'edited'
}
],
CATALOG,
2
)
).toEqual([expect.objectContaining({ outcome: 'applied' })])
doc.destroy()
})

Expand Down
21 changes: 13 additions & 8 deletions src/workbench/extensions/agent/crdt/docOpMinter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,15 @@ interface PendingWidgetValue {

type MissingSlotGrowth = NonNullable<ReturnType<typeof growForMissingSlot>>

/** Returns the node's save-format snapshot for a semantic add operation. */
/**
* Returns the node's save-format snapshot for a semantic add operation.
*
* Excludes only subgraph hosts, whose widget values are positional by design
* (`applyHostWidgets` indexes them against the live promoted inputs). Every
* other named-value node gets a declared `widgets_values_form` when one can
* be proven (schema A24); otherwise it keeps the positional snapshot rather
* than shipping an undeclared name-keyed record (KA-12).
*/
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
export function wireNodeSnapshot(node: LGraphNode): WorkflowNode | null {
let serialized: ISerialisedNode
try {
Expand All @@ -208,15 +216,12 @@ export function wireNodeSnapshot(node: LGraphNode): WorkflowNode | null {
...rest
} = wireSerialized
const snapshot = { ...rest, flags } satisfies WorkflowNode
if (!named || node.isVirtualNode) return snapshot
if (!named || node.isSubgraphNode()) return snapshot
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const values = valueWidgetsOnly(node, named)
// The form is proven against the POST-HOOK serialization, not derived from
// the live widget list alone: `serialize()` above already ran the extension
// serialize hook, which may have replaced either widget-values field.
const form = provenWidgetValuesForm(node, values, serialized)
return form
? { ...snapshot, widgets_values: values, widgets_values_form: form }
: { ...snapshot, widgets_values: values }
if (form)
return { ...snapshot, widgets_values: values, widgets_values_form: form }
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return node.isVirtualNode ? snapshot : { ...snapshot, widgets_values: values }
}

function nodeKey(graphId: string, nodeId: NodeId): string {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ describe('a human-added node across a tab switch', () => {
{
name: 'a frontend-only (virtual) node',
type: 'TestVirtual',
widgetsValues: ['x']
widgetsValues: { name: 'x' }
}
])(
'keeps $name and delivers a batch queued behind its in-flight add across a tab switch with an advancing host',
Expand Down
Loading