Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
364 changes: 364 additions & 0 deletions browser_tests/fixtures/agentAutogrowSpareInputFixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,364 @@
import type { Page } from '@playwright/test'
import { expect } from '@playwright/test'

import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player'
import type { ComfyNodeDef } from '@/schemas/nodeDefSchema'
import { toNodeId } from '@/types/nodeId'

import {
agentTest as test,
bootAgentApp
} from '@e2e/fixtures/agentPanelFixture'
import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc'
import { parseClientDocFrame } from '@e2e/fixtures/agentFollowerHostSocket'
import { Topbar } from '@e2e/fixtures/components/Topbar'
import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers'
import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute'
import { nextFrame } from '@e2e/fixtures/utils/timing'
import enMessages from '@/locales/en/main.json' with { type: 'json' }

/**
* A node with an "autogrow" (dynamic/variadic) input group — e.g. the GPT
* Image 2.5 node's `model.images.image_1/2/3...` slots — loses the spare,
* unconnected slot at the tail of that group once the tab is switched away
* and back after the in-app agent has wired something into the group over
* the CRDT graph-mutation layer. The user is left with a node whose only
* image slot is already taken and no visible way to add a second image; the
* reported symptom (PM-1155) is a wire that appears to land on the wrong
* widget, because the slot the user expected to be free is gone.
*
* Root cause: the spare slot is computed client-side by
* `src/core/graph/widgets/dynamicWidgets.ts` when an autogrow input is
* connected, and the CRDT document never carries it. A tab return
* re-subscribes the follower and replays the document over the live nodes,
* and `reconcile()` in `agentNodeMaterializer.ts` used to `continue` past
* any live node the agent scope already owns — so the payload's input list
* won, and the client-only spare was dropped.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

question (non-blocking, not a request to change this PR): who owns the node's input list?

This paragraph states it plainly — the spare slot is computed client-side and "the CRDT document never carries it" — while the document does carry the node's inputs (the seed below has inputs: [{ name: IMAGE_1_NAME, link: LINK_ID }], and updateNodeSlots is the remote writer for it). So two writers share state.inputs, arbitrated by re-deriving the spare after every reconcile.

I want to be fair about this: I don't think it's wrong. reconcileAutogrowInputs derives the spare from node.comfyDynamic.autogrow (which comes from /object_info, server-authoritative) plus the connected links (document-authoritative), so it's a pure function of two authoritative inputs rather than invented state — and building slots from /object_info is how every ComfyUI client works, agent or not.

What I'd like recorded is which of these is the intended model, since the answer decides whether the next bug in this area is a follower bug or a document bug:

  1. The slot list is document-authoritative and the grown slot should eventually be written into the document, with reconcileAutogrowInputs as an interim measure; or
  2. The slot list is deliberately client-derived from the node definition, and the document's inputs is a restoration hint that the client is expected to re-expand.

If it's (2), a line saying so here would be worth more than the fix itself for whoever reads this next. Either way this PR's array-identity fix is needed and correct — it's what makes the derived slot reach the renderer at all.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tracked as FE-2624. The PR merged with this thread unresolved, so the finding is carried in Linear rather than lost here. Filed automatically; close the ticket if the finding is no longer true at main.

*
* Fixed in #18127 (`3c202143e2`): `reconcile()` now calls
* `reconcileAutogrowInputs(live)` for exactly those already-owned live
* nodes, which re-runs the existing growth path for each autogrow group
* whose last connected slot has no spare behind it. That fix shipped with
* unit cover only.
*
* Sibling case: `agentAutogrowTabSwitchReconcile.spec.ts` drives the same
* seed and the same tab switch, and asserts the surviving slot's color and
* friendly label (#18081). This one asserts the slot that has to be there
* *next to* it.
*/

const GPT_IMAGE_NODE_TYPE = 'OpenAIGPTImageNodeV2'
const IMAGE_SOURCE_NODE_TYPE = 'TestImageSource'
const IMAGES_GROUP = 'model.images'
const IMAGE_1_NAME = `${IMAGES_GROUP}.image_1`
const IMAGE_2_NAME = `${IMAGES_GROUP}.image_2`
const IMAGE_2_FRIENDLY_LABEL = 'image_2'

const SOURCE_NODE_ID = 1
const GPT_NODE_ID = 2
const LINK_ID = 9001

const WORKFLOW_ID = '2f1a9b4e-3c7d-4e9a-9a1e-1c2d3e4f5a6b'
const THREAD_ID = 'b1c9a7d2-5e4f-4a3b-9c1d-6e7f8a9b0c1d'
const MESSAGE_ID = 'd4e5f6a7-8b9c-4d0e-9f1a-2b3c4d5e6f7a'

// Trimmed `/object_info` entries: a plain IMAGE source, and an API node whose
// only input is a `COMFY_AUTOGROW_V3` group named `model.images`, mirroring
// the real GPT Image 2.5 node's `model.images.image_N` slots.
const imageSourceNodeDef: ComfyNodeDef = {
name: IMAGE_SOURCE_NODE_TYPE,
display_name: 'Test Image Source',
description: '',
category: 'test',
python_module: 'test',
output_node: false,
output: ['IMAGE'],
output_is_list: [false],
output_name: ['IMAGE'],
input: { required: {} },
input_order: { required: [] }
}

const gptImageNodeDef: ComfyNodeDef = {
name: GPT_IMAGE_NODE_TYPE,
display_name: 'GPT Image 2.5',
description: '',
category: 'api node/image/OpenAI',
python_module: 'comfy_api_nodes.nodes_openai',
output_node: false,
api_node: true,
output: ['IMAGE'],
output_is_list: [false],
output_name: ['IMAGE'],
input: {
required: {
[IMAGES_GROUP]: [
'COMFY_AUTOGROW_V3',
{
template: {
input: { required: { image: ['IMAGE', {}] } },
names: ['image_1', 'image_2', 'image_3', 'image_4'],
min: 0
}
}
]
}
},
input_order: { required: [IMAGES_GROUP] }
}

const catalog: WidgetCatalog = {
types: {
[IMAGE_SOURCE_NODE_TYPE]: { widget_order: [] },
[GPT_IMAGE_NODE_TYPE]: { widget_order: [] }
}
}

// The graph an in-app agent conversation already left on the canvas: an
// image source wired into the GPT Image node's one autogrow slot. The
// document carries that one slot only — the spare is the client's, which is
// the whole point of the case.
const seed: WorkflowJSON = {
nodes: [
{
id: SOURCE_NODE_ID,
type: IMAGE_SOURCE_NODE_TYPE,
pos: [0, 0],
size: [210, 60],
flags: {},
order: 0,
mode: 0,
inputs: [],
outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [LINK_ID] }],
properties: {},
widgets_values: []
},
{
id: GPT_NODE_ID,
type: GPT_IMAGE_NODE_TYPE,
pos: [400, 0],
size: [300, 200],
flags: {},
order: 1,
mode: 0,
inputs: [{ name: IMAGE_1_NAME, type: 'IMAGE', link: LINK_ID }],
outputs: [{ name: 'IMAGE', type: 'IMAGE', links: [] }],
properties: {},
widgets_values: []
}
],
links: [[LINK_ID, SOURCE_NODE_ID, 0, GPT_NODE_ID, 0, 'IMAGE']],
groups: [],
config: {},
extra: {},
version: 0.4
}

async function wireAutogrowNodeAndSwitchTabs(page: Page) {
// `objectInfo: 'server'` (passed to `bootAgentApp` below) stops
// `mockCloudBootRoutes` from also registering an (empty) handler for
// this route — Playwright runs the most-recently-registered handler
// first, so a route added here after that call would never fire.
await page.route('**/api/object_info', (route) =>
route.fulfill(
jsonRoute({
[IMAGE_SOURCE_NODE_TYPE]: imageSourceNodeDef,
[GPT_IMAGE_NODE_TYPE]: gptImageNodeDef
})
)
)

const host = new HostDoc(WORKFLOW_ID, seed, catalog)
let socketSend: ((frame: unknown) => void) | null = null
let subscribedTo: string | null = null
await page.routeWebSocket(/\/ws/, (socket) => {
socketSend = (frame) => socket.send(JSON.stringify(frame))
socket.onMessage((raw) => {
const frame = parseClientDocFrame(raw)
if (
frame?.type !== 'doc_subscribe' ||
frame.workflowId !== WORKFLOW_ID ||
frame.stateVector === null
)
return
subscribedTo = frame.workflowId
socketSend!(host.subscribed())
socketSend!(host.catchUp(frame.stateVector))
})
socketSend({
type: 'status',
data: { status: { exec_info: { queue_remaining: 0 } }, sid: 's' }
})
})
await page.route('**/api/agent/threads', (route) =>
route.fulfill(jsonRoute({ threads: [] }))
)
await page.route('**/api/agent/run-mode', (route) =>
route.fulfill(jsonRoute({ mode: 'ask_approval', credit_limit: null }))
)
// The CRDT follower only binds a workflow once a real turn's ack
// names one (`bindWorkflow(ack.workflow_id)` in `useAgentSession.ts`)
// — picking a target from the switcher alone does not bind it.
await page.route('**/api/agent/threads/*/messages', (route) => {
if (route.request().method() !== 'POST') return route.fulfill(jsonRoute([]))
return route.fulfill({
status: 202,
contentType: 'application/json',
body: JSON.stringify({
thread_id: THREAD_ID,
message_id: MESSAGE_ID,
workflow_id: WORKFLOW_ID
})
})
})

await bootAgentApp(page, true, {
objectInfo: 'server',
// Only the Vue node renderer projects follower edits onto the
// canvas as DOM nodes this test can query.
settings: {
'Comfy.VueNodes.Enabled': true,
'Comfy.Graph.CanvasInfo': false
}
})

// Registered only now, same as `AgentConversationHarness.
// selectWorkflowTarget`: `bootAgentApp`'s own mocks blanket-match
// `**/api/userdata**` for every method, and Playwright runs the
// most-recently-registered matching route first, so these have to
// come after it to take over the workflow-save round trip.
let savedName: string | undefined
await page.route('**/api/userdata/*', (route) => {
const request = route.request()
const path = decodeURIComponent(
new URL(request.url()).pathname.split('/userdata/')[1]
)
if (request.method() !== 'POST' || !path.startsWith('workflows/'))
return route.fallback()
savedName = path.slice('workflows/'.length, -'.json'.length)
return route.fulfill(
jsonRoute({
path,
modified: Date.now(),
size: request.postDataBuffer()?.length ?? 0
})
)
})
await page.route('**/api/workflows?*', (route) =>
route.fulfill(
jsonRoute({
data:
savedName === undefined
? []
: [
{
id: WORKFLOW_ID,
name: savedName,
created_at: '2026-09-01T00:00:00Z',
updated_at: '2026-09-01T00:00:00Z',
created_by: 'test-user-e2e',
latest_version: 1
}
],
pagination: {
has_more: false,
limit: 100,
offset: 0,
total: savedName === undefined ? 0 : 1
}
})
)
)

const topbar = new Topbar(page)
const vueNodes = new VueNodeHelpers(page)
const gptNodeId = String(GPT_NODE_ID)
const panel = page.locator('#agent-panel-root')

// The user-facing question this whole case asks: "can I still wire a
// second image into this node?" — which is a free slot in the
// `model.images` group, named and unconnected.
const readImageSlots = () =>
page.evaluate((id) => {
const node = window.app!.graph.getNodeById(id)
return (node?.inputs ?? [])
.filter((input) => input.name.startsWith('model.images.'))
.map((input, index) => ({
name: input.name,
connected: node!.getInputLink(index) !== null

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (non-blocking): index here is the post-filter index, not the node's slot index.

.filter(...) runs before .map((input, index) => ...), so index counts only model.images.* slots while node.getInputLink(index) expects a real index into node.inputs. It happens to line up today because gptImageNodeDef declares the autogrow group as its only input, so the two indexes coincide.

That makes the helper quietly fragile in a way that matters for a regression test: if the node def ever gains an input ahead of the group (a widget-input, a prompt, anything), getInputLink starts reading a different slot's link and { image_1: connected: true, image_2: connected: false } could still be reported while the real slot state is wrong — a green test over a broken case.

Capturing the true index before filtering keeps it honest:

return (node?.inputs ?? [])
  .map((input, index) => ({ input, index }))
  .filter(({ input }) => input.name.startsWith('model.images.'))
  .map(({ input, index }) => ({
    name: input.name,
    connected: node!.getInputLink(index) !== null
  }))

Same suggestion applies to the assertions that consume it in agentAutogrowSpareInputReconcile.spec.ts, which read this helper's output directly.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tracked as FE-2623. The PR merged with this thread unresolved, so the finding is carried in Linear rather than lost here. Filed automatically; close the ticket if the finding is no longer true at main.

}))
}, toNodeId(GPT_NODE_ID))

await test.step('open the agent panel and target the workflow', async () => {
await page
.getByRole('button', { name: enMessages.agent.entryButton })
.click()
await expect(panel).toBeVisible()
await panel
.getByRole('button', { name: enMessages.agent.switchWorkflow })
.click()
await page
.getByRole('menuitemradio', { name: 'Unsaved Workflow', exact: true })
.click()
})

await test.step('send a turn so the session binds the workflow', async () => {
const composer = panel.getByRole('textbox', {
name: /^Describe ideas/
})
await composer.fill('hello')
await panel.getByRole('button', { name: enMessages.agent.send }).click()
await expect(panel.getByText('hello').first()).toBeVisible()
await expect.poll(() => subscribedTo, { timeout: 20_000 }).toBe(WORKFLOW_ID)
socketSend!({
type: 'agent_message_done',
data: { message_id: MESSAGE_ID, thread_id: THREAD_ID }
})
await expect(
panel.getByRole('button', { name: enMessages.agent.stop })
).toHaveCount(0)
})

await test.step('the wired node offers a free image slot beforehand', async () => {
await expect(vueNodes.getNodeLocator(gptNodeId)).toBeVisible()
await expect.poll(readImageSlots).toEqual([
{ name: IMAGE_1_NAME, connected: true },
{ name: IMAGE_2_NAME, connected: false }
])
await expect(vueNodes.getInputSlotRow(gptNodeId, 1)).toContainText(
IMAGE_2_FRIENDLY_LABEL
)
})

await test.step('user switches to a new tab and back', async () => {
await expect(topbar.workflowTabs.locator('.p-togglebutton')).toHaveCount(1)
await topbar.newWorkflowButton.click()
await expect(topbar.workflowTabs.locator('.p-togglebutton')).toHaveCount(2)
await topbar.getTab(0).click()
await expect(topbar.getTab(0)).toHaveClass(/p-togglebutton-checked/)
})

await page.evaluate((id) => {
const node = window.app!.graph.getNodeById(id)!
const canvas = window.app!.canvas
canvas.ds.fitToBounds(node.boundingRect, { zoom: 0.5 })
canvas.setDirty(true, true)
}, toNodeId(GPT_NODE_ID))
await nextFrame(page)
await expect(vueNodes.getNodeLocator(gptNodeId)).toBeInViewport({
ratio: 1
})
await test.info().attach('after-tab-switch', {
body: await vueNodes.getNodeLocator(gptNodeId).screenshot(),
contentType: 'image/png'
})
return { vueNodes, gptNodeId, readImageSlots }
}

export const autogrowTest = test.extend<{
autogrow: Awaited<ReturnType<typeof wireAutogrowNodeAndSwitchTabs>>
}>({
autogrow: async ({ page }, use) => {
await use(await wireAutogrowNodeAndSwitchTabs(page))
}
})
2 changes: 1 addition & 1 deletion browser_tests/fixtures/agentFollowerHostSocket.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function stringOrNull(value: unknown): string | null {
return typeof value === 'string' ? value : null
}

function parseClientDocFrame(
export function parseClientDocFrame(
raw: string | Buffer
): ParsedClientDocFrame | null {
const envelope = docFrameEnvelope(raw)
Expand Down
Loading
Loading