Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import { expect } from '@playwright/test'

import enMessages from '@/locales/en/main.json' with { type: 'json' }

import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture'

test.describe(
'Promoted widget edit on a human-added subgraph blueprint with Agent bound',
{ tag: ['@cloud', '@agent', '@subgraph', '@widget', '@vue-nodes'] },
() => {
test.use({
conversationCase: 'agent-rec-text-only-answer',
humanOpsHost: 'apply'
})

test('saves the edit to the shared document without a rejection notice', async ({
agentConversation,
comfyPage
}) => {
test.setTimeout(90_000)

await test.step('replay the recorded turn', () =>
agentConversation.runTurns())

const hostId =
await test.step('add a blueprint that promotes its prompt', async () => {
const added = await agentConversation.addBlueprint(true, [400, 400])
await expect(
agentConversation.vueNodes.getNodeLocator(added)
).toBeVisible()
const outcomes = await agentConversation.waitForHumanOps(1)
expect(
outcomes.filter(({ outcome }) => outcome === 'rejected')
).toEqual([])
await expect
.poll(() => agentConversation.hostNodeIds())
.toContain(added)
return added
})

await test.step('edit the promoted prompt on the blueprint by hand', async () => {
const field = agentConversation.vueNodes
.getNodeLocator(hostId)
.getByLabel('text', { exact: true })
await field.fill('an edited blueprint prompt')
await field.blur()
await agentConversation.waitForPendingFrames(
'6',
'text',
'blueprint edit judged'
)
})

await test.step('no rejected-edit notice is shown', async () => {
await expect
.soft(
comfyPage.toast.toastErrors.filter({
hasText: enMessages.agent.editRejected.widgetWrite
})
)
.toHaveCount(0)
})

await test.step('the edit reaches the shared document', async () => {
await expect
.poll(() => agentConversation.hostNode(hostId)?.widgets_values)
.toEqual(expect.arrayContaining(['an edited blueprint prompt']))
})
})
}
)
2 changes: 2 additions & 0 deletions src/workbench/extensions/agent/AgentPanelRoot.vue
Original file line number Diff line number Diff line change
Expand Up @@ -929,6 +929,7 @@ const {
docInputNames,
docPromotedWidgets,
mayWritePromotedHost,
docHasSubgraphDefinition,
notifyWidgetWriteRefused
} = useAgentCrdtFollower(
boundWorkflowId,
Expand Down Expand Up @@ -987,6 +988,7 @@ const docOpMinter = attachDocOpMinter({
docInputNames,
docPromotedWidgets,
mayWritePromotedHost,
docHasSubgraphDefinition,
onWidgetWriteRefused: notifyWidgetWriteRefused
})
const restoreOpMinter = attachRestoreOpMinter({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ describe('a human link into a widget input the agent-built document lacks', () =
docInputNames: (id) => readDocSlotNames(doc, String(id), 'inputs'),
docPromotedWidgets: (id) => readDocPromotedWidgets(doc, String(id)),
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {}
})
onTestFinished(() => port.detach())
Expand Down Expand Up @@ -227,6 +228,7 @@ describe('a human link into a widget input the agent-built document lacks', () =
docInputNames: (id) => readDocSlotNames(doc, String(id), 'inputs'),
docPromotedWidgets: (id) => readDocPromotedWidgets(doc, String(id)),
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {}
})
onTestFinished(() => port.detach())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ function connectAgentHeldWorkflow() {
docInputNames: (nodeId) => follower.docInputNames(nodeId),
docPromotedWidgets: (nodeId) => follower.docPromotedWidgets(nodeId),
mayWritePromotedHost: (nodeId) => follower.mayWritePromotedHost(nodeId),
docHasSubgraphDefinition: (definitionId) =>
follower.docHasSubgraphDefinition(definitionId),
onWidgetWriteRefused: () => follower.notifyWidgetWriteRefused()
})
onTestFinished(() => minter.detach())
Expand Down
81 changes: 81 additions & 0 deletions src/workbench/extensions/agent/crdt/docOpMinter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ describe('attachDocOpMinter', () => {
let docInputNames: DocOpMinterDeps['docInputNames']
let docPromotedWidgets: DocOpMinterDeps['docPromotedWidgets']
let mayWrite: boolean
let docDefinitionIds: Set<string>
let refusals: number

beforeEach(() => {
Expand All @@ -225,6 +226,7 @@ describe('attachDocOpMinter', () => {
docInputNames = () => null
docPromotedWidgets = () => null
mayWrite = true
docDefinitionIds = new Set()
refusals = 0
minter = attachDocOpMinter({
isEnabled: () => enabled,
Expand All @@ -235,6 +237,7 @@ describe('attachDocOpMinter', () => {
docInputNames: (nodeId) => docInputNames(nodeId),
docPromotedWidgets: (nodeId) => docPromotedWidgets(nodeId),
mayWritePromotedHost: () => mayWrite,
docHasSubgraphDefinition: (id) => docDefinitionIds.has(id),
onWidgetWriteRefused: () => {
refusals += 1
}
Expand Down Expand Up @@ -528,6 +531,7 @@ describe('attachDocOpMinter', () => {
docInputNames: () => null,
docPromotedWidgets: () => null,
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {}
})

Expand Down Expand Up @@ -1315,6 +1319,83 @@ describe('attachDocOpMinter', () => {
expect(minted[0]).not.toHaveProperty('promoted')
})

function handAddedPromotedHost() {
const inner = createTestSubgraph({ rootGraph: graph })
graph.subgraphs.set(inner.id, inner)
const outer = createTestSubgraph({
rootGraph: graph,
inputs: [{ name: 'text', type: 'STRING' }]
})
graph.subgraphs.set(outer.id, outer)
withGraphIntentSource('load', () => {
outer.add(createTestSubgraphNode(inner, { parentGraph: outer }))
outer.add(createTestSubgraphNode(inner, { parentGraph: outer }))
const interior = LiteGraph.createNode('TestPrompt')
assert.exists(interior)
outer.add(interior)
outer.inputNode.slots[0].connect(interior.inputs[0], interior)
})
const doc = mintDocFrom(graph)
docPromotedWidgets = (nodeId) => readDocPromotedWidgets(doc, String(nodeId))
const host = createTestSubgraphNode(outer)
graph.add(host)
return { host, inner, outer, doc }
}

it('defines a hand-added host’s subgraphs the document lacks, so its promoted write places', async () => {
const { host, inner, outer, doc } = handAddedPromotedHost()
await afterFlush()

expect(
minted.map((op) => (op.op === 'define_subgraph' ? op.subgraph_id : op.op))
).toEqual(['add_node', inner.id, outer.id])
expect(applyMinted(doc, minted)).toEqual(['applied', 'applied', 'applied'])
expect(readDocPromotedWidgets(doc, String(host.id))).toMatchObject({
promotedNames: ['text']
})

minted = []
host.widgets[0].value = 'pasted'
await afterFlush()

expect(minted).toEqual([
expect.objectContaining({
op: 'set_widget',
widget: 'text',
promoted: expect.objectContaining({ value_index: 0 })
})
])
expect(refusals).toBe(0)
expect(applyMinted(doc, minted)).toEqual(['applied'])
})

it('keeps the add and skips a definition that cannot serialize, with its dependents', async () => {
const { inner } = handAddedPromotedHost()
vi.spyOn(inner, 'asSerialisable').mockImplementation(() => {
throw new Error('extension serializer failed')
})
await afterFlush()

expect(minted.map(({ op }) => op)).toEqual(['add_node'])
expect(reportError).toHaveBeenCalledWith(
expect.any(Error),
expect.objectContaining({
errorType: 'agent_crdt_subgraph_definition_unserializable',
context: { definitionId: inner.id }
})
)
})

it('does not redefine a subgraph the document already holds', async () => {
const { inner, outer } = handAddedPromotedHost()
docDefinitionIds.add(inner.id)
await afterFlush()

expect(
minted.map((op) => (op.op === 'define_subgraph' ? op.subgraph_id : op.op))
).toEqual(['add_node', outer.id])
})

it('does not let an ephemeral widget write roll a hand edit back with it', async () => {
const { source } = seedGraph(graph)
// Built the way useNodeProgressText builds it: `serialize: false` and
Expand Down
62 changes: 57 additions & 5 deletions src/workbench/extensions/agent/crdt/docOpMinter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import type {
NodeId as WireNodeId,
PromotedHostWrite,
SubgraphDefinition,
WidgetValuesForm,
WorkflowNode
} from '@comfyorg/comfy-multi-player'
Expand All @@ -27,6 +28,7 @@ import type { LGraph } from '@/lib/litegraph/src/LGraph'
import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode'
import type { LLink } from '@/lib/litegraph/src/LLink'
import type { Subgraph } from '@/lib/litegraph/src/subgraph/Subgraph'
import type { SubgraphNode } from '@/lib/litegraph/src/subgraph/SubgraphNode'
import type { ISerialisedNode } from '@/lib/litegraph/src/types/serialisation'
import { isWidgetValue } from '@/lib/litegraph/src/types/widgets'
import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets'
Expand Down Expand Up @@ -79,6 +81,8 @@ export interface DocOpMinterDeps {
* sent the root host's `add_node` into the same document.
*/
mayWritePromotedHost(nodeId: NodeId): boolean
/** Whether the bound document holds a subgraph definition by this id. */
docHasSubgraphDefinition(definitionId: string): boolean
onWidgetWriteRefused(): void
}

Expand Down Expand Up @@ -426,20 +430,68 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter {
}
}

function serialisedDefinition(subgraph: Subgraph): SubgraphDefinition | null {
try {
return JSON.parse(JSON.stringify(subgraph.asSerialisable()))
} catch (error) {
reportError(error, {
surface: 'agent',
errorType: 'agent_crdt_subgraph_definition_unserializable',
context: { definitionId: subgraph.id }
})
return null
}
}

function mintMissingDefinitions(
hosts: readonly SubgraphNode[]
): GraphOperation[] {
const operations: GraphOperation[] = []
const available = new Map<string, boolean>()
const define = (subgraph: Subgraph): boolean => {
const settled = available.get(subgraph.id)
if (settled !== undefined) return settled
available.set(subgraph.id, false)
const dependenciesAvailable = subgraph.nodes
.filter((node) => node.isSubgraphNode())
.map((node) => define(node.subgraph))
.every(Boolean)
if (deps.docHasSubgraphDefinition(subgraph.id)) {
available.set(subgraph.id, true)
return true
}
const definition = dependenciesAvailable
? serialisedDefinition(subgraph)
: null
if (!definition) return false
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
operations.push({
op: 'define_subgraph',
subgraph_id: subgraph.id,
subgraph_definition: definition
})
available.set(subgraph.id, true)
return true
}
for (const host of hosts) define(host.subgraph)
return operations
}

function flush(): void {
flushScheduled = false
reported.clear()
const batch = pending
pending = []
pendingAdds.clear()
if (detached) return
const hosts: SubgraphNode[] = []
const operations = batch.flatMap((entry) => {
const operation =
entry.kind === 'op'
? entry.operation
: mintAddNodeOperation(entry.graph, entry.node)
return operation ? [operation] : []
if (entry.kind === 'op') return [entry.operation]
const operation = mintAddNodeOperation(entry.graph, entry.node)
if (!operation) return []
if (entry.node.isSubgraphNode()) hosts.push(entry.node)
return [operation]
})
operations.push(...mintMissingDefinitions(hosts))
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
if (operations.length > 0) deps.enqueue(operations)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ describe('agent CRDT outbound leg: link mint by live position vs. doc order', ()
docPromotedWidgets: (nodeId) =>
readDocPromotedWidgets(doc, String(nodeId)),
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {}
})
onTestFinished(() => port.detach())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@ describe('LiveGraphApplier with the real change tracker', () => {
docInputNames: () => [],
docPromotedWidgets: () => null,
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {},
enqueue
})
Expand Down Expand Up @@ -617,6 +618,7 @@ describe('LiveGraphApplier with the real change tracker', () => {
docInputNames: () => [],
docPromotedWidgets: () => null,
mayWritePromotedHost: () => true,
docHasSubgraphDefinition: () => true,
onWidgetWriteRefused: () => {},
enqueue
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ describe('a human-added node across a tab switch', () => {
docInputNames: followerApi.docInputNames,
docPromotedWidgets: followerApi.docPromotedWidgets,
mayWritePromotedHost: followerApi.mayWritePromotedHost,
docHasSubgraphDefinition: followerApi.docHasSubgraphDefinition,
onWidgetWriteRefused: followerApi.notifyWidgetWriteRefused
})
const cleanup = () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ import type { GraphOperation } from './graphOperations'
import type { ClassifiedDocUpdate } from './layoutFollowerBridge'
import { LayoutFollowerBridge } from './layoutFollowerBridge'
import type { LiveGraphApplierDeps } from './liveGraphApplier'
import { readDocPromotedWidgets } from './agentSubgraphDefinitions'
import {
readDocPromotedWidgets,
readSubgraphDefinitionIds
} from './agentSubgraphDefinitions'
import { readDocSlotNames } from './liveGraphApplier'
import { createOpCoalescer } from './opCoalescer'
import { createOpSender } from './opSender'
Expand Down Expand Up @@ -392,6 +395,8 @@ export function useAgentCrdtFollower(
follower.value?.docPromotedWidgets(nodeId) ?? null,
mayWritePromotedHost: (nodeId: NodeId) =>
follower.value?.mayWritePromotedHost(nodeId) ?? false,
docHasSubgraphDefinition: (definitionId: string) =>
follower.value?.docHasSubgraphDefinition(definitionId) ?? false,
notifyWidgetWriteRefused: () => follower.value?.notifyWidgetWriteRefused()
}
}
Expand Down Expand Up @@ -1088,6 +1093,8 @@ function startAgentCrdtFollower(
bridge.isSubscribeBaselineIntegrated ||
(sentAddsToCurrentDoc().has(String(nodeId)) &&
readDocPromotedWidgets(bridge.follower.doc, String(nodeId)) === null),
docHasSubgraphDefinition: (definitionId: string) =>
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
readSubgraphDefinitionIds(bridge.follower.doc).includes(definitionId),
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
notifyWidgetWriteRefused: () =>
rejectedOpNotifier.notifyWidgetWriteRefused()
}
Expand Down
Loading