diff --git a/browser_tests/tests/agent/agentHumanAddedBlueprintPromotedWrite.spec.ts b/browser_tests/tests/agent/agentHumanAddedBlueprintPromotedWrite.spec.ts new file mode 100644 index 00000000000..320d5750f4b --- /dev/null +++ b/browser_tests/tests/agent/agentHumanAddedBlueprintPromotedWrite.spec.ts @@ -0,0 +1,71 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' + +test.describe( + 'Promoted widget edit on a human-added subgraph blueprint with Agent bound', + { tag: ['@cloud', '@agent', '@subgraph', '@widget', '@vue-nodes'] }, + () => { + test.use({ + conversationCase: 'agent-rec-text-only-answer', + humanOpsHost: 'apply' + }) + + test('saves the edit to the shared document without a rejection notice', async ({ + agentConversation, + comfyPage + }) => { + test.setTimeout(90_000) + + await test.step('replay the recorded turn', () => + agentConversation.runTurns()) + + const hostId = + await test.step('add a blueprint that promotes its prompt', async () => { + const added = await agentConversation.addBlueprint(true, [400, 400]) + await expect( + agentConversation.vueNodes.getNodeLocator(added) + ).toBeVisible() + const outcomes = await agentConversation.waitForHumanOps(1) + expect( + outcomes.filter(({ outcome }) => outcome === 'rejected') + ).toEqual([]) + await expect + .poll(() => agentConversation.hostNodeIds()) + .toContain(added) + return added + }) + + await test.step('edit the promoted prompt on the blueprint by hand', async () => { + const field = agentConversation.vueNodes + .getNodeLocator(hostId) + .getByLabel('text', { exact: true }) + await field.fill('an edited blueprint prompt') + await field.blur() + await agentConversation.waitForPendingFrames( + '6', + 'text', + 'blueprint edit judged' + ) + }) + + await test.step('no rejected-edit notice is shown', async () => { + await expect + .soft( + comfyPage.toast.toastErrors.filter({ + hasText: enMessages.agent.editRejected.widgetWrite + }) + ) + .toHaveCount(0) + }) + + await test.step('the edit reaches the shared document', async () => { + await expect + .poll(() => agentConversation.hostNode(hostId)?.widgets_values) + .toEqual(expect.arrayContaining(['an edited blueprint prompt'])) + }) + }) + } +) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index bdaee2d5284..d5736f377c6 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -929,6 +929,7 @@ const { docInputNames, docPromotedWidgets, mayWritePromotedHost, + docHasSubgraphDefinition, notifyWidgetWriteRefused } = useAgentCrdtFollower( boundWorkflowId, @@ -987,6 +988,7 @@ const docOpMinter = attachDocOpMinter({ docInputNames, docPromotedWidgets, mayWritePromotedHost, + docHasSubgraphDefinition, onWidgetWriteRefused: notifyWidgetWriteRefused }) const restoreOpMinter = attachRestoreOpMinter({ diff --git a/src/workbench/extensions/agent/crdt/agentBuiltWidgetInputLink.regression.test.ts b/src/workbench/extensions/agent/crdt/agentBuiltWidgetInputLink.regression.test.ts index f4d5a80b2fc..67b6070a22c 100644 --- a/src/workbench/extensions/agent/crdt/agentBuiltWidgetInputLink.regression.test.ts +++ b/src/workbench/extensions/agent/crdt/agentBuiltWidgetInputLink.regression.test.ts @@ -162,6 +162,7 @@ describe('a human link into a widget input the agent-built document lacks', () = docInputNames: (id) => readDocSlotNames(doc, String(id), 'inputs'), docPromotedWidgets: (id) => readDocPromotedWidgets(doc, String(id)), mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {} }) onTestFinished(() => port.detach()) @@ -227,6 +228,7 @@ describe('a human link into a widget input the agent-built document lacks', () = docInputNames: (id) => readDocSlotNames(doc, String(id), 'inputs'), docPromotedWidgets: (id) => readDocPromotedWidgets(doc, String(id)), mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {} }) onTestFinished(() => port.detach()) diff --git a/src/workbench/extensions/agent/crdt/agentDisplayIntAdd.regression.test.ts b/src/workbench/extensions/agent/crdt/agentDisplayIntAdd.regression.test.ts index 145afcf6e50..0da33fd9f2a 100644 --- a/src/workbench/extensions/agent/crdt/agentDisplayIntAdd.regression.test.ts +++ b/src/workbench/extensions/agent/crdt/agentDisplayIntAdd.regression.test.ts @@ -116,6 +116,8 @@ function connectAgentHeldWorkflow() { docInputNames: (nodeId) => follower.docInputNames(nodeId), docPromotedWidgets: (nodeId) => follower.docPromotedWidgets(nodeId), mayWritePromotedHost: (nodeId) => follower.mayWritePromotedHost(nodeId), + docHasSubgraphDefinition: (definitionId) => + follower.docHasSubgraphDefinition(definitionId), onWidgetWriteRefused: () => follower.notifyWidgetWriteRefused() }) onTestFinished(() => minter.detach()) diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index a63797bb479..3ea4305f82e 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -214,6 +214,7 @@ describe('attachDocOpMinter', () => { let docInputNames: DocOpMinterDeps['docInputNames'] let docPromotedWidgets: DocOpMinterDeps['docPromotedWidgets'] let mayWrite: boolean + let docDefinitionIds: Set let refusals: number beforeEach(() => { @@ -225,6 +226,7 @@ describe('attachDocOpMinter', () => { docInputNames = () => null docPromotedWidgets = () => null mayWrite = true + docDefinitionIds = new Set() refusals = 0 minter = attachDocOpMinter({ isEnabled: () => enabled, @@ -235,6 +237,7 @@ describe('attachDocOpMinter', () => { docInputNames: (nodeId) => docInputNames(nodeId), docPromotedWidgets: (nodeId) => docPromotedWidgets(nodeId), mayWritePromotedHost: () => mayWrite, + docHasSubgraphDefinition: (id) => docDefinitionIds.has(id), onWidgetWriteRefused: () => { refusals += 1 } @@ -528,6 +531,7 @@ describe('attachDocOpMinter', () => { docInputNames: () => null, docPromotedWidgets: () => null, mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {} }) @@ -1315,6 +1319,83 @@ describe('attachDocOpMinter', () => { expect(minted[0]).not.toHaveProperty('promoted') }) + function handAddedPromotedHost() { + const inner = createTestSubgraph({ rootGraph: graph }) + graph.subgraphs.set(inner.id, inner) + const outer = createTestSubgraph({ + rootGraph: graph, + inputs: [{ name: 'text', type: 'STRING' }] + }) + graph.subgraphs.set(outer.id, outer) + withGraphIntentSource('load', () => { + outer.add(createTestSubgraphNode(inner, { parentGraph: outer })) + outer.add(createTestSubgraphNode(inner, { parentGraph: outer })) + const interior = LiteGraph.createNode('TestPrompt') + assert.exists(interior) + outer.add(interior) + outer.inputNode.slots[0].connect(interior.inputs[0], interior) + }) + const doc = mintDocFrom(graph) + docPromotedWidgets = (nodeId) => readDocPromotedWidgets(doc, String(nodeId)) + const host = createTestSubgraphNode(outer) + graph.add(host) + return { host, inner, outer, doc } + } + + it('defines a hand-added host’s subgraphs the document lacks, so its promoted write places', async () => { + const { host, inner, outer, doc } = handAddedPromotedHost() + await afterFlush() + + expect( + minted.map((op) => (op.op === 'define_subgraph' ? op.subgraph_id : op.op)) + ).toEqual(['add_node', inner.id, outer.id]) + expect(applyMinted(doc, minted)).toEqual(['applied', 'applied', 'applied']) + expect(readDocPromotedWidgets(doc, String(host.id))).toMatchObject({ + promotedNames: ['text'] + }) + + minted = [] + host.widgets[0].value = 'pasted' + await afterFlush() + + expect(minted).toEqual([ + expect.objectContaining({ + op: 'set_widget', + widget: 'text', + promoted: expect.objectContaining({ value_index: 0 }) + }) + ]) + expect(refusals).toBe(0) + expect(applyMinted(doc, minted)).toEqual(['applied']) + }) + + it('keeps the add and skips a definition that cannot serialize, with its dependents', async () => { + const { inner } = handAddedPromotedHost() + vi.spyOn(inner, 'asSerialisable').mockImplementation(() => { + throw new Error('extension serializer failed') + }) + await afterFlush() + + expect(minted.map(({ op }) => op)).toEqual(['add_node']) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'agent_crdt_subgraph_definition_unserializable', + context: { definitionId: inner.id } + }) + ) + }) + + it('does not redefine a subgraph the document already holds', async () => { + const { inner, outer } = handAddedPromotedHost() + docDefinitionIds.add(inner.id) + await afterFlush() + + expect( + minted.map((op) => (op.op === 'define_subgraph' ? op.subgraph_id : op.op)) + ).toEqual(['add_node', outer.id]) + }) + it('does not let an ephemeral widget write roll a hand edit back with it', async () => { const { source } = seedGraph(graph) // Built the way useNodeProgressText builds it: `serialize: false` and diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index 5de358957b8..2cd4bbe294e 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -14,6 +14,7 @@ import type { NodeId as WireNodeId, PromotedHostWrite, + SubgraphDefinition, WidgetValuesForm, WorkflowNode } from '@comfyorg/comfy-multi-player' @@ -27,6 +28,7 @@ import type { LGraph } from '@/lib/litegraph/src/LGraph' import type { LGraphNode } from '@/lib/litegraph/src/LGraphNode' import type { LLink } from '@/lib/litegraph/src/LLink' import type { Subgraph } from '@/lib/litegraph/src/subgraph/Subgraph' +import type { SubgraphNode } from '@/lib/litegraph/src/subgraph/SubgraphNode' import type { ISerialisedNode } from '@/lib/litegraph/src/types/serialisation' import { isWidgetValue } from '@/lib/litegraph/src/types/widgets' import type { IBaseWidget } from '@/lib/litegraph/src/types/widgets' @@ -79,6 +81,8 @@ export interface DocOpMinterDeps { * sent the root host's `add_node` into the same document. */ mayWritePromotedHost(nodeId: NodeId): boolean + /** Whether the bound document holds a subgraph definition by this id. */ + docHasSubgraphDefinition(definitionId: string): boolean onWidgetWriteRefused(): void } @@ -426,6 +430,52 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { } } + function serialisedDefinition(subgraph: Subgraph): SubgraphDefinition | null { + try { + return JSON.parse(JSON.stringify(subgraph.asSerialisable())) + } catch (error) { + reportError(error, { + surface: 'agent', + errorType: 'agent_crdt_subgraph_definition_unserializable', + context: { definitionId: subgraph.id } + }) + return null + } + } + + function mintMissingDefinitions( + hosts: readonly SubgraphNode[] + ): GraphOperation[] { + const operations: GraphOperation[] = [] + const available = new Map() + const define = (subgraph: Subgraph): boolean => { + const settled = available.get(subgraph.id) + if (settled !== undefined) return settled + available.set(subgraph.id, false) + const dependenciesAvailable = subgraph.nodes + .filter((node) => node.isSubgraphNode()) + .map((node) => define(node.subgraph)) + .every(Boolean) + if (deps.docHasSubgraphDefinition(subgraph.id)) { + available.set(subgraph.id, true) + return true + } + const definition = dependenciesAvailable + ? serialisedDefinition(subgraph) + : null + if (!definition) return false + operations.push({ + op: 'define_subgraph', + subgraph_id: subgraph.id, + subgraph_definition: definition + }) + available.set(subgraph.id, true) + return true + } + for (const host of hosts) define(host.subgraph) + return operations + } + function flush(): void { flushScheduled = false reported.clear() @@ -433,13 +483,15 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { pending = [] pendingAdds.clear() if (detached) return + const hosts: SubgraphNode[] = [] const operations = batch.flatMap((entry) => { - const operation = - entry.kind === 'op' - ? entry.operation - : mintAddNodeOperation(entry.graph, entry.node) - return operation ? [operation] : [] + if (entry.kind === 'op') return [entry.operation] + const operation = mintAddNodeOperation(entry.graph, entry.node) + if (!operation) return [] + if (entry.node.isSubgraphNode()) hosts.push(entry.node) + return [operation] }) + operations.push(...mintMissingDefinitions(hosts)) if (operations.length > 0) deps.enqueue(operations) } diff --git a/src/workbench/extensions/agent/crdt/linkMintOutboundPositionCorruption.test.ts b/src/workbench/extensions/agent/crdt/linkMintOutboundPositionCorruption.test.ts index ac03403a68d..c337d2297ad 100644 --- a/src/workbench/extensions/agent/crdt/linkMintOutboundPositionCorruption.test.ts +++ b/src/workbench/extensions/agent/crdt/linkMintOutboundPositionCorruption.test.ts @@ -194,6 +194,7 @@ describe('agent CRDT outbound leg: link mint by live position vs. doc order', () docPromotedWidgets: (nodeId) => readDocPromotedWidgets(doc, String(nodeId)), mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {} }) onTestFinished(() => port.detach()) diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.changeTracker.test.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.changeTracker.test.ts index 119d428bec2..7d99f6b11be 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.changeTracker.test.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.changeTracker.test.ts @@ -366,6 +366,7 @@ describe('LiveGraphApplier with the real change tracker', () => { docInputNames: () => [], docPromotedWidgets: () => null, mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {}, enqueue }) @@ -617,6 +618,7 @@ describe('LiveGraphApplier with the real change tracker', () => { docInputNames: () => [], docPromotedWidgets: () => null, mayWritePromotedHost: () => true, + docHasSubgraphDefinition: () => true, onWidgetWriteRefused: () => {}, enqueue }) diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts index 0a104bb8531..e289e49ee32 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.humanAddTabSwitch.test.ts @@ -134,6 +134,7 @@ describe('a human-added node across a tab switch', () => { docInputNames: followerApi.docInputNames, docPromotedWidgets: followerApi.docPromotedWidgets, mayWritePromotedHost: followerApi.mayWritePromotedHost, + docHasSubgraphDefinition: followerApi.docHasSubgraphDefinition, onWidgetWriteRefused: followerApi.notifyWidgetWriteRefused }) const cleanup = () => { diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 62f7f16bf7c..8f496f0ec97 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -39,7 +39,10 @@ import type { GraphOperation } from './graphOperations' import type { ClassifiedDocUpdate } from './layoutFollowerBridge' import { LayoutFollowerBridge } from './layoutFollowerBridge' import type { LiveGraphApplierDeps } from './liveGraphApplier' -import { readDocPromotedWidgets } from './agentSubgraphDefinitions' +import { + readDocPromotedWidgets, + readSubgraphDefinitionIds +} from './agentSubgraphDefinitions' import { readDocSlotNames } from './liveGraphApplier' import { createOpCoalescer } from './opCoalescer' import { createOpSender } from './opSender' @@ -392,6 +395,8 @@ export function useAgentCrdtFollower( follower.value?.docPromotedWidgets(nodeId) ?? null, mayWritePromotedHost: (nodeId: NodeId) => follower.value?.mayWritePromotedHost(nodeId) ?? false, + docHasSubgraphDefinition: (definitionId: string) => + follower.value?.docHasSubgraphDefinition(definitionId) ?? false, notifyWidgetWriteRefused: () => follower.value?.notifyWidgetWriteRefused() } } @@ -1088,6 +1093,8 @@ function startAgentCrdtFollower( bridge.isSubscribeBaselineIntegrated || (sentAddsToCurrentDoc().has(String(nodeId)) && readDocPromotedWidgets(bridge.follower.doc, String(nodeId)) === null), + docHasSubgraphDefinition: (definitionId: string) => + readSubgraphDefinitionIds(bridge.follower.doc).includes(definitionId), notifyWidgetWriteRefused: () => rejectedOpNotifier.notifyWidgetWriteRefused() }