diff --git a/browser_tests/fixtures/agentInsertWorkflowOverlapFixture.ts b/browser_tests/fixtures/agentInsertWorkflowOverlapFixture.ts new file mode 100644 index 00000000000..e046f04b52b --- /dev/null +++ b/browser_tests/fixtures/agentInsertWorkflowOverlapFixture.ts @@ -0,0 +1,15 @@ +import { mergeTests } from '@playwright/test' + +import { agentTest } from '@e2e/fixtures/agentPanelFixture' +import { bindPopulatedTab } from '@e2e/fixtures/utils/agentInsertWorkflowOverlap' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const test = mergeTests(agentTest, webSocketFixture) + +export const insertOverlapTest = test.extend<{ + populatedTab: Awaited> +}>({ + populatedTab: async ({ page, getWebSocket }, use) => { + await use(await bindPopulatedTab(page, getWebSocket)) + } +}) diff --git a/browser_tests/fixtures/data/agentInsertWorkflowOverlap.ts b/browser_tests/fixtures/data/agentInsertWorkflowOverlap.ts new file mode 100644 index 00000000000..7950f662a79 --- /dev/null +++ b/browser_tests/fixtures/data/agentInsertWorkflowOverlap.ts @@ -0,0 +1,53 @@ +import type { Op, WidgetCatalog } from '@comfyorg/comfy-multi-player' + +export const WORKFLOW_ID = 'b3e7d9c1-0000-4000-8000-000000000020' +export const MESSAGE_ID = 'b3e7d9c1-0000-4000-8000-000000000021' +export const THREAD_ID = 'b3e7d9c1-0000-4000-8000-000000000022' + +function note( + id: number, + pos: [number, number], + size: [number, number], + text: string +) { + return { + id, + type: 'MarkdownNote', + pos, + size, + mode: 0, + order: 0, + flags: {}, + inputs: [], + outputs: [], + properties: {}, + widgets_values: [text] + } +} + +export const EXISTING_NODES = [ + note(1, [0, 180], [400, 300], 'Existing node A'), + note(2, [450, 180], [400, 300], 'Existing node B'), + note(3, [900, 180], [400, 300], 'Existing node C') +] + +export const Z_IMAGE_TEMPLATE_NODES = [ + note(35, [-420, 200], [510, 772], 'Template note'), + note(57, [130, 200], [400, 470], 'Template sampler'), + note(9, [570, 200], [780, 660], 'Template save') +] + +export const OVERLAP_CATALOG: WidgetCatalog = { + types: { MarkdownNote: { widget_order: ['text'] } } +} + +export const INSERT_TEMPLATE_OPS = [ + { + op_id: 'agent-insert-workflow-overlap-op-0'.padEnd(32, '0'), + actor: 'agent:e2e:turn-1', + base_version: 1, + stamp: [1, 'agent:e2e:turn-1'], + op: 'insert_workflow', + workflow: { nodes: Z_IMAGE_TEMPLATE_NODES, links: [] } + } +] satisfies Op[] diff --git a/browser_tests/fixtures/utils/agentInsertWorkflowOverlap.ts b/browser_tests/fixtures/utils/agentInsertWorkflowOverlap.ts new file mode 100644 index 00000000000..a83ca48eccc --- /dev/null +++ b/browser_tests/fixtures/utils/agentInsertWorkflowOverlap.ts @@ -0,0 +1,117 @@ +import type { Page, WebSocketRoute } from '@playwright/test' +import { expect } from '@playwright/test' + +import { + BLANK_WORKFLOW, + bootAgentApp, + loadIntoBootWorkflow, + mockAgentTurnApi, + mockWorkflowPersistence +} from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { + EXISTING_NODES, + INSERT_TEMPLATE_OPS, + MESSAGE_ID, + OVERLAP_CATALOG, + THREAD_ID, + WORKFLOW_ID, + Z_IMAGE_TEMPLATE_NODES +} from '@e2e/fixtures/data/agentInsertWorkflowOverlap' +import { subscribeAgentWorkflow } from '@e2e/fixtures/utils/subscribeAgentWorkflow' + +function readNodeRects(page: Page) { + return page.evaluate(() => + window.app!.graph.nodes.map((node) => ({ + id: String(node.id), + text: String(node.widgets?.[0]?.value ?? ''), + x: node.pos[0], + y: node.pos[1], + width: node.size[0], + height: node.size[1] + })) + ) +} + +type NodeRect = Awaited>[number] + +function overlaps(a: NodeRect, b: NodeRect): boolean { + return ( + a.x < b.x + b.width && + b.x < a.x + a.width && + a.y < b.y + b.height && + b.y < a.y + a.height + ) +} + +export async function bindPopulatedTab( + page: Page, + getWebSocket: () => Promise +) { + await page.setViewportSize({ width: 1920, height: 1280 }) + await bootAgentApp(page, true, { + beforeNavigate: async (page) => { + await mockAgentTurnApi(page, { + message_id: MESSAGE_ID, + thread_id: THREAD_ID, + workflow_id: WORKFLOW_ID + }) + await mockWorkflowPersistence(page, WORKFLOW_ID) + } + }) + await loadIntoBootWorkflow(page, BLANK_WORKFLOW) + + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const socket = await getWebSocket() + await subscribeAgentWorkflow({ + agentPanel, + socket, + workflowId: WORKFLOW_ID, + prompt: 'Insert the Z-Image template into this tab' + }) + + const host = new HostDoc( + WORKFLOW_ID, + { nodes: EXISTING_NODES, links: [] }, + OVERLAP_CATALOG + ) + for (const frame of host.initialSync()) socket.send(JSON.stringify(frame)) + await expect + .poll(() => page.evaluate(() => window.app!.graph.nodes.length)) + .toBe(EXISTING_NODES.length) + const existingIds = (await readNodeRects(page)).map((rect) => rect.id) + + return { + async insertTemplate() { + socket.send(JSON.stringify(host.apply(INSERT_TEMPLATE_OPS))) + await expect + .poll(() => page.evaluate(() => window.app!.graph.nodes.length)) + .toBe(EXISTING_NODES.length + Z_IMAGE_TEMPLATE_NODES.length) + }, + async readPlacement() { + const rects = await readNodeRects(page) + const existing = rects.filter((rect) => existingIds.includes(rect.id)) + const inserted = rects.filter((rect) => !existingIds.includes(rect.id)) + const anchor = inserted.find((rect) => rect.text === 'Template note') + return { + overlaps: inserted.flatMap((node) => + existing + .filter((other) => overlaps(node, other)) + .map( + (other) => + `inserted "${node.text}" at (${node.x}, ${node.y}) overlaps ` + + `existing "${other.text}" at (${other.x}, ${other.y})` + ) + ), + layoutFromNote: Object.fromEntries( + inserted.map((node) => [ + node.text, + anchor ? [node.x - anchor.x, node.y - anchor.y] : null + ]) + ) + } + } + } +} diff --git a/browser_tests/fixtures/utils/subscribeAgentWorkflow.ts b/browser_tests/fixtures/utils/subscribeAgentWorkflow.ts new file mode 100644 index 00000000000..c3d7e5351c5 --- /dev/null +++ b/browser_tests/fixtures/utils/subscribeAgentWorkflow.ts @@ -0,0 +1,31 @@ +import { expect } from '@playwright/test' +import type { WebSocketRoute } from '@playwright/test' + +import type { AgentPanel } from '@e2e/fixtures/components/AgentPanel' + +interface SubscribeAgentWorkflowInput { + agentPanel: AgentPanel + socket: WebSocketRoute + workflowId: string + prompt: string +} + +export async function subscribeAgentWorkflow({ + agentPanel, + socket, + workflowId, + prompt +}: SubscribeAgentWorkflowInput): Promise { + const outboundFrames: string[] = [] + socket.onMessage((message) => outboundFrames.push(String(message))) + await agentPanel.selectWorkflow() + await agentPanel.sendMessage(prompt) + await expect + .poll(() => outboundFrames, { timeout: 15_000 }) + .toContainEqual( + expect.stringMatching( + new RegExp(`doc_subscribe.*${workflowId}|${workflowId}.*doc_subscribe`) + ) + ) + return outboundFrames +} diff --git a/browser_tests/tests/agent/agentDuplicateInsertOpaqueWidgetWrite.spec.ts b/browser_tests/tests/agent/agentDuplicateInsertOpaqueWidgetWrite.spec.ts index ec6e50c530f..28764f4b218 100644 --- a/browser_tests/tests/agent/agentDuplicateInsertOpaqueWidgetWrite.spec.ts +++ b/browser_tests/tests/agent/agentDuplicateInsertOpaqueWidgetWrite.spec.ts @@ -20,38 +20,6 @@ import { webSocketFixture } from '@e2e/fixtures/ws' const test = mergeTests(agentTest, webSocketFixture) -/** - * Repro for the Memie Osuga / stagingcloud in-app-agent report: asking the - * agent for "a z-turbo image workflow" made it call `insert_workflow` TWICE - * with two different op ids, landing two perfectly-overlapping copies of the - * same template (confirmed via the report's own CRDT debug log: two disjoint - * `insert::root:node:` namespaces sharing the same three local - * node ids). Duplicate-insert-with-no-dedup is intentional frontend behavior - * (`agentInsertWorkflow.templates.test.ts`'s "the same template inserted - * twice produces two independent, fully-wired pipelines"), so this repro - * does not treat that half as a defect -- it documents the VISUAL overlap - * that duplication produces on a real canvas, which no test asserted before, - * and then the FRONTEND-observable half of the actual complaint: a widget - * write the doc host rejects with an `opaque_widgets`-shaped failure - * (` is absent from the pinned catalog, so its widgets_values is - * stored opaquely (schema §1.2) and is not name-addressable`). The follower - * now puts the refused register back from the document - * (`AgentCrdtProjection.revertRejected`), so the widget no longer keeps - * showing a value the shared document never took. The rejection notifier - * also tells the human that the write was refused. - * - * PM-1716 made that divergence VISIBLE rather than silent: `rejectedOpNotice` - * matches the host's `code` against the applier's rejection vocabulary and - * raises a toast. The divergence itself is unchanged and is still pinned - * below -- a rejected write is not rolled back on the canvas. - * - * `opaque_widgets` is a cloud doc-host error code the frontend never - * produces, so this repro injects the host's raw wire response directly - * rather than trying to make the real `comfy-multi-player` applier produce - * it -- exactly what a real doc host running this rejection logic would put - * on the wire. - */ - const WORKFLOW_ID = 'b3f1c4a2-0000-4000-8000-000000000030' const MESSAGE_ID = 'b3f1c4a2-0000-4000-8000-000000000031' const THREAD_ID = 'b3f1c4a2-0000-4000-8000-000000000032' @@ -193,9 +161,9 @@ function insertWorkflowOp( ): Extract { return { op_id: opId.padEnd(32, '0'), - actor: 'agent:e2e', + actor: 'agent:e2e:turn-1', base_version: 1, - stamp: [1, 'agent:e2e'], + stamp: [1, 'agent:e2e:turn-1'], op: 'insert_workflow', workflow: templateWorkflow() } @@ -235,12 +203,6 @@ async function ksamplerNodeIds(page: Page): Promise { ) } -/** - * Boots the agent-enabled app, sends one turn, and lands two `insert_workflow` - * ops with different op ids -- each carrying its own fresh copy of the same - * three-node template, exactly like the two disjoint `insert::...` - * namespaces in the report's CRDT debug log. - */ async function driveThroughDuplicateInsert( page: Page, getWebSocket: () => Promise @@ -250,7 +212,6 @@ async function driveThroughDuplicateInsert( localStorage.setItem('Comfy.Agent.CrdtFollower', 'true') }) await bootAgentApp(page, true, { - settings: { 'Comfy.VueNodes.Enabled': true }, objectInfo: nodeDefs, beforeNavigate: async (page) => { await mockAgentTurnApi(page, { @@ -288,14 +249,10 @@ async function driveThroughDuplicateInsert( .poll(() => page.evaluate(() => window.app!.graph.nodes.length)) .toBe(0) - // First `insert_workflow`: the agent's own first (unnecessary) tool call. socket.send(JSON.stringify(applyInsertWorkflow(host, 'insert-zturbo-op-a'))) await expect.poll(() => ksamplerNodeIds(page)).toHaveLength(1) const [copyANodeId] = await ksamplerNodeIds(page) - // Second `insert_workflow`, a DIFFERENT op id, the SAME template -- the - // duplicate call the report's debug log caught (two disjoint - // `insert::root:node:` namespaces holding the same 3 node ids). socket.send(JSON.stringify(applyInsertWorkflow(host, 'insert-zturbo-op-b'))) await expect.poll(() => ksamplerNodeIds(page)).toHaveLength(2) const copyBNodeId = (await ksamplerNodeIds(page)).find( @@ -316,14 +273,6 @@ interface RejectedWidgetEditHandles extends DuplicateInsertHandles { seedInput: Locator } -/** - * Extends {@link driveThroughDuplicateInsert} with a human edit of copy B's - * `seed` widget, then answers the client's `doc_ops` batch with a raw - * `doc_ops_result` the way a real doc host would for a node whose class is - * absent from its pinned catalog -- `ok: false`, a `failed` entry naming the - * rejected op, and (per the applier's contract) no accompanying `doc_update`, - * since nothing landed. - */ async function driveThroughRejectedWidgetEdit( page: Page, getWebSocket: () => Promise @@ -331,22 +280,6 @@ async function driveThroughRejectedWidgetEdit( const handles = await driveThroughDuplicateInsert(page, getWebSocket) const { vueNodes, socket, outboundFrames, copyBNodeId } = handles - // The two copies sit at IDENTICAL canvas coordinates (the overlap under - // test in the other spec), which a real human could not click through - // either -- move copy B's node clear of copy A's before driving a normal, - // pointer-based edit, so this edit deterministically lands on the specific - // node id under test rather than whichever copy happens to render on top. - await page.evaluate( - ({ nodeId, dy }) => { - const node = window.app!.graph.nodes.find( - (candidate) => String(candidate.id) === nodeId - ) - if (!node) throw new Error(`node ${nodeId} not found`) - node.pos = [node.pos[0], node.pos[1] + dy] - window.app!.graph.setDirtyCanvas(true, true) - }, - { nodeId: copyBNodeId, dy: 600 } - ) await page.getByRole('button', { name: 'Fit View (.)', exact: true }).click() const seedInput = vueNodes .getNodeLocator(copyBNodeId) @@ -392,10 +325,6 @@ async function driveThroughRejectedWidgetEdit( }) ) - // Give the browser's own event loop a couple of turns to run the - // `doc_ops_result` listener the injected WS message wakes -- it is a - // synchronous handler (`useAgentCrdtFollower.ts`'s `onOpsResult`), so by - // the time these resolve it has already run, without an arbitrary sleep. await nextFrame(page) await nextFrame(page) @@ -404,12 +333,14 @@ async function driveThroughRejectedWidgetEdit( test.describe( 'Agent duplicate insert_workflow + opaque widget-write rejection', - { tag: ['@cloud', '@agent', '@canvas', '@node', '@widget'] }, + { + tag: ['@cloud', '@agent', '@canvas', '@node', '@widget', '@vue-nodes'] + }, () => { test.use({ connectWebSocketToServer: false }) test.describe.configure({ timeout: 60_000 }) - test('two insert_workflow calls with different op ids leave two fully overlapping, visually indistinguishable copies of the template', async ({ + test('two insert_workflow calls with different op ids leave two copies of the template side by side', async ({ page, getWebSocket }) => { @@ -435,10 +366,6 @@ test.describe( .getByRole('button', { name: 'Fit View (.)', exact: true }) .click() - // `Fit View` pans/zooms with a brief transition; wait for copy A's box - // to stop moving (two consecutive identical reads) before comparing it - // to copy B's, or the comparison below races the animation instead of - // measuring the settled overlap. let previousBoxA: Awaited> = null await expect .poll(async () => { @@ -452,13 +379,13 @@ test.describe( }) .toBe(true) - // Nobody previously asserted the resulting VISUAL overlap: the same - // template inserted twice renders both copies at pixel-identical - // positions, so a human cannot tell them apart on canvas. const boxA = await vueNodes.getNodeLocator(copyANodeId).boundingBox() const boxB = await vueNodes.getNodeLocator(copyBNodeId).boundingBox() expect(boxA).not.toBeNull() - expect(boxB).toEqual(boxA) + expect(boxB).not.toBeNull() + expect(boxB!.width).toBeCloseTo(boxA!.width, 0) + expect(boxB!.y).toBeCloseTo(boxA!.y, 0) + expect(boxB!.x).toBeGreaterThan(boxA!.x + boxA!.width) }) test('a rejected widget edit is reported and rolled back to the value the shared document kept', async ({ diff --git a/browser_tests/tests/agent/agentInsertWorkflowOverlap.spec.ts b/browser_tests/tests/agent/agentInsertWorkflowOverlap.spec.ts new file mode 100644 index 00000000000..930b2593b55 --- /dev/null +++ b/browser_tests/tests/agent/agentInsertWorkflowOverlap.spec.ts @@ -0,0 +1,32 @@ +import { expect } from '@playwright/test' + +import { insertOverlapTest as test } from '@e2e/fixtures/agentInsertWorkflowOverlapFixture' + +test.describe( + 'Agent insert_workflow into a populated tab (PM-2232)', + { + tag: ['@cloud', '@agent', '@canvas', '@vue-nodes'], + annotation: { + type: 'issue', + description: + 'https://linear.app/comfyorg/issue/PM-2232/workflow-inserted-into-an-existing-tab-lands-on-top-of-the-existing' + } + }, + () => { + test.use({ connectWebSocketToServer: false }) + + test('places an inserted template clear of the nodes already on the canvas', async ({ + populatedTab + }) => { + await populatedTab.insertTemplate() + + const placement = await populatedTab.readPlacement() + expect(placement.overlaps).toEqual([]) + expect(placement.layoutFromNote).toEqual({ + 'Template note': [0, 0], + 'Template sampler': [550, 0], + 'Template save': [990, 0] + }) + }) + } +) diff --git a/docs/adr/CRDT-PLACEMENT-0035-agent-inserted-nodes-land-near-existing-content.md b/docs/adr/CRDT-PLACEMENT-0035-agent-inserted-nodes-land-near-existing-content.md index 035a514ce60..71ce52e5f46 100644 --- a/docs/adr/CRDT-PLACEMENT-0035-agent-inserted-nodes-land-near-existing-content.md +++ b/docs/adr/CRDT-PLACEMENT-0035-agent-inserted-nodes-land-near-existing-content.md @@ -13,6 +13,12 @@ from `graphMutations`' prepare/commit batch and its `SemanticPlacementPort` to `LiveGraphApplier.#placeBatch`, which runs after each frame's nodes are created on the live graph. +Amended 2026-10-21 by +[PM-2232](https://linear.app/comfyorg/issue/PM-2232/workflow-inserted-into-an-existing-tab-lands-on-top-of-the-existing): +in a live frame written by an agent turn, a batch with a node that lands on an +existing node now also moves, by the same offset, whatever the viewport shows. +See "Trigger". + ## Context The in-app agent inserts nodes through the CRDT follower: the server emits @@ -82,10 +88,14 @@ remote actor and nothing mints it back. Eligibility is "created by this pass". A node the frame only updated, or recreated because its document `type` changed, is not moved. Applying the changes collected while a tab was inactive (`applyCollected`) uses the same -`#placeBatch` for the nodes it creates: a client that returns to a tab and -finds nodes inserted while it was away places them exactly as it would have -had it received them live. A fresh client replaying the whole document onto an -empty canvas has no existing content and receives no offset. +`#placeBatch` for the nodes it creates, with the lost rule only: the collected +changes do not record which writer made them, so the collision rule (see +"Trigger"), which needs a live agent-turn writer, does not run there. A client +that returns to a tab and finds far, offscreen nodes inserted while it was away +places them exactly as it would have had it received them live; an insert that +landed on existing nodes while the tab was inactive keeps its overlap (see +"Negative"). A fresh client replaying the whole document onto an empty canvas +has no existing content and receives no offset. ### Batch granularity @@ -108,17 +118,33 @@ also owns not splitting. ### Trigger -Reposition only when all of the following hold: - -1. The scope already contains at least one node before the batch. An empty - canvas keeps the template's own layout, matching the human template flow. -2. The incoming batch's bounding box is disjoint from the existing graph's - bounding box inflated by `DISCONNECTED_GAP_PX` (600). Content placed near, - overlapping, or interleaved with existing content is never touched. -3. The incoming bounding box does not intersect the current viewport, when a +Nothing moves unless the scope already contains at least one node before the +batch. An empty canvas keeps the template's own layout, matching the human +template flow. Otherwise the batch moves in either of two cases. + +**Collision.** The frame is a live write by an agent turn (its `actor` is +`agent:…` and it is not a subscribe catch-up), and any incoming node's rect +intersects any existing node's rect. A template inserted into a populated tab +arrives at its baked-in coordinates, which commonly sit on top of the user's +graph (PM-2232: the Z-Image template's nodes span x −420 to 1350, the same +region as the default workflow). The agent does not overlap nodes on purpose, +so the viewport does not veto this case. A batch that sits inside or near the +existing hull without touching any node is not a collision and is left alone. + +Catch-up frames, changes applied on tab return (`applyCollected`) and human +writes never move for a collision: a replay can recreate a node a human +deliberately pasted over another, and moving it would undo the human's +placement. + +**Lost.** All of the following hold: + +1. The incoming batch's bounding box is disjoint from the existing graph's + bounding box inflated by `DISCONNECTED_GAP_PX` (600). Content placed near + or interleaved with existing content is not lost. +2. The incoming bounding box does not intersect the current viewport, when a viewport is known for this scope (see `viewportBounds` above). Content the user can already see is not "lost", so it stays where the writer put it. An - unavailable viewport does not veto: condition 2 alone decides. + unavailable viewport does not veto: condition 1 alone decides. The anchor is existing content, not the user's current viewport — an explicit choice. Landing beside the nodes the batch relates to keeps the association @@ -127,20 +153,24 @@ have parked the viewport could drop a template into empty space unrelated to anything, and requires a viewport that background-workflow applies do not have. -The rule applies to every agent `add_node` batch, single- or multi-node. The -geometric predicate — disjoint beyond the gap _and_ offscreen — is the actual -discriminator between "lost" and "deliberate"; node count is only a proxy for -it, and a single far offscreen node is exactly as lost as forty. +The rule applies to every agent batch, single- or multi-node, whether it +arrived as `add_node` ops or as one `insert_workflow`. The geometric +predicates — a node-on-node collision, or disjoint beyond the gap _and_ +offscreen — are the actual discriminators; node count is only a proxy for +them, and a single far offscreen node is exactly as lost as forty. ### Offset One translation for the whole batch: move the batch's bounding box so its left edge sits at the existing bounding box's right edge plus `PLACEMENT_GUTTER_PX` (80), top edges aligned. The batch's internal relative -layout is preserved exactly — never per-node moves, no collision search, no -grid snapping. Both constants are conservative product heuristics, chosen so -that only unambiguously lost content moves; they are named constants in -`batchPlacement.ts` and trivially tunable. +layout is preserved exactly — never per-node moves, no collision search (the +batch moves by one fixed offset rather than hunting for a free spot), no grid +snapping. Both constants are conservative product heuristics. The gap is +chosen so that the lost rule moves only unambiguously lost content; the +collision rule moves content that is not lost, only buried under existing +nodes, by the same offset. They are named constants in `batchPlacement.ts` and +trivially tunable. ### What is deliberately not done @@ -199,9 +229,19 @@ change, and no protocol change. The doc keeps the server's coordinates. moment, so a client that receives an insert live and one that receives it on tab return can offset the same batch differently if their canvases differed. Accepted: post-insert geometry is already local-first. -- The gap and gutter values are heuristics. Chosen conservative so only - clearly-lost content moves; a legitimate placement between 0 and 600 px of - gap is never touched, and anything visible on screen is never touched. +- The gap and gutter values are heuristics. The gap is chosen conservative so + the lost rule moves only clearly-lost content; a legitimate placement + between 0 and 600 px of gap is never touched, and anything visible on screen + is touched only when a live agent turn lands it on an existing node. +- An agent that deliberately places a new node on top of an existing one, for + example to stand in for a node it deletes in a later frame, sees the new + node moved beside the graph instead. A delete and add in the same frame is + unaffected, because the deleted node is gone before placement runs. +- An agent insert that lands while its tab is inactive is applied on tab + return without collision placement, so it can still overlap. The collected + changes do not record which writer made them. +- Collision uses node body rects (`pos`/`size`), not title bars, so two + nodes that touch only through a title bar are not a collision. - Existing-node bounds are the live nodes' `pos`/`size`, which for a node created earlier in the same frame is its configured size before first render (see the measurement amendment in diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts index 6ddf0040b14..339bd00e3bf 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.integration.test.ts @@ -13,6 +13,7 @@ import { toNodeId } from '@/types/nodeId' import { AgentCrdtProjection } from './agentCrdtProjection' import { FollowerDoc } from './followerDoc' +import type { ClassifiedDocUpdate } from './layoutFollowerBridge' class TestSource extends LGraphNode { static override title = 'Test Source' @@ -45,6 +46,9 @@ const CATALOG: WidgetCatalog = { types: { TestSource: { widget_order: ['steps', 'seed'] } } } +type FrameClassification = Pick & + Partial> + function toWorkflowJson({ nodes, ...rest }: ISerialisedGraph): WorkflowJSON { return { ...rest, @@ -112,35 +116,38 @@ function snapshot(graph: LGraph) { } } -/** - * Binds a fresh follower session to the doc minted from the graph's own - * save and delivers the whole doc as the catch-up frame followed by the - * full sync, the way a return to a previously bound workflow tab does. - * Later host edits arrive as incremental frames only. - */ function bindAndCatchUp(graph: LGraph, saved: ISerialisedGraph) { const host = mint(toWorkflowJson(saved), CATALOG) const follower = new FollowerDoc() const projection = new AgentCrdtProjection(() => graph) projection.bind(WORKFLOW_ID, follower) let seq = 0 - const deliver = (update: Uint8Array) => { + const deliver = (update: Uint8Array, classification: FrameClassification) => { follower.applyRemoteUpdate(update) expect( projection.applyFrame({ workflowId: WORKFLOW_ID, seq: ++seq, update, - actor: 'agent:comfy:host', - opIds: [] + opIds: [], + ...classification }) ).toMatchObject({ applied: true }) } - deliver(Y.encodeStateAsUpdate(host)) - const hostEdit = (edit: () => void) => { + deliver(Y.encodeStateAsUpdate(host), { + actor: 'agent:comfy:host', + catchUp: true + }) + const hostEdit = ( + edit: () => void, + classification: FrameClassification = { + actor: 'agent:comfy:host', + catchUp: false + } + ) => { const before = Y.encodeStateVector(host) host.transact(edit) - deliver(Y.encodeStateAsUpdate(host, before)) + deliver(Y.encodeStateAsUpdate(host, before), classification) } const destroy = () => { projection.destroy() @@ -156,6 +163,60 @@ beforeEach(() => { }) describe('AgentCrdtProjection catch-up over a live graph', () => { + it.for([ + { + name: 'live agent frame', + actor: 'agent:test:turn-1', + catchUp: false, + expected: [680, 20] + }, + { + name: 'agent catch-up frame', + actor: 'agent:test:turn-1', + catchUp: true, + expected: [20, 30] + }, + { + name: 'live human frame', + actor: 'human:test:tab-1', + catchUp: false, + expected: [20, 30] + }, + { + name: 'unclassified agent frame', + actor: 'agent:test:turn-1', + catchUp: undefined, + expected: [20, 30] + } + ])( + 'classifies an overlapping insertion from a $name', + ({ actor, catchUp, expected }) => { + const { graph, source, note } = buildLiveGraph() + source.setSize([200, 100]) + note.setSize([200, 100]) + const { host, hostEdit, destroy } = bindAndCatchUp( + graph, + structuredClone(graph.serialize()) + ) + onTestFinished(destroy) + const inserted = { + ...structuredClone(note.serialize()), + id: 3, + pos: [20, 30] as [number, number], + title: 'Inserted note' + } + + hostEdit( + () => { + nodesMap(host).set('3', new Y.Map(Object.entries(inserted))) + }, + { actor, catchUp } + ) + + expect([...(graph.getNodeById(toNodeId(3))?.pos ?? [])]).toEqual(expected) + } + ) + it('leaves a saved workflow displayed as saved after the catch-up frame re-reconciles it', () => { const { graph, source, note } = buildLiveGraph() const saved = structuredClone(graph.serialize()) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts index af8b4a94f70..ef2a5c036a7 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtProjection.ts @@ -8,6 +8,8 @@ import { DocChangeCollector } from './docChangeCollector' import type { DocUpdate } from './docFrameClient' import type { FollowerDoc } from './followerDoc' import type { GraphOperation } from './graphOperations' +import type { ClassifiedDocUpdate } from './layoutFollowerBridge' +import { isLiveAgentUpdate } from './layoutFollowerBridge' import { LiveGraphApplier, readDocWidgetValue } from './liveGraphApplier' import type { ApplyMode, @@ -134,7 +136,9 @@ export class AgentCrdtProjection { * Applies one delivered frame's changes to the live graph. Without a graph * the changes stay collected for `applyCollected` once one appears. */ - applyFrame(update: DocUpdate): FrameOutcome { + applyFrame( + update: DocUpdate & Partial> + ): FrameOutcome { const target = this.targets.get(update.workflowId) if (!target) return { applied: false, nodes: EMPTY_DELTA } if (!this.getGraph()) { @@ -149,7 +153,8 @@ export class AgentCrdtProjection { changes, { actor: update.actor ?? 'agent-remote', - opIds: update.opIds?.filter((id) => id.length > 0) ?? [] + opIds: update.opIds?.filter((id) => id.length > 0) ?? [], + liveAgentTurn: isLiveAgentUpdate(update) }, this.takeApplyMode(update.workflowId) ) diff --git a/src/workbench/extensions/agent/crdt/batchPlacement.test.ts b/src/workbench/extensions/agent/crdt/batchPlacement.test.ts index b8c8b111c5e..f14cce83033 100644 --- a/src/workbench/extensions/agent/crdt/batchPlacement.test.ts +++ b/src/workbench/extensions/agent/crdt/batchPlacement.test.ts @@ -69,6 +69,48 @@ describe('placementOffset', () => { ).toEqual({ dx: 200 + PLACEMENT_GUTTER_PX - 9000, dy: -9000 }) }) + it.for([ + { name: 'no viewport known', viewport: null }, + { + name: 'the collision visible on screen', + viewport: rect(0, 0, 2000, 1500) + } + ])( + 'moves a batch that lands on an existing node beside existing content, $name', + ({ viewport }) => { + expect( + placementOffset({ + existing: [rect(0, 0), rect(300, 0)], + viewport, + incoming: [rect(-400, 50), rect(350, 60), rect(1000, 400)], + moveCollisions: true + }) + ).toEqual({ dx: 980, dy: -50 }) + } + ) + + it.for([ + { + name: 'inside the existing hull without touching a node', + existing: [rect(0, 0), rect(600, 400)], + incoming: [rect(300, 150)] + }, + { + name: 'touching an existing node at its edge', + existing: [rect(0, 0)], + incoming: [rect(200, 0)] + } + ])('leaves a batch $name', ({ existing, incoming }) => { + expect( + placementOffset({ + existing, + viewport: null, + incoming, + moveCollisions: true + }) + ).toBeNull() + }) + it('fires at exactly the gap threshold', () => { expect( placementOffset({ diff --git a/src/workbench/extensions/agent/crdt/batchPlacement.ts b/src/workbench/extensions/agent/crdt/batchPlacement.ts index 206833323ae..2d017b3ff58 100644 --- a/src/workbench/extensions/agent/crdt/batchPlacement.ts +++ b/src/workbench/extensions/agent/crdt/batchPlacement.ts @@ -14,6 +14,7 @@ interface PlacementInput { existing: readonly PlacementRect[] viewport: PlacementRect | null incoming: readonly PlacementRect[] + moveCollisions?: boolean } export const DISCONNECTED_GAP_PX = 600 @@ -50,15 +51,28 @@ function intersects(a: Bounds, b: Bounds): boolean { ) } -/** - * One shared translation that moves a disconnected, offscreen batch of newly - * inserted nodes beside the existing content's hull, or null when the batch - * should keep the coordinates it arrived with. See ADR-CRDT-PLACEMENT-0035. - */ +function collides( + existing: readonly PlacementRect[], + incoming: readonly PlacementRect[] +): boolean { + const existingBounds = existing.map(toBounds) + return incoming.some((rect) => { + const bounds = toBounds(rect) + return existingBounds.some((other) => intersects(bounds, other)) + }) +} + +/** Returns a batch translation, or null when its coordinates stay unchanged. */ export function placementOffset(input: PlacementInput): PlacementOffset | null { if (input.existing.length === 0 || input.incoming.length === 0) return null const anchor = hull(input.existing) const batch = hull(input.incoming) + const offset = { + dx: anchor.right + PLACEMENT_GUTTER_PX - batch.left, + dy: anchor.top - batch.top + } + if (input.moveCollisions && collides(input.existing, input.incoming)) + return offset const inflated = { left: anchor.left - DISCONNECTED_GAP_PX, top: anchor.top - DISCONNECTED_GAP_PX, @@ -67,8 +81,5 @@ export function placementOffset(input: PlacementInput): PlacementOffset | null { } if (intersects(inflated, batch)) return null if (input.viewport && intersects(toBounds(input.viewport), batch)) return null - return { - dx: anchor.right + PLACEMENT_GUTTER_PX - batch.left, - dy: anchor.top - batch.top - } + return offset } diff --git a/src/workbench/extensions/agent/crdt/layoutFollowerBridge.ts b/src/workbench/extensions/agent/crdt/layoutFollowerBridge.ts index d56860a41e3..83f72f86a07 100644 --- a/src/workbench/extensions/agent/crdt/layoutFollowerBridge.ts +++ b/src/workbench/extensions/agent/crdt/layoutFollowerBridge.ts @@ -19,6 +19,13 @@ export interface ClassifiedDocUpdate extends DocUpdate { catchUp: boolean } +export function isLiveAgentUpdate( + update: Pick & + Partial> +): boolean { + return update.catchUp === false && update.actor?.startsWith('agent:') === true +} + /** * Outbound frames are advisory: the follower's correctness never depends on one * arriving. A transport that cannot carry a frame reports `false`; one that diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts index f4cb238603b..cfe010bc898 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.test.ts @@ -27,7 +27,10 @@ import type { WidgetValue } from '@/types/simplifiedWidget' import { followedDoc } from './__fixtures__/followedDoc' import { LiveGraphApplier } from './liveGraphApplier' -import type { LiveGraphApplierDeps } from './liveGraphApplier' +import type { + LiveGraphApplierDeps, + RemoteApplyContext +} from './liveGraphApplier' import { changesForRejectedOps } from './rejectedOpChanges' vi.mock(import('@/platform/telemetry/reportError')) @@ -1658,6 +1661,32 @@ describe('LiveGraphApplier', () => { expect(positionOf(3)).toEqual([690, 0]) }) + it('moves a created batch off the nodes it lands on only for a live agent turn', () => { + const { graph, doc, collector, applier, applyCollected } = setup({ + nodes: [sourceNode(1)], + links: [] + }) + applyCollected() + const addOnTopOfNode1 = (id: number, context: RemoteApplyContext) => { + doc.transact(() => + nodesMap(doc).set( + String(id), + new Y.Map(Object.entries(sinkNode(id, [50, 20]))) + ) + ) + applier.applyChanges(doc, collector.take(), context) + } + const positionOf = (id: number) => [ + ...(graph.getNodeById(toNodeId(id))?.pos ?? []) + ] + + addOnTopOfNode1(2, CONTEXT) + addOnTopOfNode1(3, { ...CONTEXT, liveAgentTurn: true }) + + expect(positionOf(2)).toEqual([50, 20]) + expect(positionOf(3)).toEqual([330, 0]) + }) + it('scopes every write to the remote actor', () => { const actors: string[] = [] const { doc, collector, applier, applyCollected } = setup( diff --git a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts index 764cad48c2c..e8c43ada532 100644 --- a/src/workbench/extensions/agent/crdt/liveGraphApplier.ts +++ b/src/workbench/extensions/agent/crdt/liveGraphApplier.ts @@ -67,6 +67,7 @@ export interface FrameChanges { export interface RemoteApplyContext { actor: string opIds: readonly string[] + liveAgentTurn?: boolean } export interface LiveGraphApplierDeps { @@ -638,7 +639,10 @@ export class LiveGraphApplier { } this.applyLinks(graph, doc, [...changes.links], context) - if (mode === 'merge') this.placeBatch(graph, created) + if (mode === 'merge') + this.placeBatch(graph, created, { + moveCollisions: context.liveAgentTurn === true + }) return { createdNodeIds: created, failureCount: this.currentFailureCount() @@ -1240,7 +1244,11 @@ export class LiveGraphApplier { return created ? 'connected' : 'refused' } - private placeBatch(graph: LGraph, created: readonly NodeId[]): void { + private placeBatch( + graph: LGraph, + created: readonly NodeId[], + { moveCollisions }: { moveCollisions: boolean } + ): void { if (created.length === 0) return const createdSet = new Set(created) const rect = (node: LGraphNode): PlacementRect => ({ @@ -1255,7 +1263,8 @@ export class LiveGraphApplier { .filter((node) => !createdSet.has(node.id)) .map(rect), viewport: this.deps.viewportBounds?.() ?? null, - incoming: incoming.map(rect) + incoming: incoming.map(rect), + moveCollisions }) if (!offset) return for (const node of incoming) { diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 5e59cdba197..e12d0698603 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -26,9 +26,10 @@ import { useAgentPanelStore } from '@/workbench/extensions/agent/stores/agent/ag import type { DocNodeDelta, FrameOutcome } from './agentCrdtProjection' import type { DocFrameTransport } from './docFrameClient' import type { GraphOperation } from './graphOperations' +import { LayoutFollowerBridge } from './layoutFollowerBridge' import type { BatchOutcome, OpSenderDeps } from './opSender' -const bridgeState = vi.hoisted(() => { +const bridgeState = (() => { class FakeBridge extends EventTarget { subscribe = vi.fn() unsubscribe = vi.fn() @@ -52,7 +53,7 @@ const bridgeState = vi.hoisted(() => { FakeBridge, current: null as InstanceType | null } -}) +})() const clientState = vi.hoisted(() => ({ destroy: vi.fn(), @@ -116,15 +117,7 @@ const apiState = vi.hoisted(() => { } }) -vi.mock(import('./layoutFollowerBridge'), () => ({ - LayoutFollowerBridge: class { - constructor() { - const bridge = new bridgeState.FakeBridge() - bridgeState.current = bridge - return bridge - } - } -})) +vi.mock(import('./layoutFollowerBridge'), { spy: true }) vi.mock(import('./docFrameClient'), () => ({ DocFrameClient: class { @@ -259,6 +252,11 @@ describe('useAgentCrdtFollower', () => { useAgentPanelStore().enabled = true sessionStorage.clear() bridgeState.current = null + vi.mocked(LayoutFollowerBridge).mockImplementation(function () { + const bridge = new bridgeState.FakeBridge() + bridgeState.current = bridge + return fromPartial(bridge) + }) clientState.transport = null projectionState.applyFrame .mockReset() diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 8f496f0ec97..b2fac05f20c 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -37,7 +37,7 @@ import { DocFrameClient } from './docFrameClient' import { RESEED_CONFLICT, isRetryableReseedCode } from './docFrameCodes' import type { GraphOperation } from './graphOperations' import type { ClassifiedDocUpdate } from './layoutFollowerBridge' -import { LayoutFollowerBridge } from './layoutFollowerBridge' +import { LayoutFollowerBridge, isLiveAgentUpdate } from './layoutFollowerBridge' import type { LiveGraphApplierDeps } from './liveGraphApplier' import { readDocPromotedWidgets, @@ -194,9 +194,8 @@ function notifyAgentMaterialization( const nodeId = parseNodeId(id) if (nodeId) pendingLiveNodeIds.delete(nodeId) } - const isLiveAgentUpdate = - !update.catchUp && update.actor?.startsWith('agent:') === true - if (isLiveAgentUpdate) { + const liveAgentUpdate = isLiveAgentUpdate(update) + if (liveAgentUpdate) { for (const nodeId of materialized) pendingLiveNodeIds.add(nodeId) for (const id of nodes.added) { const nodeId = parseNodeId(id) @@ -209,7 +208,7 @@ function notifyAgentMaterialization( ]) emitPendingMaterializations( update.workflowId, - isLiveAgentUpdate ? update.actor : undefined, + liveAgentUpdate ? update.actor : undefined, available, pendingLiveNodeIds, events