diff --git a/browser_tests/README.md b/browser_tests/README.md index 612a16d705a..83f67ba52be 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -657,6 +657,50 @@ test.afterEach(async ({ comfyPage }) => { `debugShowCanvasOverlay()`, `debugGetCanvasDataURL()` are for local debugging only. Never commit them. +## Replay coverage for agent bug fixes + +### Playbook + +```bash +DISTRIBUTION=cloud DEV_SERVER_COMFYUI_URL=http://127.0.0.1:8188 pnpm dev +``` + +```bash +PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 DISTRIBUTION=cloud pnpm exec playwright test agentConversation --project=cloud +``` + +Watch one: add `--headed -g `. Recorded gaps: `AGENT_REPLAY_TIMING=recorded`. + +When a fix changes how the agent's turns affect the app (graph edits, +CRDT frames, panel state), add a conversation replay case alongside the +fix so the bug stays fixed: + +1. **Capture the conversation.** Reproduce the bug's turn against the + cloud agent run locally in its non-standalone mode, with Postgres, + Redis and the doc host beside it. In that mode the agent writes the + per-op audit rows (`agent_tool_calls` parent and child rows) to + Postgres, which is what the exporter reads; the doc host is a separate + required service and writes none of them. The same agent in + standalone mode (SQLite, no doc host) never writes them, so it can + only yield text-only or tool-error turns. Record it with the recorder + (`scripts/agentConversationRecord.ts`, landing in #16782; until it merges, + record from that PR's branch), which writes the conversation JSON into + `conversations/` marked `response_side: 'recorded'`. Never write + `graph_ops` by hand and never relabel a synthesized response as recorded. +2. **Add the replay case.** Drive the fixture through the conversation + replay fixture (`agentConversationFixture`), asserting the + canvas-observable outcome the bug corrupted (graph end-state or + panel state - not mock call counts, not model text). Write what a + user sees for each turn (complete assistant text, tool rows in order) + into `agentConversationExpectations.ts`; the replay asserts those + literals and never predicts production rendering. +3. **Prove it bites.** Before merging, run the case once against the + fix's parent commit (red) and once at the fix (green); paste both + run lines in the PR description. A replay case that never went red + against the bug does not count as regression coverage. + +Name the case after the behavior it protects (`agent-.json`, with the fix PR cited in the fixture's `source.note`). The exporter writes the provenance the replay keeps: `source.capture` (thread and message ids) and `source.note` (row ids and the raw capture hash), which is enough to re-export the recording. + ## Test Data & Typed API Mocks Mock data in `fixtures/data/` exports **typed** objects that conform to diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts new file mode 100644 index 00000000000..e8ea56acc96 --- /dev/null +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -0,0 +1,608 @@ +import type { Locator, Page, WebSocketRoute } from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +import { createI18n } from 'vue-i18n' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { ObjectInfoResponse } from '@/schemas/nodeDefSchema' +import type { + AgentCancelAccepted, + AgentMessages, + AgentWsEvent +} from '@/workbench/extensions/agent/schemas/agentApiSchema' +import { parseServerDocFrame } from '@/workbench/extensions/agent/crdt/docFrameClient' +import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { agentTest, bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import { HostDoc } from '@e2e/fixtures/agentConversationHostDoc' +import type { HostFrame } from '@e2e/fixtures/agentConversationHostDoc' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' +import type { + AgentConversation, + AgentConversationTurn, + RecordedWsEvent +} from '@e2e/fixtures/data/agent/agentConversation' +import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' +import type { ExpectedTurn } from '@e2e/fixtures/data/agent/agentConversationExpectations' +import { RECORDED_EXPECTATIONS } from '@e2e/fixtures/data/agent/agentConversationExpectations' + +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' +// One synthetic message id per turn; the recorded ids never reach the page. +const turnId = (turn: number): string => + `0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5${turn.toString(16).padStart(3, '0')}` +const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' +const PANEL_MOUNT_TIMEOUT = 30_000 +const SUBSCRIBE_TIMEOUT = 15_000 +const CANCEL_TIMEOUT = 10_000 + +const OPEN_AGENT_LABEL = enMessages.agent.askComfyAgent +const SEND_LABEL = enMessages.agent.send +const STOP_LABEL = enMessages.agent.stop +// The composer names itself with the rendered message, escapes resolved; the +// app's own i18n module is a Vite build, so the fixture renders the message +// with the same library over the same locale file. +const COMPOSER_LABEL = createI18n({ + legacy: false, + locale: 'en', + messages: { en: enMessages } +}).global.t('agent.placeholder') +// Matches "Worked", "Worked for 3 seconds" and "Worked for 1m 2s" (agent.worked*). +const SUMMARY_LABEL = new RegExp(`^${enMessages.agent.worked}( for .+)?$`) +const FAILED_GLYPH = /lucide--circle-x/ +const THINKING_GLYPH = '[class*="lucide--brain"]' + +interface RecordedLink { + fromNode: string + fromSlot: number + toNode: string + toSlot: number +} + +interface RecordedWidgetValue { + nodeId: string + widget: string + value: string | number +} + +interface PanelCounts { + streams: number + summaries: number +} + +// [id, from, from_slot, to, to_slot, type], as the projection stores a link. +const zProjectedLink = z + .tuple([ + z.unknown(), + z.union([z.string(), z.number()]), + z.number(), + z.union([z.string(), z.number()]), + z.number() + ]) + .rest(z.unknown()) + +// The projected node fields the canvas assertions read. +const zProjectedNode = z + .object({ + id: z.union([z.string(), z.number()]), + type: z.string(), + title: z.string().optional(), + inputs: z + .array(z.object({ widget: z.unknown().optional() }).passthrough()) + .optional() + }) + .passthrough() + +function linkKey(link: RecordedLink): string { + return `${link.fromNode}:${link.fromSlot}->${link.toNode}:${link.toSlot}` +} + +function byLinkKey(a: RecordedLink, b: RecordedLink): number { + return linkKey(a).localeCompare(linkKey(b)) +} + +function collapse(text: string): string { + return text.replace(/\s+/g, ' ').trim() +} + +async function withTimeout( + promise: Promise, + ms: number, + message: string +): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(message)), ms) + }) + try { + await Promise.race([promise, timeout]) + } finally { + clearTimeout(timer) + } +} + +// Runs one recorded prompt/response through the real panel over a routed /ws socket. +class AgentConversationHarness { + readonly panel: Locator + readonly vueNodes: VueNodeHelpers + + private readonly host: HostDoc + private readonly streams: Locator + private readonly summaries: Locator + private readonly seedIds: Set + // Every node id the host has held so far, seed included. + private readonly seenIds: Set + private readonly expectations: ExpectedTurn[] + private socket: WebSocketRoute | null = null + private postedTurns = 0 + private readonly displayNames = new Map() + // Resolved when the panel cancels the turn the recording stopped. + private readonly cancelWaiters = new Map void>() + private resolveSubscribed: (() => void) | null = null + private readonly subscribed = new Promise((resolve) => { + this.resolveSubscribed = resolve + }) + + constructor( + private readonly page: Page, + readonly conversation: AgentConversation, + readonly replayTiming: ReplayTiming, + caseId: string + ) { + const { workflow } = conversation + this.host = new HostDoc(workflow.id, workflow.seed, workflow.catalog) + this.seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) + this.seenIds = new Set(this.seedIds) + const expectations = RECORDED_EXPECTATIONS[caseId] + const recorded = expectations?.length ?? 0 + if (recorded !== conversation.turns.length) + throw new Error( + `agentConversationExpectations.ts records ${recorded} turn(s) for ${caseId}; the recording has ${conversation.turns.length}` + ) + this.expectations = expectations ?? [] + this.panel = page.locator('#agent-panel-root') + this.streams = this.panel.getByTestId('markdown-stream') + this.summaries = this.panel.getByRole('button', { name: SUMMARY_LABEL }) + this.vueNodes = new VueNodeHelpers(page) + } + + async boot(agentFlag: boolean): Promise { + await this.mockAgentApi() + // The follower re-drives a pending subscribe only on a status frame, which every real connect sends. + await this.page.routeWebSocket(/\/ws/, (socket) => { + this.socket = socket + socket.onMessage((raw) => this.onClientFrame(raw)) + socket.send( + JSON.stringify({ + type: 'status', + data: { + status: { exec_info: { queue_remaining: 0 } }, + sid: SOCKET_SID + } + }) + ) + }) + const objectInfo = this.page.waitForResponse((response) => + new URL(response.url()).pathname.endsWith('/api/object_info') + ) + await bootAgentApp(this.page, agentFlag, { + // Only the Vue node renderer projects follower edits onto the canvas. + settings: { + 'Comfy.VueNodes.Enabled': true, + 'Comfy.Graph.CanvasInfo': false + }, + // Replayed nodes materialize from registered node types; the recordings use core nodes only. + objectInfo: 'server' + }) + const definitions = (await (await objectInfo).json()) as ObjectInfoResponse + for (const [type, definition] of Object.entries(definitions)) + this.displayNames.set(type, definition.display_name || definition.name) + + await this.page.getByRole('button', { name: OPEN_AGENT_LABEL }).click() + await expect(this.panel).toBeVisible({ timeout: PANEL_MOUNT_TIMEOUT }) + } + + async sendPrompt(turn = 0): Promise { + const { content } = this.conversation.turns[turn].request + const composer = this.panel.getByRole('textbox', { name: COMPOSER_LABEL }) + await composer.fill(content) + await this.panel.getByRole('button', { name: SEND_LABEL }).click() + // Replay frames are dropped until the page has applied the ack's thread id. + // useAgentSession records the user turn straight after storing that id, so + // the rendered turn says the ack landed without reading private storage. + await expect(this.panel.getByText(content).first()).toBeVisible() + } + + async replayResponse(turn = 0): Promise { + const startedAt = Date.now() + const entries = this.conversation.turns[turn].response.entries() + for (const [index, entry] of entries) { + // A timer can fire a millisecond early, so wait until the offset has really passed. + while ( + this.replayTiming === 'recorded' && + entry.at_ms !== undefined && + Date.now() - startedAt < entry.at_ms + ) + await new Promise((resolve) => + setTimeout(resolve, entry.at_ms! - (Date.now() - startedAt)) + ) + if (entry.kind === 'event') this.send(this.stampTurn(entry.event, turn)) + else { + await this.waitForSubscribe() + this.send(this.host.apply(entry.ops)) + for (const id of Object.keys(this.host.graph().nodes)) + this.seenIds.add(id) + } + // The recorded turn was stopped here, so the panel stops here too. + if (index === this.conversation.turns[turn].cancel_after) + await this.stopTurn(turn) + } + } + + // Every turn in order, each judged on the panel and the canvas as it lands. + async runTurns(): Promise { + for (const turn of this.conversation.turns.keys()) { + const before = await this.panelCounts() + await this.sendPrompt(turn) + await this.replayResponse(turn) + await this.waitForTurnComplete() + await this.expectTurnRendered(turn, before) + await this.expectCanvasReplayed(turn) + } + } + + private async panelCounts(): Promise { + return { + streams: await this.streams.count(), + summaries: await this.summaries.count() + } + } + + // Everything the recording played through the given turn (all turns by default). + private entries(throughTurn?: number): AgentConversationTurn['response'] { + const last = throughTurn === undefined ? undefined : throughTurn + 1 + return this.conversation.turns + .slice(0, last) + .flatMap((turn) => turn.response) + } + + // Clicks Stop, then holds the recorded tail until the panel's cancel for this + // turn reaches the server: the completed conversation is reachable only + // through that request. + private async stopTurn(turn: number): Promise { + const accepted = new Promise((resolve) => + this.cancelWaiters.set(turnId(turn), resolve) + ) + await this.panel.getByRole('button', { name: STOP_LABEL }).click() + await withTimeout( + accepted, + CANCEL_TIMEOUT, + 'the panel never cancelled the stopped turn' + ) + } + + async waitForTurnComplete(): Promise { + await expect( + this.panel.getByRole('button', { name: SEND_LABEL }) + ).toBeVisible() + await expect( + this.panel.getByRole('button', { name: STOP_LABEL }) + ).toHaveCount(0) + } + + // Last write wins per widget; the rendered control shows only the final value. + private recordedWidgetValues(throughTurn: number): RecordedWidgetValue[] { + const graph = this.host.graph() + const latest = new Map() + for (const entry of this.entries(throughTurn)) { + if (entry.kind !== 'graph_ops') continue + for (const op of entry.ops) { + if (op.op !== 'set_widget' || typeof op.widget !== 'string') continue + const nodeId = String(op.node_id) + const widget = op.widget + const widgets = graph.nodes[nodeId]?.widgets as + | Record + | undefined + const value = widgets?.[widget] + if (typeof value === 'string' || typeof value === 'number') + latest.set(`${nodeId}/${widget}`, { nodeId, widget, value }) + } + } + return [...latest.values()] + } + + private displayName(type: string): string { + const name = this.displayNames.get(type) + if (name === undefined) + throw new Error(`the server registers no node type ${type}`) + return name + } + + // A recorded title renders verbatim and a node the agent added shows its + // display name. A node the catch-up materialized is not settled today: + // the follower's reconcile titles it by type in most replays and by + // display name in some (#17171), so that path accepts either spelling of + // the same identity until the fix lands, and then narrows to the name. + private expectedTitle( + id: string, + node: { type: string; title?: string } + ): string | RegExp { + if (node.title) return node.title + const name = this.displayName(node.type) + if (!this.seedIds.has(id)) return name + const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + return new RegExp(`^(?:${escape(name)}|${escape(node.type)})$`) + } + + // The renderer's link map names the endpoints no DOM surface does; the + // painted result is a screenshot expectation in the replay spec. + private async renderedLinks(): Promise { + const links = await this.page.evaluate(() => + [...window.app!.graph.links.values()].map((link) => ({ + fromNode: String(link.origin_id), + fromSlot: link.origin_slot, + toNode: String(link.target_id), + toSlot: link.target_slot + })) + ) + return links.sort(byLinkKey) + } + + // What this turn put on the panel, against the recording's explicit + // expectations: the complete assistant text, and the tool call rows in order + // as a user reads them once the turn's work summary is open. + private async expectTurnRendered( + turn: number, + before: PanelCounts + ): Promise { + const expected = this.expectations[turn] + await expect + .poll(async () => + collapse( + (await this.streams.allInnerTexts()).slice(before.streams).join(' ') + ) + ) + .toBe(expected.text) + + // A finished turn folds its tool calls into one closed summary; the + // thinking rows it lists between them are not part of the recording. + const rows = expected.groups.flat() + await expect(this.summaries).toHaveCount( + before.summaries + (rows.length > 0 ? 1 : 0) + ) + if (rows.length === 0) return + const summary = this.summaries.nth(before.summaries) + await expect(summary).toHaveAttribute('aria-expanded', 'false') + await summary.click() + const items = summary + .locator('..') + .getByRole('listitem') + .filter({ hasNot: this.page.locator(THINKING_GLYPH) }) + await expect(items).toHaveCount(rows.length) + for (const [at, row] of rows.entries()) { + const item = items.nth(at) + await expect(item.getByText(row.label, { exact: true })).toBeVisible() + const glyph = item.locator('span').first() + if (row.failed) await expect(glyph).toHaveClass(FAILED_GLYPH) + else await expect(glyph).not.toHaveClass(FAILED_GLYPH) + if (row.count > 1) await expect(item).toContainText(`×${row.count}`) + else await expect(item).not.toContainText('×') + } + } + + // What the canvas shows after the given turn, judged the way a user would + // (which nodes, under which titles, with which widget values, wired on + // both slot rows) against the workflow the production library projects. + async expectCanvasReplayed(throughTurn: number): Promise { + const projected = this.host.projection() + const nodes = projected.nodes.map((node) => zProjectedNode.parse(node)) + const present = new Set(nodes.map((node) => String(node.id))) + for (const id of this.seenIds) + if (!present.has(id)) + await expect(this.vueNodes.getNodeLocator(id)).toHaveCount(0) + for (const node of nodes) { + const id = String(node.id) + const locator = this.vueNodes.getNodeLocator(id) + await expect(locator).toBeVisible() + await expect(locator.getByTestId('node-title')).toHaveText( + this.expectedTitle(id, node) + ) + } + await expect(this.page.getByTestId('node-title')).toHaveCount(nodes.length) + + for (const { nodeId, widget, value } of this.recordedWidgetValues( + throughTurn + )) { + const field = this.vueNodes + .getNodeLocator(nodeId) + .getByLabel(widget, { exact: true }) + if (typeof value === 'number') { + // Number widgets format their input (0.5 renders as 0.50), so compare the number. + await expect + .poll(async () => + Number(await field.locator('input').first().inputValue()) + ) + .toBe(value) + continue + } + const tag = await field.evaluate((el) => el.tagName.toLowerCase()) + if (tag === 'button') await expect(field).toContainText(value) + else await expect(field).toHaveValue(value) + } + + const links = projected.links + .map((raw): RecordedLink => { + const [, fromNode, fromSlot, toNode, toSlot] = zProjectedLink.parse(raw) + return { + fromNode: String(fromNode), + fromSlot, + toNode: String(toNode), + toSlot + } + }) + .sort(byLinkKey) + await expect.poll(() => this.renderedLinks()).toEqual(links) + // A widget-backed input renders no slot row on an uncollapsed node + // (NodeSlots.vue); the wire above already covers that end. + for (const link of links) { + await expect( + this.vueNodes.getOutputSlotRow(link.fromNode, link.fromSlot) + ).toHaveClass(/lg-slot--connected/) + const target = nodes.find((node) => String(node.id) === link.toNode) + if (target?.inputs?.[link.toSlot]?.widget == null) + await expect( + this.vueNodes.getInputSlotRow(link.toNode, link.toSlot) + ).toHaveClass(/lg-slot--connected/) + } + } + + private async mockAgentApi(): Promise { + const { page } = this + await page.route('**/api/agent/threads', (route) => + route.fulfill(jsonRoute({ threads: [] })) + ) + await page.route('**/api/agent/threads/*/messages', (route) => { + const request = route.request() + if (request.method() === 'POST') { + this.postedTurns += 1 + return route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify({ + thread_id: THREAD_ID, + message_id: turnId(this.postedTurns - 1), + workflow_id: this.conversation.workflow.id + }) + }) + } + const history: AgentMessages = [] + return route.fulfill(jsonRoute(history)) + }) + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => { + const target = /\/threads\/([^/]+)\/messages\/([^/]+)\/cancel/.exec( + route.request().url() + ) + const release = + target === null || target[1] !== THREAD_ID + ? undefined + : this.cancelWaiters.get(target[2]) + // A real server knows nothing about any other message. + if (release === undefined) return route.fulfill({ status: 404 }) + release() + const cancelled: AgentCancelAccepted = { status: 'cancelling' } + return route.fulfill(jsonRoute(cancelled)) + }) + await page.route('**/api/workflows**', (route) => + route.fulfill( + jsonRoute({ + data: [], + pagination: { has_more: false, limit: 100, offset: 0, total: 0 } + }) + ) + ) + } + + private stampTurn(event: RecordedWsEvent, turn: number): AgentWsEvent { + const stamped = { + type: event.type, + data: { ...event.data, message_id: turnId(turn), thread_id: THREAD_ID } + } + const parsed = parseAgentWsEvent(stamped) + if (!parsed.success) + throw new Error( + `recorded ${event.type} frame is not a valid agent event: ${parsed.error.message}` + ) + return parsed.data + } + + private send(frame: AgentWsEvent | HostFrame): void { + // Every host frame must satisfy production's own parser, so a host that + // stopped emitting a required field fails here, not silently on the client. + if ( + (frame.type.startsWith('doc_') || frame.type === 'awareness') && + parseServerDocFrame(frame) === null + ) + throw new Error(`host frame ${frame.type} is not a valid doc frame`) + if (!this.socket) throw new Error('the app has not opened /ws yet') + this.socket.send(JSON.stringify(frame)) + } + + private onClientFrame(raw: string | Buffer): void { + const frame: unknown = JSON.parse(raw.toString()) + if (typeof frame !== 'object' || frame === null) return + const { type, data } = frame as { type?: unknown; data?: unknown } + if (type !== 'doc_subscribe' || typeof data !== 'object' || data === null) + return + const { workflow_id, state_vector_b64 } = data as { + workflow_id?: unknown + state_vector_b64?: unknown + } + if ( + workflow_id !== this.conversation.workflow.id || + typeof state_vector_b64 !== 'string' + ) + return + this.send(this.host.subscribed()) + this.send(this.host.catchUp(state_vector_b64)) + this.resolveSubscribed?.() + } + + private waitForSubscribe(): Promise { + return withTimeout( + this.subscribed, + SUBSCRIBE_TIMEOUT, + 'the follower never subscribed to the conversation workflow; graph_ops need an agent_active_tab (or a bound tab) first' + ) + } +} + +export type ReplayTiming = 'immediate' | 'recorded' + +function defaultReplayTiming(): ReplayTiming { + const value = process.env.AGENT_REPLAY_TIMING + if (value === undefined || value === 'immediate') return 'immediate' + if (value === 'recorded') return 'recorded' + throw new Error( + `AGENT_REPLAY_TIMING must be immediate or recorded, got ${value}` + ) +} + +interface ConversationFixtures { + conversationCase: string + // 'recorded' replays the fixture's at_ms gaps; the default follows AGENT_REPLAY_TIMING. + replayTiming: ReplayTiming + agentConversation: AgentConversationHarness +} + +// Wide enough to keep the whole seed graph beside the docked panel; the video follows the viewport instead of Playwright's 800px cap. +const VIEWPORT = { width: 2560, height: 1440 } + +export const agentConversationTest = agentTest.extend({ + conversationCase: ['', { option: true }], + replayTiming: [defaultReplayTiming(), { option: true }], + viewport: VIEWPORT, + video: { + mode: + process.env.PLAYWRIGHT_LOCAL || process.env.RECORD_VIDEO === 'true' + ? 'on' + : 'off', + size: VIEWPORT + }, + agentConversation: async ( + { page, agentFlagEnabled, conversationCase, replayTiming }, + use + ) => { + if (conversationCase.length === 0) + throw new Error('test.use({ conversationCase }) names the conversation') + const harness = new AgentConversationHarness( + page, + loadAgentConversation(conversationCase), + replayTiming, + conversationCase + ) + await harness.boot(agentFlagEnabled) + await use(harness) + } +}) diff --git a/browser_tests/fixtures/agentConversationHostDoc.ts b/browser_tests/fixtures/agentConversationHostDoc.ts new file mode 100644 index 00000000000..9ebd18a0aa6 --- /dev/null +++ b/browser_tests/fixtures/agentConversationHostDoc.ts @@ -0,0 +1,117 @@ +import { + applyOps, + mint, + project, + readGraph +} from '@comfyorg/comfy-multi-player' +import type { + GraphSnapshot, + WidgetCatalog, + WorkflowJSON +} from '@comfyorg/comfy-multi-player' +import * as Y from 'yjs' + +import type { ServerDocFrame } from '@/workbench/extensions/agent/crdt/docFrameClient' +import { DOC_PROTOCOL_VERSION } from '@/workbench/extensions/agent/crdt/docFrameClient' +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' + +import type { RecordedGraphOperation } from '@e2e/fixtures/data/agent/agentConversation' +import { mintWireOps } from '@/workbench/extensions/agent/crdt/opEnvelope' + +const HOST_ACTOR = 'agent:comfy:host' + +// The shape the fake host puts on the wire; production's parseServerDocFrame +// validates each one at send time. +export interface HostFrame { + type: ServerDocFrame['type'] + data: Record +} + +function toBase64(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('base64') +} + +function fromBase64(value: string): Uint8Array { + return new Uint8Array(Buffer.from(value, 'base64')) +} + +// The doc host stand-in: the real library in-process, only the transport is local. +export class HostDoc { + private readonly doc: Y.Doc + private seq = 1 + + constructor( + private readonly workflowId: string, + seed: WorkflowJSON, + private readonly catalog: WidgetCatalog + ) { + this.doc = mint(seed, catalog) + } + + graph(): GraphSnapshot { + return readGraph(this.doc) + } + + // The canonical workflow the library projects from the document: the + // nodes, titles, inputs and link tuples the canvas is expected to show. + projection(): WorkflowJSON { + return project(this.doc, this.catalog) + } + + subscribed(): HostFrame { + return { + type: 'doc_subscribed', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + ok: true, + seq: this.seq + } + } + } + + catchUp(stateVectorB64: string): HostFrame { + const update = Y.encodeStateAsUpdate(this.doc, fromBase64(stateVectorB64)) + return this.updateFrame(update, HOST_ACTOR, []) + } + + // The applier below is the only judge of a recorded op; the cast hands it + // the structural record and nothing reads the ops as typed before it runs. + apply(operations: RecordedGraphOperation[]): HostFrame { + const before = Y.encodeStateVector(this.doc) + const ops = mintWireOps(operations as GraphOperation[], { + actor: HOST_ACTOR, + baseVersion: this.seq + }) + const result = applyOps(this.doc, ops, this.catalog) + const rejected = result.outcomes.filter((o) => o.outcome !== 'applied') + if (rejected.length > 0) + throw new Error( + `conversation graph_ops did not apply: ${JSON.stringify(rejected)}` + ) + this.seq += 1 + return this.updateFrame( + Y.encodeStateAsUpdate(this.doc, before), + HOST_ACTOR, + ops.map((op) => op.op_id) + ) + } + + private updateFrame( + update: Uint8Array, + actor: string, + opIds: string[] + ): HostFrame { + return { + type: 'doc_update', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + seq: this.seq, + update_b64: toBase64(update), + actor, + op_ids: opIds + } + } + } +} diff --git a/browser_tests/fixtures/agentPanelFixture.ts b/browser_tests/fixtures/agentPanelFixture.ts index 033c519adb7..58badbfb828 100644 --- a/browser_tests/fixtures/agentPanelFixture.ts +++ b/browser_tests/fixtures/agentPanelFixture.ts @@ -23,16 +23,29 @@ function agentFeatures(agentFlag: boolean): RemoteConfig { } } +interface BootAgentAppOptions { + /** Extra `/api/settings` entries layered over the panel defaults. */ + settings?: Record + /** `'server'` loads real node definitions instead of the empty catalog. */ + objectInfo?: 'server' +} + async function mockAgentBoot( page: Page, - { agentFlag }: { agentFlag: boolean } + { + agentFlag, + settings, + objectInfo + }: { agentFlag: boolean } & BootAgentAppOptions ): Promise { await mockCloudBoot(page, { features: agentFeatures(agentFlag), settings: { 'Comfy.TutorialCompleted': true, - 'Comfy.RightSidePanel.ShowErrorsTab': false - } + 'Comfy.RightSidePanel.ShowErrorsTab': false, + ...settings + }, + objectInfo }) await mockBilling(page) const emptyAssets: ListAssetsResponse = { @@ -65,14 +78,15 @@ export const agentTest = cloudAppFixture.extend({ export async function bootAgentApp( page: Page, - agentFlag: boolean + agentFlag: boolean, + options: BootAgentAppOptions = {} ): Promise { // The shell's onboarding coach is a modal; pre-seed its dismissal so the // panel chrome is interactable, as the canonical agent suite does. await page.addInitScript(() => { localStorage.setItem('Comfy.AgentPanel.onboarded', 'true') }) - await mockAgentBoot(page, { agentFlag }) + await mockAgentBoot(page, { agentFlag, ...options }) await bootCloud(page) await page.goto(APP_URL) await waitForCloudApp(page) diff --git a/browser_tests/fixtures/data/agent/agentConversation.test.ts b/browser_tests/fixtures/data/agent/agentConversation.test.ts new file mode 100644 index 00000000000..6f1b1cd7dea --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversation.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest' + +import { readFileSync, readdirSync } from 'node:fs' +import { join } from 'node:path' + +import { + assertOpsApply, + zAgentConversation, + zRecordedWsEvent +} from '@e2e/fixtures/data/agent/agentConversation' +import { RECORDED_EXPECTATIONS } from '@e2e/fixtures/data/agent/agentConversationExpectations' +import { zAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +const recorded = { + schema_version: 'agent-conversation.v2', + source: { + repo: 'Comfy-Org/evals', + suite: 'agent', + case_id: 'recorded-case', + response_side: 'recorded', + capture: { + backend: 'Comfy-Org/cloud', + thread_id: 'thread-1', + exported_at: '2026-09-02T16:00:00.000Z' + } + }, + workflow: { + id: '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01', + name: 'Captured workflow', + catalog: { types: {} }, + seed: { nodes: [], links: [] } + }, + turns: [ + { + message_id: 'message-1', + request: { content: 'Add a node' }, + response: [ + { kind: 'event', event: { type: 'agent_message_done', data: {} } } + ] + } + ] +} + +describe('zAgentConversation', () => { + it('accepts a recorded conversation with provenance and message ids', () => { + expect(zAgentConversation.parse(recorded)).toEqual(recorded) + }) + + it('refuses a recorded turn without the message id it came from', () => { + const { message_id: _messageId, ...anonymous } = recorded.turns[0] + expect(() => + zAgentConversation.parse({ ...recorded, turns: [anonymous] }) + ).toThrow('recorded turns carry the message id') + }) + + it('refuses a recorded label without backend provenance', () => { + const { capture: _capture, ...source } = recorded.source + expect(() => zAgentConversation.parse({ ...recorded, source })).toThrow( + 'recorded responses require backend capture provenance' + ) + }) + + it('refuses a turn with two completion events', () => { + const done = { + kind: 'event', + event: { type: 'agent_message_done', data: {} } + } + const tab = { + kind: 'event', + event: { + type: 'agent_active_tab', + data: { workflow_id: recorded.workflow.id, name: 'Captured workflow' } + } + } + const turn = { ...recorded.turns[0], response: [tab, done, tab, done] } + expect(() => + zAgentConversation.parse({ ...recorded, turns: [turn] }) + ).toThrow('exactly one agent_message_done') + }) +}) + +describe('zRecordedWsEvent', () => { + const discriminators = ( + union: typeof zAgentWsEvent | typeof zRecordedWsEvent + ) => union.options.map((option) => option.shape.type.value) + + it('mirrors every production event type', () => { + expect(discriminators(zRecordedWsEvent)).toEqual( + discriminators(zAgentWsEvent) + ) + }) + + it('validates the production fields without the ids the replay mints', () => { + const frame = { + type: 'agent_tool_call', + data: { + tool_call_id: 'toolu_1', + tool_name: 'apply_ops', + status: 'success', + duration_ms: 12 + } + } + expect(zRecordedWsEvent.parse(frame)).toEqual(frame) + expect(() => + zRecordedWsEvent.parse({ + ...frame, + data: { ...frame.data, status: 'done' } + }) + ).toThrow() + expect(() => + zRecordedWsEvent.parse({ type: 'agent_done', data: {} }) + ).toThrow() + }) +}) + +describe('committed recordings', () => { + // import.meta.url is not a file URL under vitest, so the loaders are bypassed. + const dir = join( + process.cwd(), + 'browser_tests/fixtures/data/agent/conversations' + ) + const files = readdirSync(dir).filter((file) => file.endsWith('.json')) + const load = (file: string): unknown => + JSON.parse(readFileSync(join(dir, file), 'utf8')) + + it('every recording parses against the production event union', () => { + expect(files.length).toBeGreaterThan(0) + for (const file of files) { + const raw = load(file) + const conversation = zAgentConversation.parse(raw) + expect(() => assertOpsApply(conversation), file).not.toThrow() + expect({ file, workflow: conversation.workflow }).toEqual({ + file, + workflow: (raw as { workflow: unknown }).workflow + }) + const frames = conversation.turns + .flatMap((turn) => turn.response) + .filter((entry) => entry.kind === 'event') + expect({ file, frames: frames.length }).not.toEqual({ file, frames: 0 }) + } + }) + + it('every recording has explicit visible expectations for each turn', () => { + const turns = Object.fromEntries( + files.map((file) => [ + file.slice(0, -'.json'.length), + zAgentConversation.parse(load(file)).turns.length + ]) + ) + expect( + Object.fromEntries( + Object.entries(RECORDED_EXPECTATIONS).map(([caseId, expected]) => [ + caseId, + expected?.length ?? 0 + ]) + ) + ).toEqual(turns) + }) + + it('the cancelled recording stops with text still to render', () => { + const conversation = zAgentConversation.parse( + load('agent-rec-cancelled-turn.json') + ) + const [turn] = conversation.turns + expect(turn.cancel_after).toBeDefined() + expect( + turn.response.some( + (entry) => + entry.kind === 'event' && + entry.event.type === 'agent_message_delta' && + entry.event.data.delta.length > 0 + ) + ).toBe(true) + }) + + it('refuses a set_widget whose value never reaches the document', () => { + const raw = load('agent-rec-set-widget-existing.json') as { + turns: Array<{ + response: Array<{ kind: string; ops?: Array> }> + }> + } + const write = raw.turns[0].response + .flatMap((entry) => entry.ops ?? []) + .find((op) => op.op === 'set_widget')! + delete write.value + expect(() => assertOpsApply(zAgentConversation.parse(raw))).toThrow( + /carries no value at projection\.nodes\[\d+\]\.widgets_values\[\d+\]/ + ) + }) +}) diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts new file mode 100644 index 00000000000..b809ea41757 --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -0,0 +1,278 @@ +import { readFileSync, readdirSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +import { + FROZEN_OPS, + applyOps, + mint, + project +} from '@comfyorg/comfy-multi-player' +import type { OpBase } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' +import { mintWireOps } from '@/workbench/extensions/agent/crdt/opEnvelope' +import { zAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +// A recording keeps every production field except the two ids the replay +// mints per run (agentConversationFixture stampTurn). +const mintedIds: { thread_id: true; message_id: true } = { + thread_id: true, + message_id: true +} +// Every member gets the same transform, so the slot order only names them. +const [ + thinking, + toolCall, + messageDelta, + messageDone, + activeTab, + ask, + askResolved +] = zAgentWsEvent.options + +export const zRecordedWsEvent = z.discriminatedUnion('type', [ + thinking.extend({ data: thinking.shape.data.omit(mintedIds) }), + toolCall.extend({ data: toolCall.shape.data.omit(mintedIds) }), + messageDelta.extend({ data: messageDelta.shape.data.omit(mintedIds) }), + messageDone.extend({ data: messageDone.shape.data.omit(mintedIds) }), + activeTab.extend({ data: activeTab.shape.data.omit(mintedIds) }), + ask.extend({ data: ask.shape.data.omit(mintedIds) }), + askResolved.extend({ data: askResolved.shape.data.omit(mintedIds) }) +]) +export type RecordedWsEvent = z.infer + +// GraphOperation is the wire op minus this envelope, which the replay remints +// through mintWireOps. Exhaustive by construction: a new OpBase field fails to +// typecheck here rather than slipping into a recording. +const OP_ENVELOPE: Record = { + op_id: true, + actor: true, + base_version: true, + stamp: true +} +const OP_ENVELOPE_KEYS = Object.keys(OP_ENVELOPE) + +// Only the vocabulary and the absence of the envelope are checked here; a +// recorded op stays structural. Nothing in this fixture parses the semantic +// operation contract, which the package owns; assertOpsApply below applies +// the ops to a document and reports what that application shows. +const zGraphOperation = z + .object({ op: z.enum(FROZEN_OPS) }) + .passthrough() + .refine( + (op) => OP_ENVELOPE_KEYS.every((key) => !(key in op)), + 'a recorded op carries the semantic operation only; the wire envelope is minted at replay' + ) +export type RecordedGraphOperation = z.infer + +// WorkflowJSON and WorkflowNode declare an index signature, so the schema +// validates the guaranteed fields and keeps the rest. +const zWorkflowJson = z + .object({ + nodes: z.array( + z + .object({ id: z.union([z.string(), z.number()]), type: z.string() }) + .passthrough() + ), + links: z.array(z.unknown()) + }) + .passthrough() + +const zWidgetCatalogEntry = z + .object({ + widget_order: z.array(z.string()), + autogrow_templates: z + .record( + z.string(), + z + .object({ + prefix: z.string().optional(), + names: z.array(z.string()).optional() + }) + .passthrough() + ) + .optional() + }) + .strict() + +// WidgetCatalog and WidgetCatalogEntry declare no index signature. +const zWidgetCatalog = z + .object({ + comment: z.string().optional(), + types: z.record(z.string(), zWidgetCatalogEntry) + }) + .strict() + +const zAgentConversationWorkflow = z.object({ + id: z.string().uuid(), + name: z.string().min(1), + catalog: zWidgetCatalog, + seed: zWorkflowJson +}) + +const zAgentConversationRequest = z.object({ + content: z.string().min(1) +}) + +// Offset from the turn's first frame, so replays can reproduce real gaps. +const zAtMs = z.number().int().nonnegative().optional() + +const zResponseEntry = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('event'), event: zRecordedWsEvent, at_ms: zAtMs }), + z.object({ + kind: z.literal('graph_ops'), + ops: z.array(zGraphOperation).min(1), + at_ms: zAtMs + }) +]) + +const zTurn = z + .object({ + message_id: z.string().min(1).optional(), + request: zAgentConversationRequest, + // Response entry the recorded cancel followed; the replay stops there too. + cancel_after: z.number().int().nonnegative().optional(), + response: z.array(zResponseEntry).min(1) + }) + .superRefine((turn, ctx) => { + const dones = turn.response.flatMap((entry, index) => + entry.kind === 'event' && entry.event.type === 'agent_message_done' + ? [index] + : [] + ) + if (dones.length !== 1 || dones[0] !== turn.response.length - 1) + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['response'], + message: + 'a turn carries exactly one agent_message_done event, as its last entry' + }) + if ( + turn.cancel_after !== undefined && + turn.cancel_after >= turn.response.length - 1 + ) + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['cancel_after'], + message: + 'cancel_after must precede the final agent_message_done entry; the replay stops a turn that is still running' + }) + }) +export type AgentConversationTurn = z.infer + +export const zAgentConversation = z + .object({ + schema_version: z.literal('agent-conversation.v2'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + response_side: z.enum(['recorded', 'synthesized']), + note: z.string().optional(), + capture: z + .object({ + backend: z.literal('Comfy-Org/cloud'), + thread_id: z.string().min(1), + exported_at: z.string().datetime() + }) + .optional() + }), + workflow: zAgentConversationWorkflow, + // One thread; each turn lands on the graph the previous turn left. + turns: z.array(zTurn).min(1) + }) + .superRefine((conversation, ctx) => { + if (conversation.source.response_side !== 'recorded') return + if (conversation.source.capture === undefined) + ctx.addIssue({ + code: 'custom', + path: ['source', 'capture'], + message: 'recorded responses require backend capture provenance' + }) + for (const [index, turn] of conversation.turns.entries()) + if (turn.message_id === undefined) + ctx.addIssue({ + code: 'custom', + path: ['turns', index, 'message_id'], + message: 'recorded turns carry the message id they were captured from' + }) + }) +export type AgentConversation = z.infer + +export function listRecordedConversations(): string[] { + const dir = fileURLToPath(new URL('./conversations/', import.meta.url)) + return readdirSync(dir) + .filter((file) => file.endsWith('.json')) + .map((file) => file.slice(0, -'.json'.length)) + .map((caseId) => { + if (loadAgentConversation(caseId).source.response_side !== 'recorded') + throw new Error( + `${caseId} is not marked response_side: recorded; the replay suite only carries recordings` + ) + return caseId + }) + .sort() +} + +// The first place the projection holds no value, if any. The persisted draft +// is this projection as JSON, so a value JSON cannot carry never reached the +// document, whatever the applier said about the op that wrote it. +function missingValue(value: unknown, path: string): string | null { + if (value === undefined) return path + if (Array.isArray(value)) { + for (const [index, item] of value.entries()) { + const hit = missingValue(item, `${path}[${index}]`) + if (hit !== null) return hit + } + return null + } + if (value !== null && typeof value === 'object') { + for (const [key, item] of Object.entries(value)) { + const hit = missingValue(item, `${path}.${key}`) + if (hit !== null) return hit + } + } + return null +} + +// Applies every recorded op to a document minted from the recording's seed +// and catalog. This is application, not parsing: it shows the applier accepts +// each op against that document and that the projection it leaves carries a +// value everywhere, which is what the replay needs and all it proves. +export function assertOpsApply(conversation: AgentConversation): void { + const { seed, catalog } = conversation.workflow + const doc = mint(seed, catalog) + let version = 1 + for (const [turnIndex, turn] of conversation.turns.entries()) + for (const [entryIndex, entry] of turn.response.entries()) { + if (entry.kind !== 'graph_ops') continue + const label = `turn ${turnIndex} entry ${entryIndex}` + const ops = mintWireOps(entry.ops as GraphOperation[], { + actor: 'agent:comfy:host', + baseVersion: version + }) + version += 1 + const rejected = applyOps(doc, ops, catalog).outcomes.filter( + (outcome) => outcome.outcome !== 'applied' + ) + if (rejected.length > 0) + throw new Error( + `${label}: the applier rejected ${JSON.stringify(rejected)}` + ) + const hole = missingValue(project(doc, catalog), 'projection') + if (hole !== null) + throw new Error(`${label}: the projection carries no value at ${hole}`) + } +} + +export function loadAgentConversation(caseId: string): AgentConversation { + const file = fileURLToPath( + new URL(`./conversations/${caseId}.json`, import.meta.url) + ) + const conversation = zAgentConversation.parse( + JSON.parse(readFileSync(file, 'utf-8')) + ) + assertOpsApply(conversation) + return conversation +} diff --git a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts new file mode 100644 index 00000000000..2eb4ac5292c --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts @@ -0,0 +1,471 @@ +// What a user sees on the panel for each recorded turn: the complete assistant +// text, and the tool call groups in order with each row's label, multiplicity +// and failure state. Explicit per recording, so the replay never predicts +// production rendering. +interface ExpectedToolRow { + label: string + count: number + failed: boolean +} + +export interface ExpectedTurn { + text: string + groups: ExpectedToolRow[][] +} + +export const RECORDED_EXPECTATIONS: Partial> = { + 'agent-l4-zimage-string-node-prompt': [ + { + text: "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all. I couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up. Two ways forward, your call: Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct. Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first.", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Find nodes', + count: 1, + failed: false + } + ], + [ + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'Validate', + count: 1, + failed: true + }, + { + label: 'Show node', + count: 1, + failed: false + }, + { + label: 'List model picks', + count: 1, + failed: true + }, + { + label: 'List generate models', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-add-set-delete': [ + { + text: "I'm on the \"Text to image\" tab now, and the second text encode node is in place, taking its text understanding from the same checkpoint loader as the first one. Two things worth flagging, since I only did exactly what you asked: Its text box is empty, and it isn't connected onward to anything yet — most likely you want it as the negative prompt into the sampler, whose negative input is currently empty. That sampler is also missing its image-size input, so the workflow won't run as it stands regardless of this new node. Say the word and I'll wire the new node in as the negative prompt (with whatever text you want) and add the missing size input so it's ready to run.", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + }, + { + text: 'Done — the new node now reads "blurry, low quality". It\'s still not wired into the sampler, so it won\'t affect a render until I connect it as the negative prompt. Want me to do that?', + groups: [ + [ + { + label: 'Set widget', + count: 1, + failed: false + } + ] + ] + }, + { + text: 'Removed — the canvas is back to the five original nodes.', + groups: [ + [ + { + label: 'Delete node', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-batched-ops': [ + { + text: "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt.", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-cancelled-turn': [ + { + text: 'Stopped at your request. Before stopping I made 1 edit to your canvas.', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Ls nodes', + count: 1, + failed: false + } + ], + [ + { + label: 'Add node', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-clarifying-question': [ + { + text: "I'm on the \"Text to image\" tab now, but I don't have any record of your usual sampler settings — nothing saved from earlier in our conversation, and this is the first workflow I've seen from you. Rather than guess and quietly change your image, tell me the values and I'll set them (and remember them for next time). Right now the sampler is on: Steps: 20 CFG: 7 Sampler: euler Scheduler: normal Denoise: 1.0 Seed: 0, randomized each run Just say what you want them changed to — e.g. \"30 steps, cfg 4.5, dpmpp_2m karras\" — or tell me the name of a workflow where I can read them off, and I'll copy them across. One other thing worth flagging: this graph is missing a negative prompt and an image-size input into the sampler, so it won't run as-is. I can add both while I'm in there.", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'List workflows', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + }, + { + text: "Steps are now 30, and I've noted that as your default for next time. The graph still won't run as-is — the sampler has no negative prompt or image-size input wired in. Want me to add those?", + groups: [ + [ + { + label: 'List slots', + count: 1, + failed: false + }, + { + label: 'Saved a preference', + count: 1, + failed: false + }, + { + label: 'Set widget', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-refiner-between': [ + { + text: "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node. Two things were already unfinished in this graph before I touched it, and they'll block a run: Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler. The checkpoint it points at isn't installed here, so no model file is actually available. I set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Validate', + count: 1, + failed: true + } + ] + ] + } + ], + 'agent-rec-replace-prompt-encoder': [ + { + text: 'Done — I\'m on the "Text to image" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler\'s positive input. I carried over the same prompt text ("a photo of a pier"). One thing unrelated to this swap: the sampler\'s negative prompt and empty latent inputs were already unconnected before I started, so the graph won\'t run until those are filled in. Want me to add them?', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'Delete node', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-set-widget-existing': [ + { + text: 'You\'re now on the "Text to image" tab, with steps at 30 and cfg at 5. Nothing else was touched.', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Ls nodes', + count: 1, + failed: false + }, + { + label: 'List slots', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'List slots', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-text-only-answer': [ + { + text: 'I\'m on the "Text to image" tab now. It\'s a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently "a photo of a pier" — into guidance, generates an image over 20 steps, and saves the result. One thing to flag: it isn\'t runnable as-is. Two inputs on the generation step are still empty — there\'s no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-rec-three-sequential-adds': [ + { + text: 'All three nodes are on the "Text to image" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked.', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Add node', + count: 3, + failed: false + } + ] + ] + } + ], + 'agent-rec-tool-error': [ + { + text: 'I\'m now on the "Text to image" tab, but there\'s no node called "DoesNotExist" — nothing was added, and the canvas is unchanged. If you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I\'ll find the real one and wire it in.', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Add node', + count: 1, + failed: true + } + ] + ] + } + ], + 'agent-rec-two-turn-dependent-edit': [ + { + text: 'I\'m on the "Text to image" tab now, and the second text box is added with "blurry, low quality" and fed from the checkpoint loader. One thing worth flagging: it isn\'t connected to anything downstream yet, so it has no effect on the image. The sampler\'s negative input is currently empty (and its latent input too) — say the word and I\'ll wire this new text into the negative slot.', + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + }, + { + text: 'Connected — the "blurry, low quality" text now feeds the negative input. The graph still won\'t run as-is: nothing is feeding the sampler\'s latent input, so it needs an empty latent (which is also where the image size gets set). Want me to add one?', + groups: [ + [ + { + label: 'Connect', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + } + ] + ] + } + ], + 'agent-workflow-editing-05': [ + { + text: "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved. Two things worth knowing: The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider. The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?", + groups: [ + [ + { + label: 'Switched tabs', + count: 1, + failed: false + }, + { + label: 'Print workflow', + count: 1, + failed: false + }, + { + label: 'Find nodes', + count: 1, + failed: false + }, + { + label: 'Apply ops', + count: 1, + failed: false + }, + { + label: 'List slots', + count: 1, + failed: false + }, + { + label: 'Set widget', + count: 1, + failed: true + }, + { + label: 'Validate', + count: 1, + failed: true + } + ] + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json new file mode 100644 index 00000000000..1db1de5620b --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -0,0 +1,518 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-l4-zimage-string-node-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 91a905f3-b619-44e3-a6e5-5a047d8d0066; message 4293701d-cd6f-41cf-b03d-ea73df6af846; workflow 891d1304-09ae-4d51-9bea-aaa4756822d4 (seeded by throwaway turn 62bbe9f8-bcb1-4a84-9275-97384a9482f2; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ea9a0cdc-498e-4c8e-ba09-e487cbf02e12', '3893c0d2-605b-43bc-8d77-ff249b267087', '3ae90aa2-14e3-4001-a3c9-90eadef4ad86', '5656b627-7364-4759-8471-f0dedbf3b125', '7e4bd8e9-223d-4a79-985e-7bd262e8b5b9', 'c7f90263-0a57-4a48-8573-75bdb1746a6e', 'acc3faea-b08b-4e4d-8f4b-cb999a8cc6d5', 'a928d523-16e8-4d26-81ad-1b7cdd884c68', '42abc204-3da4-410c-bf83-1b206d567401']; rows agent-l4-zimage-string-node-prompt.t-a1.rows.json; raw capture sha256 f4a2076ae60f8e5eefc7799fd0a14d58d4c3829796ff89f88a0e4548c952b3fe", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "91a905f3-b619-44e3-a6e5-5a047d8d0066", + "exported_at": "2026-09-03T12:40:57Z" + } + }, + "workflow": { + "id": "891d1304-09ae-4d51-9bea-aaa4756822d4", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "4293701d-cd6f-41cf-b03d-ea73df6af846", + "request": { + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look at that workflow." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "891d1304-09ae-4d51-9bea-aaa4756822d4" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01KZvQdg1MdhKgbWQsvScSpN", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 774, + "status": "success", + "tool_call_id": "toolu_015MGXQKWWD64ZWnv7jWQNLw", + "tool_name": "print_workflow" + } + }, + "at_ms": 2809 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 308, + "status": "success", + "tool_call_id": "toolu_015ngjjFSSFFFRrsQvJwbnut", + "tool_name": "find_nodes" + } + }, + "at_ms": 7587 + }, + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "The prompt node is easy — but this graph is also missing a couple of connections it needs to run, so I'll complete those too." + } + }, + "at_ms": 13907 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [395, 280], + "node": { + "id": 3876406316923056, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 3876406316923056, + "class_type": "PrimitiveStringMultiline" + }, + { + "op": "set_widget", + "old": null, + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 3876406316923056 + }, + { + "op": "connect", + "link_id": 1887935066439213, + "to_node": 6, + "to_slot": 1, + "from_node": 3876406316923056, + "from_slot": 0, + "link_type": "STRING" + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3880387162517873, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 3880387162517873, + "class_type": "CLIPTextEncode" + }, + { + "op": "set_widget", + "old": null, + "value": "", + "widget": "text", + "node_id": 3880387162517873 + }, + { + "op": "connect", + "link_id": 2636276550287764, + "to_node": 3880387162517873, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + }, + { + "op": "connect", + "link_id": 2833535472085255, + "to_node": 3, + "to_slot": 2, + "from_node": 3880387162517873, + "from_slot": 0, + "link_type": "CONDITIONING" + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 3876084544359242, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "node_id": 3876084544359242, + "class_type": "EmptyLatentImage" + }, + { + "op": "connect", + "link_id": 2211554611670965, + "to_node": 3, + "to_slot": 3, + "from_node": 3876084544359242, + "from_slot": 0, + "link_type": "LATENT" + } + ], + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 447, + "status": "success", + "tool_call_id": "toolu_0126og8icdp1EUnSmtrcSZv6", + "tool_name": "apply_ops" + } + }, + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 249, + "status": "error", + "tool_call_id": "toolu_01RQTp9JXwFc1AdgmAKBiMYm", + "tool_name": "validate" + } + }, + "at_ms": 16172 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 232, + "status": "success", + "tool_call_id": "toolu_01Lq8eZseF7AhmTd9UuxvcNb", + "tool_name": "show_node" + } + }, + "at_ms": 19599 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 248, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01VcWs7EVFuk18ct7nX9nEoi", + "tool_name": "list_model_picks" + } + }, + "at_ms": 24061 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 887, + "status": "success", + "tool_call_id": "toolu_01AB6XXLQrdF3EFbDRV9GtSx", + "tool_name": "list_generate_models" + } + }, + "at_ms": 28706 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 324, + "status": "success", + "tool_call_id": "toolu_01U2bR9WL8Pc5H35X2JERBwr", + "tool_name": "print_workflow" + } + }, + "at_ms": 38928 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all.\n\nI couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up.\n\nTwo ways forward, your call:\n- Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct.\n- Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first." + } + }, + "at_ms": 44823 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 4467, + "cache_read_input_tokens": 395126, + "input_tokens": 20, + "output_tokens": 2467, + "total_tokens": 2487 + } + } + }, + "at_ms": 44827 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-add-set-delete.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-add-set-delete.json new file mode 100644 index 00000000000..85cf2ea2966 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-add-set-delete.json @@ -0,0 +1,469 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-add-set-delete", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6b1502e7-a553-4d04-aa07-07786ab73223; messages 1a1dd9a7-3654-4869-beb4-7d9f1b3d8f67, 9be1516f-95b5-47b2-b939-46c60f141da7, 0b966fe0-ac97-4506-abbc-743051b5c526; workflow 79cf8c10-c40b-4f3a-8c88-c3961e3882fb (seeded by throwaway turn 2cc6c8bb-0057-4905-a4e7-acf0c50f440f; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 573561a7-72dc-43de-a5a5-12be3d9fb403, 6d659f56-9cf1-49bb-91c8-72174fb8143a, c2370db1-c495-4c11-a4cd-4e0f80af092c, c295afbc-ff57-485c-b7ae-4ea4183b3fef, d61f5547-6761-4cb7-9f3c-f4ef8f47c3e1, ebb740d8-5eea-46c2-bc71-a4a1fb49d27f; rows agent-rec-add-set-delete.b1.rows.1.json, agent-rec-add-set-delete.b1.rows.2.json, agent-rec-add-set-delete.b1.rows.3.json; raw capture sha256 b9fad5c571ab000aa3fb30d17016756cab45774bc4ccf3231d5200d867d92aef", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "6b1502e7-a553-4d04-aa07-07786ab73223", + "exported_at": "2026-09-03T16:13:44.733Z" + } + }, + "workflow": { + "id": "79cf8c10-c40b-4f3a-8c88-c3961e3882fb", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "1a1dd9a7-3654-4869-beb4-7d9f1b3d8f67", + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "79cf8c10-c40b-4f3a-8c88-c3961e3882fb" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 12, + "status": "success", + "tool_call_id": "toolu_0121DiG2dcAjwMFc9MmhTwmR", + "tool_name": "switch_tab" + } + }, + "at_ms": 13 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 1087, + "status": "success", + "tool_call_id": "toolu_014y3jiwHxLmhhhnNtrx1FSz", + "tool_name": "print_workflow" + } + }, + "at_ms": 3440 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2785690574723683, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 2785690574723683, + "class_type": "CLIPTextEncode" + }, + { + "op": "connect", + "link_id": 3432143374160979, + "to_node": 2785690574723683, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + } + ], + "at_ms": 13977 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 567, + "status": "success", + "tool_call_id": "toolu_017yZpv6KmDDDbG6dxLL9YtE", + "tool_name": "apply_ops" + } + }, + "at_ms": 13977 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 552, + "status": "success", + "tool_call_id": "toolu_01C6dAj6DS4pZZCs2hqkFzeg", + "tool_name": "print_workflow" + } + }, + "at_ms": 17960 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now, and the second text encode node is in place, taking its text understanding from the same checkpoint loader as the first one.\n\nTwo things worth flagging, since I only did exactly what you asked:\n\n- Its text box is empty, and it isn't connected onward to anything yet — most likely you want it as the negative prompt into the sampler, whose negative input is currently empty.\n- That sampler is also missing its image-size input, so the workflow won't run as it stands regardless of this new node.\n\nSay the word and I'll wire the new node in as the negative prompt (with whatever text you want) and add the missing size input so it's ready to run." + } + }, + "at_ms": 22703 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1934, + "cache_read_input_tokens": 192393, + "input_tokens": 10, + "output_tokens": 1057, + "total_tokens": 1067 + } + } + }, + "at_ms": 22708 + } + ] + }, + { + "message_id": "9be1516f-95b5-47b2-b939-46c60f141da7", + "request": { + "content": "Set the text on that new node to 'blurry, low quality'." + }, + "response": [ + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": null, + "value": "blurry, low quality", + "widget": "text", + "node_id": 2785690574723683 + } + ], + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 687, + "status": "success", + "tool_call_id": "toolu_013VjTaSsWFeoXEkR6AUb9Gb", + "tool_name": "set_widget" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the new node now reads \"blurry, low quality\". It's still not wired into the sampler, so it won't affect a render until I connect it as the negative prompt. Want me to do that?" + } + }, + "at_ms": 2300 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 993, + "cache_read_input_tokens": 76366, + "input_tokens": 4, + "output_tokens": 142, + "total_tokens": 146 + } + } + }, + "at_ms": 2304 + } + ] + }, + { + "message_id": "0b966fe0-ac97-4506-abbc-743051b5c526", + "request": { + "content": "Now delete that node again." + }, + "response": [ + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "node_id": 2785690574723683, + "removed_links": [3432143374160979] + } + ], + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 561, + "status": "success", + "tool_call_id": "toolu_01VeBQyvK7MXKuSEjCGrmvK4", + "tool_name": "delete_node" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Removed — the canvas is back to the five original nodes." + } + }, + "at_ms": 2811 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 698, + "cache_read_input_tokens": 76763, + "input_tokens": 4, + "output_tokens": 96, + "total_tokens": 100 + } + } + }, + "at_ms": 2814 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json new file mode 100644 index 00000000000..afc63d41698 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json @@ -0,0 +1,368 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-batched-ops", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread a6fd73be-e957-45b7-a5e0-5edd091670fb; message 9f88da92-5074-4091-850b-7cbc65005cd1; workflow e539bb63-bc5d-4f97-b918-d85862c0f591 (seeded by throwaway turn 365de61d-2131-4966-af0b-fe7ccda288bf; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ae918468-c270-4d05-b0ef-31198a63cf0e', '069a03fa-3094-4fa4-821d-797672439db7', '044e431e-059e-45b1-b82d-45e25a92f2bf']; rows agent-rec-batched-ops.t-a1.rows.json; raw capture sha256 92757967c82950945ea1a183dbcfcb8d111ae768b3ea16a72348bb0e216b709e", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "a6fd73be-e957-45b7-a5e0-5edd091670fb", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "e539bb63-bc5d-4f97-b918-d85862c0f591", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "9f88da92-5074-4091-850b-7cbc65005cd1", + "request": { + "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "e539bb63-bc5d-4f97-b918-d85862c0f591" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 7, + "status": "success", + "tool_call_id": "toolu_01UJ5hWx9Rxkvwon9F7nMGaK", + "tool_name": "switch_tab" + } + }, + "at_ms": 2 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_011QfP8raZRXJbK5WZYpKTkv", + "tool_name": "print_workflow" + } + }, + "at_ms": 2139 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2559261621971657, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 2559261621971657, + "class_type": "CLIPTextEncode" + }, + { + "op": "add_node", + "pos": [715, 406], + "node": { + "id": 2214070360209981, + "pos": [715, 406], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 2214070360209981, + "class_type": "CLIPTextEncode" + }, + { + "op": "connect", + "link_id": 879068618000012, + "to_node": 2559261621971657, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + }, + { + "op": "connect", + "link_id": 3832075311322831, + "to_node": 2214070360209981, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + } + ], + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01PhJ8AyewzRU6PHk7aFGfbC", + "tool_name": "apply_ops" + } + }, + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt." + } + }, + "at_ms": 7784 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1437, + "cache_read_input_tokens": 153211, + "input_tokens": 8, + "output_tokens": 574, + "total_tokens": 582 + } + } + }, + "at_ms": 7786 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-cancelled-turn.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-cancelled-turn.json new file mode 100644 index 00000000000..2bfee054f00 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-cancelled-turn.json @@ -0,0 +1,334 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-cancelled-turn", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8087 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread d9e8410c-563d-43d4-be7e-d0020023d0e2; messages 54844cd9-fd59-4c46-a271-58af8bb87403; workflow 18e8fe8a-8fd3-4fa0-bf55-4437ada01f52 (seeded by throwaway turn 30151cd9-31ec-41ba-92f3-39c2dadfb6ea; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 71f16f5f-3bb5-4cbe-95d6-59f615ad4384, 8422cdee-e5be-4899-8ad8-574900174c12, f3be238b-83f9-4f86-920f-da847b990c41; rows agent-rec-cancelled-turn.t2.rows.1.json; raw capture sha256 eb5705cde75b76dff0ba93403bc661f85d8aae7cc0a10a2d307c9bf740486cb5", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "d9e8410c-563d-43d4-be7e-d0020023d0e2", + "exported_at": "2026-09-03T16:42:17.010Z" + } + }, + "workflow": { + "id": "18e8fe8a-8fd3-4fa0-bf55-4437ada01f52", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "54844cd9-fd59-4c46-a271-58af8bb87403", + "request": { + "content": "Switch to the tab 'Text to image', then add three separate CLIPTextEncode nodes one at a time, wiring each one to the checkpoint loader's CLIP output as you go." + }, + "cancel_after": 6, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first, then inspect the graph to find the checkpoint loader." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "18e8fe8a-8fd3-4fa0-bf55-4437ada01f52" + } + }, + "at_ms": 10 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01Wy5EjYokiTEtfYMBWaXiSX", + "tool_name": "switch_tab" + } + }, + "at_ms": 13 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 472, + "status": "success", + "tool_call_id": "toolu_01TH5jYJncr9pdyeaTC3DNfc", + "tool_name": "ls_nodes" + } + }, + "at_ms": 3059 + }, + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "The checkpoint loader is node 4. I'll add the first node node and wire it to the loader's CLIP output." + } + }, + "at_ms": 5482 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1845, 0], + "node": { + "id": 729482957923442, + "pos": [1845, 0], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 729482957923442, + "class_type": "CLIPTextEncode" + } + ], + "at_ms": 5927 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 437, + "status": "success", + "tool_call_id": "toolu_018huMdgs7tpu1NfVwYVMgNh", + "tool_name": "add_node" + } + }, + "at_ms": 5927 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Stopped at your request. Before stopping I made 1 edit to your canvas." + } + }, + "at_ms": 6821 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": null + } + }, + "at_ms": 6823 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-clarifying-question.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-clarifying-question.json new file mode 100644 index 00000000000..b957db9d6d4 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-clarifying-question.json @@ -0,0 +1,378 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-clarifying-question", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 381048aa-cb22-412d-bf2e-c16ad80e06fa; messages 7840b6d9-6ff2-4a68-bd13-61b3a034fa64, 27548445-c720-4c52-b70d-e91ba8e39d84; workflow b950700b-5321-4486-be99-a3183c04fdd6 (seeded by throwaway turn 21cc3da4-3ec9-4a98-9dff-078dccdc203c; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 002c548d-a946-421a-a3a0-b07498eaf398, bb6e0c45-fee8-44ae-9cf0-f7be4a13ace1, bd0b33ed-e8f7-4e1a-9806-d4f48f646a2e, c45c5ccf-2d33-45b1-b735-3e62a8531c48, ceec4e8f-8008-4fa2-b74b-f9d9f9ef0a11, d197b92d-3788-43d9-b054-fa216de9e262; rows agent-rec-clarifying-question.f1.rows.1.json, agent-rec-clarifying-question.f1.rows.2.json; raw capture sha256 145e059591ebff7d96d0423832525d16e9492d34ed4405cebf531f2587b882bd", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "381048aa-cb22-412d-bf2e-c16ad80e06fa", + "exported_at": "2026-09-03T16:15:02.197Z" + } + }, + "workflow": { + "id": "b950700b-5321-4486-be99-a3183c04fdd6", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "7840b6d9-6ff2-4a68-bd13-61b3a034fa64", + "request": { + "content": "Switch to the tab 'Text to image', then change the sampler settings to what I usually use." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b950700b-5321-4486-be99-a3183c04fdd6" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01D7DvH5K6Le4AHND8CpE67L", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 485, + "status": "success", + "tool_call_id": "toolu_014TjmyKK7o5K5KvknkdnyNg", + "tool_name": "list_workflows" + } + }, + "at_ms": 495 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 651, + "status": "success", + "tool_call_id": "toolu_01XmVQKJjiMPr5EKeqZAP4DV", + "tool_name": "print_workflow" + } + }, + "at_ms": 5050 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now, but I don't have any record of your usual sampler settings — nothing saved from earlier in our conversation, and this is the first workflow I've seen from you. Rather than guess and quietly change your image, tell me the values and I'll set them (and remember them for next time).\n\nRight now the sampler is on:\n\n- **Steps:** 20\n- **CFG:** 7\n- **Sampler:** euler\n- **Scheduler:** normal\n- **Denoise:** 1.0\n- **Seed:** 0, randomized each run\n\nJust say what you want them changed to — e.g. \"30 steps, cfg 4.5, dpmpp_2m karras\" — or tell me the name of a workflow where I can read them off, and I'll copy them across.\n\nOne other thing worth flagging: this graph is missing a negative prompt and an image-size input into the sampler, so it won't run as-is. I can add both while I'm in there." + } + }, + "at_ms": 15015 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1107, + "cache_read_input_tokens": 114403, + "input_tokens": 6, + "output_tokens": 1188, + "total_tokens": 1194 + } + } + }, + "at_ms": 15024 + } + ] + }, + { + "message_id": "27548445-c720-4c52-b70d-e91ba8e39d84", + "request": { + "content": "I usually use 30 steps. Please set that." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 410, + "status": "success", + "tool_call_id": "toolu_015dGdCUUtvrvn6mmPnmJFqf", + "tool_name": "list_slots" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 4, + "status": "success", + "tool_call_id": "toolu_01BWJuBc6xzadt7MaMEyRWRR", + "tool_name": "remember" + } + }, + "at_ms": 5 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "value": 30, + "widget": "steps", + "node_id": 3 + } + ], + "at_ms": 2167 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_018mxuaGyTs3SbYEqAm7fCR1", + "tool_name": "set_widget" + } + }, + "at_ms": 2167 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Steps are now 30, and I've noted that as your default for next time.\n\nThe graph still won't run as-is — the sampler has no negative prompt or image-size input wired in. Want me to add those?" + } + }, + "at_ms": 4544 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1485, + "cache_read_input_tokens": 115445, + "input_tokens": 6, + "output_tokens": 507, + "total_tokens": 513 + } + } + }, + "at_ms": 4549 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json new file mode 100644 index 00000000000..2cfd5baf674 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json @@ -0,0 +1,420 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-refiner-between", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a; message b363781f-6c9a-4087-9190-d478781bbcd2; workflow b7d1e09f-682a-48f5-a388-d74a33ad5184 (seeded by throwaway turn 52d56bcf-7eb0-4131-a287-223e0f15e2e9; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2f4c246a-f0c7-4ca4-ad48-814bc2fe9ab6', '9727d8d7-19f4-48eb-98e6-5c054d3d6cf0', '22b43a2d-5fe8-49c0-8339-e2b3b52bf184', '33953182-42ff-46a6-a0d5-313c5a55d630', '349abb83-7fed-424e-8fb5-dc94e412aa90']; rows agent-rec-refiner-between.t-a1.rows.json; raw capture sha256 242b1c822b8b7697bdcdf1448030181ef0bb1049806870e9075077ca44897768", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "b7d1e09f-682a-48f5-a388-d74a33ad5184", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "b363781f-6c9a-4087-9190-d478781bbcd2", + "request": { + "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b7d1e09f-682a-48f5-a388-d74a33ad5184" + } + }, + "at_ms": 5 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 5, + "status": "success", + "tool_call_id": "toolu_01Bq7cRyMamrEnmo4yHuMphE", + "tool_name": "switch_tab" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 747, + "status": "success", + "tool_call_id": "toolu_01FEPdyTv3FtKtL8S6zTKj33", + "tool_name": "print_workflow" + } + }, + "at_ms": 3053 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1565, 280], + "node": { + "id": 2092962753103866, + "pos": [1565, 280], + "mode": 0, + "size": [240, 290], + "type": "KSampler", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "model", + "type": "MODEL" + }, + { + "link": null, + "name": "positive", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "negative", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "latent_image", + "type": "LATENT" + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + }, + "node_id": 2092962753103866, + "class_type": "KSampler" + }, + { + "op": "connect", + "link_id": 4414751122447181, + "to_node": 2092962753103866, + "to_slot": 3, + "from_node": 3, + "from_slot": 0, + "link_type": "LATENT" + }, + { + "op": "connect", + "link_id": 863234800411381, + "to_node": 2092962753103866, + "to_slot": 0, + "from_node": 4, + "from_slot": 0, + "link_type": "MODEL" + }, + { + "op": "connect", + "link_id": 3220756471207314, + "to_node": 2092962753103866, + "to_slot": 1, + "from_node": 6, + "from_slot": 0, + "link_type": "CONDITIONING" + }, + { + "op": "connect", + "link_id": 619699595909791, + "to_node": 8, + "to_slot": 0, + "from_node": 2092962753103866, + "from_slot": 0, + "link_type": "LATENT" + }, + { + "op": "set_widget", + "old": 1, + "value": 0.5, + "widget": "denoise", + "node_id": 2092962753103866 + }, + { + "op": "set_widget", + "old": 20, + "value": 20, + "widget": "steps", + "node_id": 2092962753103866 + } + ], + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 198, + "status": "success", + "tool_call_id": "toolu_01LqYJ1bi41MTaeDXdzC9A9j", + "tool_name": "apply_ops" + } + }, + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 379, + "status": "success", + "tool_call_id": "toolu_01DAfeiUo97WETr7TRCrv4Cp", + "tool_name": "print_workflow" + } + }, + "at_ms": 14468 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_01G8xCCA9NuoEur1n8RySCWK", + "tool_name": "validate" + } + }, + "at_ms": 19339 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node.\n\nTwo things were already unfinished in this graph before I touched it, and they'll block a run:\n\n- Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler.\n- The checkpoint it points at isn't installed here, so no model file is actually available.\n\nI set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?" + } + }, + "at_ms": 29476 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2906, + "cache_read_input_tokens": 232638, + "input_tokens": 12, + "output_tokens": 1422, + "total_tokens": 1434 + } + } + }, + "at_ms": 29482 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json new file mode 100644 index 00000000000..7e669851e28 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json @@ -0,0 +1,391 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-replace-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2ce25f4b-d581-4348-ab9c-3f4170222f6f; message 37504547-e90d-4c14-91c7-b8f8fb65a0eb; workflow 5a6b35c4-8902-479d-87ab-8a9a07f08b90 (seeded by throwaway turn 3add9fbe-1dbb-4d5f-b255-d0fde410ed5e; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d6e50c4a-5ef2-46bb-a48d-46473a36e13f', '6471d9bc-2e6f-41f1-b2c9-b417e7b9ecea', '3bda2337-3236-4585-9803-f8cbc1625d0a', '7cba8d9d-ebe4-481f-82e9-354377d2826e', '9d9678dd-2f0c-47eb-98da-0ec3720ff91d']; rows agent-rec-replace-prompt-encoder.t-a1.rows.json; raw capture sha256 eabe622a684331aa98da2a1a797cf0f761e3c866bcce8feaf2e1083a6fb7ff26", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "2ce25f4b-d581-4348-ab9c-3f4170222f6f", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "37504547-e90d-4c14-91c7-b8f8fb65a0eb", + "request": { + "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 10, + "status": "success", + "tool_call_id": "toolu_01PbXSHPoAUnHhX2dqzJXiLB", + "tool_name": "switch_tab" + } + }, + "at_ms": 11 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 295, + "status": "success", + "tool_call_id": "toolu_01C2g2RACFUykeibS1je1Tbh", + "tool_name": "print_workflow" + } + }, + "at_ms": 3673 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 4181654812796082, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 4181654812796082, + "class_type": "CLIPTextEncode" + }, + { + "op": "connect", + "link_id": 3320191292493608, + "to_node": 4181654812796082, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + }, + { + "op": "connect", + "link_id": 2686273519871711, + "to_node": 3, + "to_slot": 1, + "from_node": 4181654812796082, + "from_slot": 0, + "link_type": "CONDITIONING" + }, + { + "op": "set_widget", + "old": null, + "value": "a photo of a pier", + "widget": "text", + "node_id": 4181654812796082 + } + ], + "at_ms": 10451 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 223, + "status": "success", + "tool_call_id": "toolu_018Ao8cDcxmDBaz9qV7PWpGx", + "tool_name": "apply_ops" + } + }, + "at_ms": 10451 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "node_id": 6, + "removed_links": [1] + } + ], + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 202, + "status": "success", + "tool_call_id": "toolu_01QkN5N4e8fkpBrT9aWbb4aK", + "tool_name": "delete_node" + } + }, + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01QUTSXXBfU7oShpWRkxtLzD", + "tool_name": "print_workflow" + } + }, + "at_ms": 18022 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm on the \"Text to image\" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler's positive input. I carried over the same prompt text (\"a photo of a pier\").\n\nOne thing unrelated to this swap: the sampler's negative prompt and empty latent inputs were already unconnected before I started, so the graph won't run until those are filled in. Want me to add them?" + } + }, + "at_ms": 22947 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2216, + "cache_read_input_tokens": 232121, + "input_tokens": 12, + "output_tokens": 885, + "total_tokens": 897 + } + } + }, + "at_ms": 22951 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json new file mode 100644 index 00000000000..3c900f8532a --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json @@ -0,0 +1,338 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-set-widget-existing", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread b5d11187-d40e-4be1-aeab-39869516773b; message 19230873-0f07-4f85-a0ae-e9f1d740cb3a; workflow 5733b17e-39f4-40f5-aa1c-468a4f355b55 (seeded by throwaway turn d00f4278-8e50-4671-a49b-8a237f1797b4; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2383f8e0-be34-4bb1-88c5-bbe0fd34ee95', '2b9744df-c52a-4b45-b0e8-96b64642075d', 'c93c9e49-1d3d-4507-9c3d-481a0910d2c4', 'c2b5e940-a8f3-49eb-b043-e6f8360c686c', 'b538294c-6add-4ebe-b351-c09cce102a0f']; rows agent-rec-set-widget-existing.t3-a1.rows.json; raw capture sha256 6a5a92d256bb83cfd9fb95be1f182300da2361bb15a4325ce2365de6f79ad0a9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "b5d11187-d40e-4be1-aeab-39869516773b", + "exported_at": "2026-09-03T12:38:52Z" + } + }, + "workflow": { + "id": "5733b17e-39f4-40f5-aa1c-468a4f355b55", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "19230873-0f07-4f85-a0ae-e9f1d740cb3a", + "request": { + "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5733b17e-39f4-40f5-aa1c-468a4f355b55" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 12, + "status": "success", + "tool_call_id": "toolu_0127MABi8i6cpznwzfMSk2b6", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 498, + "status": "success", + "tool_call_id": "toolu_01DTMkDXHKWELHfKUTXUynfK", + "tool_name": "ls_nodes" + } + }, + "at_ms": 3903 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 425, + "status": "success", + "tool_call_id": "toolu_018Ck46iqTkvLjT12cDdcyH6", + "tool_name": "list_slots" + } + }, + "at_ms": 9834 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "value": 30, + "widget": "steps", + "node_id": 3 + }, + { + "op": "set_widget", + "old": 7, + "value": 5, + "widget": "cfg", + "node_id": 3 + } + ], + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 253, + "status": "success", + "tool_call_id": "toolu_01BaTP7o2iAhJbBjQ9tChoEG", + "tool_name": "apply_ops" + } + }, + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01MLp2wvSBojXfwPYKqT7dvB", + "tool_name": "list_slots" + } + }, + "at_ms": 21478 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "You're now on the \"Text to image\" tab, with steps at 30 and cfg at 5. Nothing else was touched." + } + }, + "at_ms": 23927 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1549, + "cache_read_input_tokens": 230832, + "input_tokens": 12, + "output_tokens": 509, + "total_tokens": 521 + } + } + }, + "at_ms": 23931 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json new file mode 100644 index 00000000000..66fc1ca04a2 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json @@ -0,0 +1,279 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-text-only-answer", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread f36ef047-82b4-4f2b-b480-6df821df46da; message 7de6039c-c1b4-437c-871c-2f1fc67e1750; workflow d762837d-c64d-4829-9aba-05e71b64adc3 (seeded by throwaway turn c3727b7b-569e-4eda-983a-7ccfffbf4496; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['7622ee7b-b97a-4899-8931-99488b82bee5', '5547227c-9bef-4e95-9bd1-053fb658e7b4']; rows agent-rec-text-only-answer.t-a1.rows.json; raw capture sha256 24906bcf7826bc7860dd7b109ce8d243a1c6322f56a646ea53e5645cc795c387", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "f36ef047-82b4-4f2b-b480-6df821df46da", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "d762837d-c64d-4829-9aba-05e71b64adc3", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "7de6039c-c1b4-437c-871c-2f1fc67e1750", + "request": { + "content": "Switch to the tab 'Text to image', then tell me what this workflow does." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d762837d-c64d-4829-9aba-05e71b64adc3" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_013WLWbbrNwTqV9UguY6rEu6", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_01PjsdjzNXWZ8oDYHQAbjo5o", + "tool_name": "print_workflow" + } + }, + "at_ms": 2364 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now.\n\nIt's a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently \"a photo of a pier\" — into guidance, generates an image over 20 steps, and saves the result.\n\nOne thing to flag: it isn't runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?" + } + }, + "at_ms": 6244 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 960, + "cache_read_input_tokens": 114249, + "input_tokens": 6, + "output_tokens": 349, + "total_tokens": 355 + } + } + }, + "at_ms": 6246 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json new file mode 100644 index 00000000000..3e536d53c65 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json @@ -0,0 +1,404 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-three-sequential-adds", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 3710f44b-9941-4bc4-a806-5c15d00cd807; message d0d6b5dc-b189-491a-804e-f52fdb9495cb; workflow 7afd1f94-6cb5-4923-9388-18851b0fba5d (seeded by throwaway turn 9285739d-3bbe-475f-a681-6f9761c7a621; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['c27f6e4e-220a-42fc-8fd1-97c30d6d3c98', 'f020586d-9444-457f-85e9-8c96b2be6292', '3832f4c1-77c3-44a9-b890-83b82db5513a', 'df9c490c-5a9a-43b3-b6d7-931c40084eaf']; rows agent-rec-three-sequential-adds.cc-a1.rows.json; raw capture sha256 6a1dfc3c7d4961130ecc8e1b96edd76c9ec288602e38d86432b16e2ae3d342b6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "3710f44b-9941-4bc4-a806-5c15d00cd807", + "exported_at": "2026-09-03T14:05:21Z" + } + }, + "workflow": { + "id": "7afd1f94-6cb5-4923-9388-18851b0fba5d", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "d0d6b5dc-b189-491a-804e-f52fdb9495cb", + "request": { + "content": "Switch to the tab 'Text to image', then add three nodes one at a time, each with its own separate tool call and nothing else in that call: first a PrimitiveStringMultiline, then an EmptyLatentImage, then a CLIPTextEncode. Do not connect or configure them." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch focus to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "7afd1f94-6cb5-4923-9388-18851b0fba5d" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_0199bXc7oQVbgJA4aY56VcfY", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1845, 0], + "node": { + "id": 2478057798252548, + "pos": [1845, 0], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 2478057798252548, + "class_type": "PrimitiveStringMultiline" + } + ], + "at_ms": 2686 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 964, + "status": "success", + "tool_call_id": "toolu_014VjtpNeXB9q2rBXxK9bkm9", + "tool_name": "add_node" + } + }, + "at_ms": 2686 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2165, 0], + "node": { + "id": 2772376668635982, + "pos": [2165, 0], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "node_id": 2772376668635982, + "class_type": "EmptyLatentImage" + } + ], + "at_ms": 4441 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 220, + "status": "success", + "tool_call_id": "toolu_01M6Jpuof94JLfgBij4etkaN", + "tool_name": "add_node" + } + }, + "at_ms": 4441 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2485, 0], + "node": { + "id": 2514973844700532, + "pos": [2485, 0], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 2514973844700532, + "class_type": "CLIPTextEncode" + } + ], + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 192, + "status": "success", + "tool_call_id": "toolu_01FzkN53SHp6ePAFbmX2H8oX", + "tool_name": "add_node" + } + }, + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "All three nodes are on the \"Text to image\" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked." + } + }, + "at_ms": 9873 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1944, + "cache_read_input_tokens": 192480, + "input_tokens": 10, + "output_tokens": 322, + "total_tokens": 332 + } + } + }, + "at_ms": 9875 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json new file mode 100644 index 00000000000..0508a4b8c07 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json @@ -0,0 +1,280 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-tool-error", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386; message a5ec5ead-1909-4322-9d62-edbb03a3ded9; workflow 5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5 (seeded by throwaway turn 788de1fc-829b-4636-b715-2879479effbc; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['39a22dd2-4aaa-4d7d-9505-779ab6386ee8', 'aeb6b2f3-8d0c-4131-a913-144322ca42c8']; rows agent-rec-tool-error.t-a1.rows.json; raw capture sha256 83bd2c2c30c00883df39527e5f482deccc5e10c34a0d6554c34bbff8d725c0e6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "a5ec5ead-1909-4322-9d62-edbb03a3ded9", + "request": { + "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01YWm16Ejys8FJBofYwL7k9E", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 346, + "error_code": "node_not_found", + "status": "error", + "tool_call_id": "toolu_01XqvtmCXKboog1UTZ3MGNJR", + "tool_name": "add_node" + } + }, + "at_ms": 2654 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm now on the \"Text to image\" tab, but there's no node called \"DoesNotExist\" — nothing was added, and the canvas is unchanged.\n\nIf you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I'll find the real one and wire it in." + } + }, + "at_ms": 5701 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 752, + "cache_read_input_tokens": 114291, + "input_tokens": 6, + "output_tokens": 295, + "total_tokens": 301 + } + } + }, + "at_ms": 5703 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json new file mode 100644 index 00000000000..ca9c3198759 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json @@ -0,0 +1,433 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-two-turn-dependent-edit", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6cafa19d-11f0-45e9-bcf1-e52529eb9ebb; messages 445f7340-ea9c-4456-8ffa-5d37ea6c7b98, 07fc770a-df43-42d8-8f2c-07bf9c2db9f8; workflow d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447 (seeded by throwaway turn af193614-a407-4987-99a5-97a04d52eb35; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 42fbcd91-26cb-415a-b471-8daa44a8bd6c, 55c090e2-11f7-48b7-93dc-5fa9fd73e1ac, abbd53d0-2352-4d5d-bfd0-0bd2601a23ce, b5bde96e-939a-4b5b-992d-5fc9ece40a96, da2d0551-1fcb-435d-9dd1-745a451d75f9, f7084475-8dd0-4789-a14f-0a2cd47379dc; rows agent-rec-two-turn-dependent-edit.m1.rows.1.json, agent-rec-two-turn-dependent-edit.m1.rows.2.json; raw capture sha256 49a64b8ac694b4778b78b3df03a372eaed59e60baca70299ee5cee867b23be43", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "6cafa19d-11f0-45e9-bcf1-e52529eb9ebb", + "exported_at": "2026-09-03T16:03:42.552Z" + } + }, + "workflow": { + "id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "445f7340-ea9c-4456-8ffa-5d37ea6c7b98", + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and give it the text 'blurry, low quality'." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at the graph first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01WdrFCAxyg6dbYLfwVHfPTD", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 752, + "status": "success", + "tool_call_id": "toolu_01E4tYHkXrhtb6ktKRki4z4r", + "tool_name": "print_workflow" + } + }, + "at_ms": 2435 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3337990138003756, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "node_id": 3337990138003756, + "class_type": "CLIPTextEncode" + }, + { + "op": "connect", + "link_id": 2557345504960691, + "to_node": 3337990138003756, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP" + }, + { + "op": "set_widget", + "old": null, + "value": "blurry, low quality", + "widget": "text", + "node_id": 3337990138003756 + } + ], + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 195, + "status": "success", + "tool_call_id": "toolu_013FPeYq4Ya1M9VJKsMPJE4j", + "tool_name": "apply_ops" + } + }, + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 196, + "status": "success", + "tool_call_id": "toolu_01RL1yma6QfBbL3cZQs3qsf6", + "tool_name": "print_workflow" + } + }, + "at_ms": 11731 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now, and the second text box is added with \"blurry, low quality\" and fed from the checkpoint loader.\n\nOne thing worth flagging: it isn't connected to anything downstream yet, so it has no effect on the image. The sampler's negative input is currently empty (and its latent input too) — say the word and I'll wire this new text into the negative slot." + } + }, + "at_ms": 15233 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2013, + "cache_read_input_tokens": 192537, + "input_tokens": 10, + "output_tokens": 822, + "total_tokens": 832 + } + } + }, + "at_ms": 15235 + } + ] + }, + { + "message_id": "07fc770a-df43-42d8-8f2c-07bf9c2db9f8", + "request": { + "content": "Now connect that new CLIPTextEncode node to the KSampler's negative input." + }, + "response": [ + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "link_id": 1795569972720099, + "to_node": 3, + "to_slot": 2, + "from_node": 3337990138003756, + "from_slot": 0, + "link_type": "CONDITIONING" + } + ], + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 298, + "status": "success", + "tool_call_id": "toolu_016EPZQoQXsKsbm5qAzKPHGD", + "tool_name": "connect" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01RZH4jfk5SdpGS1xF2dMfvw", + "tool_name": "print_workflow" + } + }, + "at_ms": 4272 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Connected — the \"blurry, low quality\" text now feeds the negative input.\n\nThe graph still won't run as-is: nothing is feeding the sampler's latent input, so it needs an empty latent (which is also where the image size gets set). Want me to add one?" + } + }, + "at_ms": 8606 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1542, + "cache_read_input_tokens": 115114, + "input_tokens": 6, + "output_tokens": 376, + "total_tokens": 382 + } + } + }, + "at_ms": 8610 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json new file mode 100644 index 00000000000..0319b22d23b --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -0,0 +1,408 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-workflow-editing-05", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3; message b199d9ab-1800-49b2-8a43-458cda7f104d; workflow f47e55ab-2cd5-414e-90d3-23a5fc7eee22 (seeded by throwaway turn 3d61dd80-dd03-4957-8506-734392c9583c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['6206d0ef-f781-41db-81bd-4138950ccd2e', 'fb873a4b-59b4-4088-ae32-f43e0a7382c5', '473ea6be-f9c3-4a75-8776-0e7823cd5054', 'e6ec2a79-8f0b-408a-b01e-f9362ae8fde4', '47d7581e-2d55-49ea-ac3e-1d6520d80454', 'abd6c054-6b5b-49a2-8633-881078e9b07c', '3f83a246-0181-4d1a-b2b5-c1fcfac38fe0']; rows agent-workflow-editing-05.t-a1.rows.json; raw capture sha256 982353ccbf5d92ca70a7857eb7da662aa2d19b0fcebb27181a71c3a95b762658", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22", + "name": "Image edit", + "catalog": { + "types": { + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "VAEEncode": { + "widget_order": [] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "ImageStitch": { + "widget_order": [ + "direction", + "match_image_size", + "spacing_width", + "spacing_color" + ] + }, + "PreviewImage": { + "widget_order": [] + } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [1] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "VAEEncode", + "pos": [400, 0], + "size": [210, 46], + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 1 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2] + } + ], + "widgets_values": [] + }, + { + "id": 3, + "type": "KSampler", + "pos": [700, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [3] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1100, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 3 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [4] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1400, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 1, 0, 2, 0, "IMAGE"], + [2, 2, 0, 3, 3, "LATENT"], + [3, 3, 0, 8, 0, "LATENT"], + [4, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "b199d9ab-1800-49b2-8a43-458cda7f104d", + "request": { + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Image edit", + "workflow_id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_01RMDkomGUUpDL8wixWq9Fhe", + "tool_name": "switch_tab" + } + }, + "at_ms": 14 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 366, + "status": "success", + "tool_call_id": "toolu_01JECLyu71YJ1niLhepMB63u", + "tool_name": "print_workflow" + } + }, + "at_ms": 2905 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 194, + "status": "success", + "tool_call_id": "toolu_01Sttwp3yTjiF6LdrL5GdmVy", + "tool_name": "find_nodes" + } + }, + "at_ms": 7034 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1710, 280], + "node": { + "id": 1177948620476144, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], + "type": "ImageStitch", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, + "widgets_values": ["right", true, 0, "white"] + }, + "node_id": 1177948620476144, + "class_type": "ImageStitch" + }, + { + "op": "connect", + "link_id": 3298141227647238, + "to_node": 1177948620476144, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE" + }, + { + "op": "connect", + "link_id": 2572689854678654, + "to_node": 1177948620476144, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE" + }, + { + "op": "connect", + "link_id": 3588278739910384, + "to_node": 9, + "to_slot": 0, + "from_node": 1177948620476144, + "from_slot": 0, + "link_type": "IMAGE" + } + ], + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 208, + "status": "success", + "tool_call_id": "toolu_01DNrpeJQDjW2v32xme5eyDq", + "tool_name": "apply_ops" + } + }, + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 210, + "status": "success", + "tool_call_id": "toolu_01VFMK15TdDdT4zFK3w8bRqE", + "tool_name": "list_slots" + } + }, + "at_ms": 20740 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 206, + "error_code": "conflict", + "status": "error", + "tool_call_id": "toolu_01GZUZLG9RtoeGym25Hy5p5P", + "tool_name": "set_widget" + } + }, + "at_ms": 25170 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 216, + "status": "error", + "tool_call_id": "toolu_01FCgU1tuBtusNgYiPMRaUQL", + "tool_name": "validate" + } + }, + "at_ms": 29613 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved.\n\nTwo things worth knowing:\n\n- The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider.\n- The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?" + } + }, + "at_ms": 38675 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 3425, + "cache_read_input_tokens": 313028, + "input_tokens": 16, + "output_tokens": 1658, + "total_tokens": 1674 + } + } + }, + "at_ms": 38677 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/helpers/CloudWorkspaceMockHelper.ts b/browser_tests/fixtures/helpers/CloudWorkspaceMockHelper.ts index d5d5473a894..8de7f84921a 100644 --- a/browser_tests/fixtures/helpers/CloudWorkspaceMockHelper.ts +++ b/browser_tests/fixtures/helpers/CloudWorkspaceMockHelper.ts @@ -11,7 +11,6 @@ import type { } from '@/platform/workspace/api/workspaceApi' import { createWorkspaceBillingCapabilities } from '@e2e/fixtures/data/billingCapabilities' -import { mockSystemStats } from '@e2e/fixtures/data/systemStats' import { CLOUD_REMOTE_CONFIG, DEFAULT_TEAM_MEMBERS, @@ -21,6 +20,8 @@ import { } from '@e2e/fixtures/data/cloudWorkspace' import { CloudAuthHelper } from '@e2e/fixtures/helpers/CloudAuthHelper' import { TestIds } from '@e2e/fixtures/selectors' +import { mockCloudBootRoutes } from '@e2e/fixtures/utils/cloudBootMocks' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' import { mockWorkspaceTokenMint } from '@e2e/fixtures/utils/workspaceMocks' interface RoleChangeRequest { @@ -33,12 +34,6 @@ interface MemberMockState { patches: RoleChangeRequest[] } -const jsonRoute = (body: unknown) => ({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(body) -}) - /** * Boots the cloud app against fully mocked workspace + billing endpoints so * member/role specs can drive a raw `page` (the `comfyPage` fixture would try @@ -106,42 +101,18 @@ export class CloudWorkspaceMockHelper { } const { page } = this - await page.route('**/api/features', (r) => - r.fulfill(jsonRoute(CLOUD_REMOTE_CONFIG)) - ) - await page.route('**/api/system_stats', (r) => - r.fulfill(jsonRoute(mockSystemStats)) - ) - await page.route('**/api/users', (r) => - r.fulfill( - jsonRoute({ - storage: 'server', - migrated: true, - users: { 'test-user-e2e': 'E2E Test User' } - }) - ) - ) - // A non-empty settings payload with TutorialCompleted marks the user as - // returning, so the new-user Templates dialog never auto-opens to block the - // Settings button. Errors tab off suppresses the model-folder 401 toast. - await page.route('**/api/settings', (r) => - r.fulfill( - jsonRoute({ - 'Comfy.TutorialCompleted': true, - 'Comfy.RightSidePanel.ShowErrorsTab': false - }) - ) - ) - await page.route('**/api/userdata**', (r) => r.fulfill(jsonRoute([]))) - await page.route('**/api/extensions', (r) => r.fulfill(jsonRoute([]))) - await page.route('**/api/object_info', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/global_subgraphs', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/i18n', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/auth/session', (r) => - r.fulfill(jsonRoute({ token: 'mock-workspace-token' })) - ) + await mockCloudBootRoutes(page, { + features: CLOUD_REMOTE_CONFIG, + // A non-empty settings payload with TutorialCompleted marks the user as + // returning, so the new-user Templates dialog never auto-opens to block + // the Settings button. Errors tab off suppresses the model-folder 401 + // toast. + settings: { + 'Comfy.TutorialCompleted': true, + 'Comfy.RightSidePanel.ShowErrorsTab': false + } + }) await mockWorkspaceTokenMint(page, activeWorkspace) - await page.route('**/releases**', (r) => r.fulfill(jsonRoute([]))) await page.route('**/api/workspaces', (r) => r.fulfill(jsonRoute({ workspaces: [activeWorkspace] })) diff --git a/browser_tests/fixtures/utils/cloudBootMocks.ts b/browser_tests/fixtures/utils/cloudBootMocks.ts index f6970bf3a33..ab0b38e1bbf 100644 --- a/browser_tests/fixtures/utils/cloudBootMocks.ts +++ b/browser_tests/fixtures/utils/cloudBootMocks.ts @@ -12,16 +12,18 @@ interface CloudBootOptions { features: RemoteConfig /** Body for `/api/settings` (defaults to `{}`). */ settings?: unknown + /** `'server'` lets `/api/object_info` reach the backend so real node definitions load. */ + objectInfo?: 'server' } /** - * Stub the core endpoints the cloud app hits on boot so a raw `page` reaches the - * working app without falling through to the OSS devtools backend. Specs layer - * their own feature- or flow-specific routes on top. + * Stub the endpoints every cloud boot path mocks the same way. Callers layer + * their own workspace, billing or flow routes on top; Playwright matches the + * most recently registered handler first, so those overrides still win. */ -export async function mockCloudBoot( +export async function mockCloudBootRoutes( page: Page, - { features, settings = {} }: CloudBootOptions + { features, settings = {}, objectInfo }: CloudBootOptions ) { await page.route('**/api/features', (r) => r.fulfill(jsonRoute(features))) await page.route('**/api/system_stats', (r) => @@ -36,22 +38,32 @@ export async function mockCloudBoot( }) ) ) - await page.route('**/api/user', (r) => - r.fulfill(jsonRoute({ status: 'active' })) - ) await page.route('**/api/settings', (r) => r.fulfill(jsonRoute(settings))) await page.route('**/api/userdata**', (r) => r.fulfill(jsonRoute([]))) await page.route('**/api/extensions', (r) => r.fulfill(jsonRoute([]))) - await page.route('**/api/object_info', (r) => r.fulfill(jsonRoute({}))) + if (objectInfo !== 'server') + await page.route('**/api/object_info', (r) => r.fulfill(jsonRoute({}))) await page.route('**/api/global_subgraphs', (r) => r.fulfill(jsonRoute({}))) await page.route('**/api/i18n', (r) => r.fulfill(jsonRoute({}))) await page.route('**/api/auth/session', (r) => r.fulfill(jsonRoute({ token: 'mock-workspace-token' })) ) - await mockWorkspace(page, workspace('personal', 'owner'), []) await page.route('**/releases**', (r) => r.fulfill(jsonRoute([]))) } +/** + * Stub the core endpoints the cloud app hits on boot so a raw `page` reaches the + * working app without falling through to the OSS devtools backend. Specs layer + * their own feature- or flow-specific routes on top. + */ +export async function mockCloudBoot(page: Page, options: CloudBootOptions) { + await mockCloudBootRoutes(page, options) + await page.route('**/api/user', (r) => + r.fulfill(jsonRoute({ status: 'active' })) + ) + await mockWorkspace(page, workspace('personal', 'owner'), []) +} + /** * Mock Firebase auth and pre-select the e2e user so the cloud app boots * signed-in. The signed-in email (`CLOUD_SELF_EMAIL`) is what the diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts new file mode 100644 index 00000000000..cf2c2762f2b --- /dev/null +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -0,0 +1,48 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { listRecordedConversations } from '@e2e/fixtures/data/agent/agentConversation' + +// A recording whose second turn wires two nodes; the first turn only adds. +const WIRING_CASE = 'agent-rec-two-turn-dependent-edit' + +test.describe('Agent conversation replay', { tag: '@cloud' }, () => { + test.describe('wire evidence', () => { + test.use({ conversationCase: WIRING_CASE }) + + // The second turn's only edit is a connect, so what the canvas shows after + // it is the wire itself: the app's own render loop paints it, and the + // expectation is the picture, not a reconstruction of the renderer. + test('paints the wire the second turn connects @screenshot', async ({ + agentConversation, + page + }) => { + test.setTimeout(90_000) + await agentConversation.runTurns() + + await expect(page.locator('#graph-canvas')).toHaveScreenshot( + 'two-turn-dependent-edit-wired.png', + { mask: [agentConversation.panel] } + ) + }) + }) + + for (const conversationCase of listRecordedConversations()) { + test.describe(`recorded ${conversationCase}`, () => { + test.use({ conversationCase }) + + test('replays every recorded turn onto the panel and the canvas', async ({ + agentConversation + }) => { + test.setTimeout(90_000) + await agentConversation.runTurns() + + await expect( + agentConversation.panel.getByRole('button', { + name: `Open ${agentConversation.conversation.workflow.name}` + }) + ).toBeVisible() + }) + }) + } +}) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts-snapshots/two-turn-dependent-edit-wired-cloud-linux.png b/browser_tests/tests/agent/agentConversationReplay.spec.ts-snapshots/two-turn-dependent-edit-wired-cloud-linux.png new file mode 100644 index 00000000000..fbf4f885a40 Binary files /dev/null and b/browser_tests/tests/agent/agentConversationReplay.spec.ts-snapshots/two-turn-dependent-edit-wired-cloud-linux.png differ diff --git a/browser_tests/tests/agent/agentPanelMocks.ts b/browser_tests/tests/agent/agentPanelMocks.ts index b4a431926e3..7439832231c 100644 --- a/browser_tests/tests/agent/agentPanelMocks.ts +++ b/browser_tests/tests/agent/agentPanelMocks.ts @@ -9,8 +9,8 @@ import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' -import { mockSystemStats } from '@e2e/fixtures/data/systemStats' import { mockBilling } from '@e2e/fixtures/utils/cloudBillingMocks' +import { mockCloudBootRoutes } from '@e2e/fixtures/utils/cloudBootMocks' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' const THREAD_ID = 'd4c016c4-3b8c-44cf-97de-1ae27e43e718' @@ -146,37 +146,13 @@ async function mockAgentBoot( r.fulfill(jsonRoute({ assets: [] })) ) - await page.route('**/api/features', (r) => - r.fulfill(jsonRoute(agentFeatures(agentFlag))) - ) - await page.route('**/api/system_stats', (r) => - r.fulfill(jsonRoute(mockSystemStats)) - ) - await page.route('**/api/users', (r) => - r.fulfill( - jsonRoute({ - storage: 'server', - migrated: true, - users: { 'test-user-e2e': 'E2E Test User' } - }) - ) - ) - await page.route('**/api/settings', (r) => - r.fulfill( - jsonRoute({ - 'Comfy.TutorialCompleted': true, - 'Comfy.RightSidePanel.ShowErrorsTab': false - }) - ) - ) - await page.route('**/api/userdata**', (r) => r.fulfill(jsonRoute([]))) - await page.route('**/api/extensions', (r) => r.fulfill(jsonRoute([]))) - await page.route('**/api/object_info', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/global_subgraphs', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/i18n', (r) => r.fulfill(jsonRoute({}))) - await page.route('**/api/auth/session', (r) => - r.fulfill(jsonRoute({ token: 'mock-workspace-token' })) - ) + await mockCloudBootRoutes(page, { + features: agentFeatures(agentFlag), + settings: { + 'Comfy.TutorialCompleted': true, + 'Comfy.RightSidePanel.ShowErrorsTab': false + } + }) await page.route('**/api/auth/token', (r) => r.fulfill( jsonRoute({ @@ -188,7 +164,6 @@ async function mockAgentBoot( }) ) ) - await page.route('**/releases**', (r) => r.fulfill(jsonRoute([]))) await page.route('**/api/workspaces', (r) => r.fulfill( jsonRoute({ diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index c203ee2dffe..186326824c3 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -159,6 +159,7 @@ const workflowService = vi.hoisted(() => ({ workflowStore.openWorkflowsInBackground({ right: [tab.path] }) workflowStore.activeWorkflow = await known.load() } + return true }) })) @@ -2596,6 +2597,7 @@ describe('AgentPanelRoot workflow binding', () => { mockMessagesEndpoint('wf-42') await renderAndSend('work here') + const mint = vi.spyOn(workflowStore, 'createNewTemporary') ws.emit('agent_active_tab', { workflow_id: 'wf-77', @@ -2607,7 +2609,13 @@ describe('AgentPanelRoot workflow binding', () => { expect(workflowService.openWorkflow).toHaveBeenCalled() ) const minted = workflowStore.getWorkflowByPath('workflows/Video test.json') + expect(mint).toHaveBeenCalledOnce() expect(minted?.filename).toBe('Video test') + expect(JSON.parse(minted?.originalContent ?? 'null')).toMatchObject({ + nodes: [], + links: [], + extra: { ds: { offset: [0, 0], scale: 1 } } + }) // The host minted the doc server-side; the follower fills the canvas. // Nothing loads, saves, or adopts here. expect(workflowService.saveWorkflowAs).not.toHaveBeenCalled() @@ -2649,6 +2657,60 @@ describe('AgentPanelRoot workflow binding', () => { ) }) + it('agent_active_tab closes the minted tab when opening it fails', async () => { + makeTab('wf-42') + mockMessagesEndpoint('wf-42') + + await renderAndSend('work here') + workflowService.openWorkflow.mockRejectedValueOnce(new Error('disk full')) + + ws.emit('agent_active_tab', { + workflow_id: 'wf-77', + name: 'Video test', + thread_id: 'th-1' + }) + + await vi.waitFor(() => + expect(workflowService.openWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ path: 'workflows/Video test.json' }) + ) + ) + await vi.waitFor(() => + expect( + workflowStore.getWorkflowByPath('workflows/Video test.json') + ).toBeNull() + ) + }) + + it('agent_active_tab closes the minted tab when opening it reports failure', async () => { + makeTab('wf-42') + mockMessagesEndpoint('wf-42') + + await renderAndSend('work here') + workflowService.openWorkflow.mockResolvedValueOnce(false) + + ws.emit('agent_active_tab', { + workflow_id: 'wf-77', + name: 'Video test', + thread_id: 'th-1' + }) + + await vi.waitFor(() => + expect(workflowService.openWorkflow).toHaveBeenCalledWith( + expect.objectContaining({ path: 'workflows/Video test.json' }) + ) + ) + await vi.waitFor(() => + expect( + workflowStore.getWorkflowByPath('workflows/Video test.json') + ).toBeNull() + ) + expect( + useAgentWorkflowTabBindingStore().tabPathFor('wf-77') + ).toBeUndefined() + expect(telemetry.trackAgentWorkflowApplied).not.toHaveBeenCalled() + }) + it('agent_active_tab strips dotfile prefixes hidden behind whitespace', async () => { makeTab('wf-42') mockMessagesEndpoint('wf-42') @@ -2711,6 +2773,7 @@ describe('AgentPanelRoot workflow binding', () => { }) const known = workflowStore.getWorkflowByPath(slow.path) if (known) workflowStore.activeWorkflow = await known.load() + return true } ) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 1affee303d4..8372522e5a8 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -47,6 +47,8 @@ import { ACTOR_CONFIG } from '@/renderer/core/layout/constants' import { LayoutSource } from '@/renderer/core/layout/types' import { api } from '@/scripts/api' import { app } from '@/scripts/app' +import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema' +import { blankGraph } from '@/scripts/defaultGraph' import { useAgentNodeSelectionStore } from '@/stores/agentNodeSelectionStore' import { useExecutionErrorStore } from '@/stores/executionErrorStore' import { useWorkflowTabActivityStore } from '@/stores/workflowTabActivityStore' @@ -159,6 +161,11 @@ const { dismissedSelectionSignature, enabled: agentEnabled } = const agentNodeSelectionStore = useAgentNodeSelectionStore() const tabActivity = useWorkflowTabActivityStore() const CREATING_TAB_MIN_DURATION_MS = 500 +// Opens at the template's view so the follower's first nodes land on screen. +const agentTabGraph: ComfyWorkflowJSON = { + ...blankGraph, + extra: { ds: { offset: [0, 0], scale: 1 } } +} const canvasStore = useCanvasStore() const graphMutationsByWorkflow = new Map< @@ -633,9 +640,6 @@ async function onAgentActiveTab( }) return } - // A new agent workflow opens as an EMPTY tab: the host minted its doc - // server-side (seed-at-bind), and the follower fills the canvas through - // the ordinary subscribe catch-up - no snapshot fetch, no draft apply. const creatingStartedAt = Date.now() tabActivity.setCreating(true) const remainingCreatingTime = @@ -643,9 +647,24 @@ async function onAgentActiveTab( if (remainingCreatingTime > 0) await new Promise((resolve) => setTimeout(resolve, remainingCreatingTime)) if (stale()) return - const tab = workflowStore.createTemporary(agentTabFilename(data.name)) + const tab = workflowStore.createNewTemporary( + agentTabFilename(data.name), + agentTabGraph + ) tabActivity.setCreating(false) - await workflowService.openWorkflow(tab) + // openWorkflow reports a handled load failure by resolving false, having + // restored the retained workflow; the minted tab is a ghost either way. + let opened = false + try { + opened = await workflowService.openWorkflow(tab) + } catch (error) { + await workflowStore.closeWorkflow(tab) + throw error + } + if (!opened) { + await workflowStore.closeWorkflow(tab) + return + } if (stale()) { // A newer activation superseded this one mid-open: close the minted // tab rather than stranding a ghost the user never asked for. diff --git a/src/workbench/extensions/agent/crdt/agentNodeMaterializer.ts b/src/workbench/extensions/agent/crdt/agentNodeMaterializer.ts index 24991fed87a..faf04f2247c 100644 --- a/src/workbench/extensions/agent/crdt/agentNodeMaterializer.ts +++ b/src/workbench/extensions/agent/crdt/agentNodeMaterializer.ts @@ -22,7 +22,13 @@ import { runMintPortsSuppressed } from './mintPortWiring' export type MaterializableGraph = Pick< LGraph, - 'id' | 'rootGraph' | '_nodes' | '_nodes_by_id' | 'add' | 'remove' + | 'id' + | 'rootGraph' + | '_nodes' + | '_nodes_by_id' + | 'add' + | 'remove' + | 'setDirtyCanvas' > /** diff --git a/src/workbench/extensions/agent/crdt/docFrameClient.ts b/src/workbench/extensions/agent/crdt/docFrameClient.ts index 318ccb5963b..53d8d995e56 100644 --- a/src/workbench/extensions/agent/crdt/docFrameClient.ts +++ b/src/workbench/extensions/agent/crdt/docFrameClient.ts @@ -2,7 +2,7 @@ import type { Op } from '@comfyorg/comfy-multi-player' import { reportError } from '@/platform/telemetry/reportError' -const DOC_PROTOCOL_VERSION = 1 +export const DOC_PROTOCOL_VERSION = 1 /** Keep this encoded-field cap aligned with cloud's `MaxDocFrameB64Len`. */ const MAX_DOC_UPDATE_B64_LENGTH = 8 << 20 const MAX_WORKFLOW_ID_LENGTH = 128 @@ -85,6 +85,7 @@ export type ServerDocFrame = | { type: 'doc_reset'; data: DocReset } | { type: 'awareness'; data: DocAwareness } +/** A frame as it travels the wire, before {@link parseServerDocFrame} reads it. */ export interface DocFrameTransport { /** * Best-effort send. Returns `true` when the frame left the transport and diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 6e805620ef1..1d366139f48 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -686,7 +686,8 @@ describe('useAgentCrdtFollower', () => { // distinct reference the composable hands through. const { fakeDefinitions } = definitionsState const fakeGraph = { - rootGraph: { subgraphs: new Map() } + rootGraph: { subgraphs: new Map() }, + setDirtyCanvas: vi.fn() } as unknown as MaterializableGraph it('reconciles the live graph after every applied frame', () => { @@ -699,6 +700,9 @@ describe('useAgentCrdtFollower', () => { fakeGraph, fakeDefinitions ) + // A frame that only connects nodes changes no layout, so the repaint + // has to come from here or the new wire stays invisible until a pan. + expect(fakeGraph.setDirtyCanvas).toHaveBeenCalledWith(true, true) // Definitions come from the doc the bridge currently follows, so a // doc_reset remint (which swaps the FollowerDoc) is read fresh. expect(definitionsState.readSubgraphDefinitions).toHaveBeenCalledWith( @@ -711,7 +715,8 @@ describe('useAgentCrdtFollower', () => { const registeredGraph = { rootGraph: { subgraphs: new Map([[fakeDefinitions[0].id, {}]]) - } + }, + setDirtyCanvas: vi.fn() } as unknown as MaterializableGraph const { unmount } = mountFollower('wf-1', true, () => registeredGraph) diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 2926a4089b5..e76af459ef3 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -594,6 +594,9 @@ export function useAgentCrdtFollower( ? readSubgraphDefinitions(bridge.follower.doc) : [] const nodeIds = reconcileAgentAdapters(graph, definitions) + // A frame that only wires or rewires nodes moves no layout, so nothing + // else asks the canvas to paint the new links. + graph.setDirtyCanvas(true, true) if (nodeIds.length > 0) { recordDevEvent('agent_node_adapters_materialized', { workflowId: docId,