From 7017d697ef11795bee512c96c951e68a74d618ea Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 12:12:29 +0000 Subject: [PATCH 01/49] test(agent): add conversation replay fixture for agent panel e2e Add a request/response fixture pattern for the in-app agent: one user prompt in, an ordered agent response out (chat frames plus the semantic graph ops the doc host folds into the workflow document). - agentConversationFixture.ts drives the real composer, mocks the /api/agent routes, owns the /ws route, mints the seed workflow with the pinned comfy-multi-player applier, answers the follower's doc_subscribe with a catch-up, replays each graph_ops entry as a doc_update frame, and snapshots the rendered nodes and slot connectivity of the agent's doc. - data/agent/agentConversation.ts is the zod-typed conversation schema and loader; the two JSON conversations derive from evals agent-suite cases (agent-l4-zimage-string-node-prompt, agent-workflow-editing-05) with the response side synthesized from each case's expected outcome. - agentConversationReplay.spec.ts replays both and asserts the tool-call group, tab link, final message, CRDT status, and end graph state. - bootAgentApp accepts extra settings so Vue nodes can be enabled, which is the renderer that projects follower edits onto the canvas. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0181SuRew6ZZf4o47oPVdn5r --- .../fixtures/agentConversationFixture.ts | 376 ++++++++++++++++++ browser_tests/fixtures/agentPanelFixture.ts | 15 +- .../fixtures/data/agent/agentConversation.ts | 81 ++++ .../agent-l4-zimage-string-node-prompt.json | 277 +++++++++++++ .../agent-workflow-editing-05.json | 277 +++++++++++++ .../agent/agentConversationReplay.spec.ts | 137 +++++++ 6 files changed, 1159 insertions(+), 4 deletions(-) create mode 100644 browser_tests/fixtures/agentConversationFixture.ts create mode 100644 browser_tests/fixtures/data/agent/agentConversation.ts create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json create mode 100644 browser_tests/tests/agent/agentConversationReplay.spec.ts diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts new file mode 100644 index 000000000000..31ba17592f64 --- /dev/null +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -0,0 +1,376 @@ +import type { Locator, Page, WebSocketRoute } from '@playwright/test' +import { expect } from '@playwright/test' + +import { applyOps, mint, readGraph } from '@comfyorg/comfy-multi-player' +import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import * as Y from 'yjs' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' +import { mintWireOps } from '@/workbench/extensions/agent/crdt/opEnvelope' +import type { + AgentCancelAccepted, + AgentMessages, + AgentTurnAccepted, + AgentWsEvent +} from '@/workbench/extensions/agent/schemas/agentApiSchema' +import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { agentTest, bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import type { + AgentConversation, + RecordedWsEvent +} from '@e2e/fixtures/data/agent/agentConversation' +import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' +const TURN_ID = '0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5f60' +const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' +const HOST_ACTOR = 'agent:comfy' +const DOC_PROTOCOL_VERSION = 1 +const PANEL_MOUNT_TIMEOUT = 30_000 +const SUBSCRIBE_TIMEOUT = 15_000 + +const OPEN_AGENT_LABEL = enMessages.agent.askComfyAgent +const SEND_LABEL = enMessages.agent.send +const STOP_LABEL = enMessages.agent.stop + +interface DocFrame { + type: 'doc_subscribed' | 'doc_update' + data: Record +} + +function toBase64(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('base64') +} + +function fromBase64(value: string): Uint8Array { + return new Uint8Array(Buffer.from(value, 'base64')) +} + +/** + * The doc host's side of the CRDT channel, in-process: the seed workflow is + * minted with the same pinned applier the real host runs, each `graph_ops` + * entry is applied to it, and the resulting Yjs delta is what the follower + * receives as a `doc_update` frame. + */ +class HostDoc { + private readonly doc: Y.Doc + private seq = 1 + + constructor( + private readonly workflowId: string, + seed: WorkflowJSON, + private readonly catalog: WidgetCatalog + ) { + this.doc = mint(seed, catalog) + } + + nodeIds(): Set { + return new Set(Object.keys(readGraph(this.doc).nodes)) + } + + subscribed(): DocFrame { + return { + type: 'doc_subscribed', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + ok: true, + seq: this.seq + } + } + } + + catchUp(stateVectorB64: string): DocFrame { + const update = Y.encodeStateAsUpdate(this.doc, fromBase64(stateVectorB64)) + return this.updateFrame(update, HOST_ACTOR, []) + } + + apply(operations: GraphOperation[]): DocFrame { + const before = Y.encodeStateVector(this.doc) + const ops = mintWireOps(operations, { + actor: HOST_ACTOR, + baseVersion: this.seq + }) + const result = applyOps(this.doc, ops, this.catalog) + const rejected = result.outcomes.filter((o) => o.outcome !== 'applied') + if (rejected.length > 0) + throw new Error( + `conversation graph_ops did not apply: ${JSON.stringify(rejected)}` + ) + this.seq += 1 + return this.updateFrame( + Y.encodeStateAsUpdate(this.doc, before), + HOST_ACTOR, + ops.map((op) => op.op_id) + ) + } + + private updateFrame( + update: Uint8Array, + actor: string, + opIds: string[] + ): DocFrame { + return { + type: 'doc_update', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + seq: this.seq, + update_b64: toBase64(update), + actor, + op_ids: opIds + } + } + } +} + +interface GraphNodeSnapshot { + id: string + title: string + /** Connected state of each rendered input slot, in slot order. */ + inputs: boolean[] + /** Connected state of each rendered output slot, in slot order. */ + outputs: boolean[] +} + +/** + * Drives one recorded user prompt -> agent response exchange through the + * real agent panel: the composer posts the prompt, the mocked backend acks + * it, and the recorded response is replayed over the routed `/ws` socket as + * chat frames plus doc-host `doc_update` frames the CRDT follower applies to + * the canvas. + */ +export class AgentConversationHarness { + readonly postedMessages: string[] = [] + readonly panel: Locator + readonly ack: AgentTurnAccepted + + private readonly host: HostDoc + private socket: WebSocketRoute | null = null + private resolveSubscribed: (() => void) | null = null + private readonly subscribed = new Promise((resolve) => { + this.resolveSubscribed = resolve + }) + + constructor( + private readonly page: Page, + readonly conversation: AgentConversation + ) { + const { workflow } = conversation + this.host = new HostDoc(workflow.id, workflow.seed, workflow.catalog) + this.ack = { + thread_id: THREAD_ID, + message_id: TURN_ID, + workflow_id: workflow.id + } + this.panel = page.locator('#agent-panel-root') + } + + async boot(agentFlag: boolean): Promise { + await this.mockAgentApi() + // The app may reopen `/ws` (the connect awaits an auth token), and the + // follower only re-drives a pending subscribe on a `status` frame, which + // the real server sends on every connect. So every routed socket becomes + // the live one and gets that frame. + await this.page.routeWebSocket(/\/ws/, (socket) => { + this.socket = socket + socket.onMessage((raw) => this.onClientFrame(raw)) + socket.send( + JSON.stringify({ + type: 'status', + data: { + status: { exec_info: { queue_remaining: 0 } }, + sid: SOCKET_SID + } + }) + ) + }) + await bootAgentApp(this.page, agentFlag, { + // Follower edits land in the ECS stores, which only the Vue node + // renderer projects onto the canvas. + settings: { 'Comfy.VueNodes.Enabled': true } + }) + + await this.page.getByRole('button', { name: OPEN_AGENT_LABEL }).click() + await expect(this.panel).toBeVisible({ timeout: PANEL_MOUNT_TIMEOUT }) + } + + async sendPrompt(): Promise { + const { content } = this.conversation.request + const composer = this.panel.getByRole('textbox', { + name: /^Describe ideas/ + }) + await composer.fill(content) + await this.panel.getByRole('button', { name: SEND_LABEL }).click() + await expect.poll(() => this.postedMessages.length).toBeGreaterThan(0) + expect(this.postedMessages[0]).toContain(content) + } + + async replayResponse(): Promise { + for (const entry of this.conversation.response) { + if (entry.kind === 'event') { + this.send(this.stampTurn(entry.event)) + continue + } + await this.waitForSubscribe() + this.send(this.host.apply(entry.ops)) + } + } + + async waitForTurnComplete(): Promise { + await expect( + this.panel.getByRole('button', { name: SEND_LABEL }) + ).toBeVisible() + await expect( + this.panel.getByRole('button', { name: STOP_LABEL }) + ).toHaveCount(0) + } + + /** + * The agent's workflow document as rendered on the canvas: every node the + * host doc owns, with its slot connectivity. Scoped to the doc's node ids + * because the tab the agent opens starts from the default template, whose + * nodes sit next to the follower's and would pin that template here. + */ + async graphSnapshot(): Promise { + const docNodeIds = this.host.nodeIds() + const snapshot = await this.page + .locator('[data-node-id]') + .evaluateAll((nodes) => + nodes.map((node) => { + const connected = (selector: string) => + Array.from(node.querySelectorAll(selector)).map((slot) => + slot.classList.contains('lg-slot--connected') + ) + return { + id: node.getAttribute('data-node-id') ?? '', + title: + node + .querySelector('[data-testid="node-title"]') + ?.textContent?.trim() ?? '', + inputs: connected('.lg-slot--input'), + outputs: connected('.lg-slot--output') + } + }) + ) + return snapshot + .filter((node) => docNodeIds.has(node.id)) + .sort((a, b) => Number(a.id) - Number(b.id)) + } + + private async mockAgentApi(): Promise { + const { page } = this + await page.route('**/api/agent/threads', (route) => + route.fulfill(jsonRoute({ threads: [] })) + ) + await page.route('**/api/agent/threads/*/messages', (route) => { + const request = route.request() + if (request.method() === 'POST') { + this.postedMessages.push(request.postData() ?? '') + return route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify(this.ack) + }) + } + const history: AgentMessages = [] + return route.fulfill(jsonRoute(history)) + }) + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => { + const cancelled: AgentCancelAccepted = { status: 'cancelling' } + return route.fulfill(jsonRoute(cancelled)) + }) + await page.route('**/api/workflows**', (route) => + route.fulfill( + jsonRoute({ + data: [], + pagination: { has_more: false, limit: 100, offset: 0, total: 0 } + }) + ) + ) + } + + private stampTurn(event: RecordedWsEvent): AgentWsEvent { + const stamped = { + type: event.type, + data: { ...event.data, message_id: TURN_ID, thread_id: THREAD_ID } + } + const parsed = parseAgentWsEvent(stamped) + if (!parsed.success) + throw new Error( + `recorded ${event.type} frame is not a valid agent event: ${parsed.error.message}` + ) + return parsed.data + } + + private send(frame: AgentWsEvent | DocFrame): void { + if (!this.socket) throw new Error('the app has not opened /ws yet') + this.socket.send(JSON.stringify(frame)) + } + + private onClientFrame(raw: string | Buffer): void { + const frame: unknown = JSON.parse(raw.toString()) + if (typeof frame !== 'object' || frame === null) return + const { type, data } = frame as { type?: unknown; data?: unknown } + if (type !== 'doc_subscribe' || typeof data !== 'object' || data === null) + return + const { workflow_id, state_vector_b64 } = data as { + workflow_id?: unknown + state_vector_b64?: unknown + } + if ( + workflow_id !== this.conversation.workflow.id || + typeof state_vector_b64 !== 'string' + ) + return + this.send(this.host.subscribed()) + this.send(this.host.catchUp(state_vector_b64)) + this.resolveSubscribed?.() + } + + private async waitForSubscribe(): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + 'the follower never subscribed to the conversation workflow; graph_ops need an agent_active_tab (or a bound tab) first' + ) + ), + SUBSCRIBE_TIMEOUT + ) + }) + try { + await Promise.race([this.subscribed, timeout]) + } finally { + clearTimeout(timer) + } + } +} + +interface ConversationFixtures { + /** Case id of the conversation under `fixtures/data/agent/conversations`. */ + conversationCase: string + agentConversation: AgentConversationHarness +} + +export const agentConversationTest = agentTest.extend({ + conversationCase: ['', { option: true }], + agentConversation: async ( + { page, agentFlagEnabled, conversationCase }, + use + ) => { + if (conversationCase.length === 0) + throw new Error('test.use({ conversationCase }) names the conversation') + const harness = new AgentConversationHarness( + page, + loadAgentConversation(conversationCase) + ) + await harness.boot(agentFlagEnabled) + await use(harness) + } +}) diff --git a/browser_tests/fixtures/agentPanelFixture.ts b/browser_tests/fixtures/agentPanelFixture.ts index 033c519adb70..4ee0352f537d 100644 --- a/browser_tests/fixtures/agentPanelFixture.ts +++ b/browser_tests/fixtures/agentPanelFixture.ts @@ -23,15 +23,21 @@ function agentFeatures(agentFlag: boolean): RemoteConfig { } } +interface BootAgentAppOptions { + /** Extra `/api/settings` entries layered over the panel defaults. */ + settings?: Record +} + async function mockAgentBoot( page: Page, - { agentFlag }: { agentFlag: boolean } + { agentFlag, settings }: { agentFlag: boolean } & BootAgentAppOptions ): Promise { await mockCloudBoot(page, { features: agentFeatures(agentFlag), settings: { 'Comfy.TutorialCompleted': true, - 'Comfy.RightSidePanel.ShowErrorsTab': false + 'Comfy.RightSidePanel.ShowErrorsTab': false, + ...settings } }) await mockBilling(page) @@ -65,14 +71,15 @@ export const agentTest = cloudAppFixture.extend({ export async function bootAgentApp( page: Page, - agentFlag: boolean + agentFlag: boolean, + options: BootAgentAppOptions = {} ): Promise { // The shell's onboarding coach is a modal; pre-seed its dismissal so the // panel chrome is interactable, as the canonical agent suite does. await page.addInitScript(() => { localStorage.setItem('Comfy.AgentPanel.onboarded', 'true') }) - await mockAgentBoot(page, { agentFlag }) + await mockAgentBoot(page, { agentFlag, ...options }) await bootCloud(page) await page.goto(APP_URL) await waitForCloudApp(page) diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts new file mode 100644 index 000000000000..2a948ab58748 --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -0,0 +1,81 @@ +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** + * A realtime agent frame as recorded, minus the turn identity + * (`message_id` / `thread_id`): the replay stamps those from the ack it + * minted, then validates the result against `zAgentWsEvent` before sending. + */ +const zRecordedWsEvent = z.object({ + type: z.string(), + data: z.record(z.string(), z.unknown()) +}) +export type RecordedWsEvent = z.infer + +// Payload shapes are the pinned applier's contract (it rejects a malformed +// op at replay time); the fixture only pins the vocabulary. +const zGraphOperation = z.custom( + (value) => + isRecord(value) && + typeof value.op === 'string' && + (FROZEN_OPS as readonly string[]).includes(value.op) +) + +const zWorkflowJson = z.custom( + (value) => + isRecord(value) && Array.isArray(value.nodes) && Array.isArray(value.links) +) + +const zWidgetCatalog = z.custom( + (value) => isRecord(value) && isRecord(value.types) +) + +const zResponseEntry = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('event'), event: zRecordedWsEvent }), + z.object({ + kind: z.literal('graph_ops'), + ops: z.array(zGraphOperation).min(1) + }) +]) + +/** + * One user prompt and the ordered agent response it produced: chat frames + * (`event`) interleaved with the semantic graph operations (`graph_ops`) the + * doc host folds into the workflow document. + */ +const zAgentConversation = z.object({ + schema_version: z.literal('agent-conversation.v1'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + response_side: z.enum(['recorded', 'synthesized']), + note: z.string().optional() + }), + workflow: z.object({ + id: z.string().uuid(), + name: z.string().min(1), + catalog: zWidgetCatalog, + seed: zWorkflowJson + }), + request: z.object({ content: z.string().min(1) }), + response: z.array(zResponseEntry).min(1) +}) +export type AgentConversation = z.infer + +export function loadAgentConversation(caseId: string): AgentConversation { + const file = fileURLToPath( + new URL(`./conversations/${caseId}.json`, import.meta.url) + ) + return zAgentConversation.parse(JSON.parse(readFileSync(file, 'utf-8'))) +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json new file mode 100644 index 000000000000..afad82463e76 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -0,0 +1,277 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/evals", + "suite": "agent", + "case_id": "agent-l4-zimage-string-node-prompt", + "response_side": "synthesized", + "note": "Response synthesized from the case's expected outcome (string node added, text set, STRING wired into the prompt input). The eval seed is a Z-Image subgraph; this replay uses a flat text-to-image seed so the wiring lands on a plain CLIPTextEncode. The case's trailing run is not replayed." + }, + "workflow": { + "id": "6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { "widget_order": ["ckpt_name"] }, + "CLIPTextEncode": { "widget_order": ["text"] }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { "widget_order": [] }, + "SaveImage": { "widget_order": ["filename_prefix"] }, + "PrimitiveStringMultiline": { "widget_order": ["value"] } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { "name": "MODEL", "type": "MODEL", "links": [3] }, + { "name": "CLIP", "type": "CLIP", "links": [1] }, + { "name": "VAE", "type": "VAE", "links": [5] } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { "name": "clip", "type": "CLIP", "link": 1 }, + { + "name": "text", + "type": "STRING", + "widget": { "name": "text" }, + "link": null + } + ], + "outputs": [ + { "name": "CONDITIONING", "type": "CONDITIONING", "links": [2] } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { "name": "model", "type": "MODEL", "link": 3 }, + { "name": "positive", "type": "CONDITIONING", "link": 2 }, + { "name": "negative", "type": "CONDITIONING", "link": null }, + { "name": "latent_image", "type": "LATENT", "link": null } + ], + "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [4] }], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { "name": "samples", "type": "LATENT", "link": 4 }, + { "name": "vae", "type": "VAE", "link": 5 } + ], + "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [6] }], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [{ "name": "images", "type": "IMAGE", "link": 6 }], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll add a multiline string node, set its text, and wire its STRING output into the positive prompt's text input." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "workflow_id": "6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01", + "name": "Text to image" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-add-string-node", + "tool_name": "add_node", + "status": "running" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "node_id": 10, + "class_type": "PrimitiveStringMultiline", + "pos": [0, 300], + "node": { + "id": 10, + "type": "PrimitiveStringMultiline", + "title": "Prompt text", + "pos": [0, 300], + "size": [400, 200], + "inputs": [], + "outputs": [{ "name": "STRING", "type": "STRING", "links": [] }], + "widgets_values": [""] + } + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-add-string-node", + "tool_name": "add_node", + "status": "success", + "duration_ms": 400 + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-set-prompt-text", + "tool_name": "set_widget", + "status": "running" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "node_id": 10, + "widget": "value", + "value": "a red bicycle on a pier at dusk" + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-set-prompt-text", + "tool_name": "set_widget", + "status": "success", + "duration_ms": 200 + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-connect-prompt", + "tool_name": "connect", + "status": "running" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "link_id": 7, + "from_node": 10, + "from_slot": 0, + "to_node": 6, + "to_slot": 1, + "link_type": "STRING" + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-connect-prompt", + "tool_name": "connect", + "status": "success", + "duration_ms": 300 + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Added a **Prompt text** node with *a red bicycle on a pier at dusk* and wired its STRING output into the positive prompt, so the prompt is now driven by that node. Say the word and I'll run it." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "input_tokens": 5120, + "output_tokens": 310, + "total_tokens": 5430, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json new file mode 100644 index 000000000000..4cdeb940f97f --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -0,0 +1,277 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/evals", + "suite": "agent", + "case_id": "agent-workflow-editing-05", + "response_side": "synthesized", + "note": "Response synthesized from the case's expected outcome: an ImageStitch node shows the generated output next to the input image. The eval case ships no seed, so a small image-to-image seed stands in." + }, + "workflow": { + "id": "2b7d9a34-5c1e-4f0a-8e6b-7d2f3a9c1b45", + "name": "Image edit", + "catalog": { + "types": { + "LoadImage": { "widget_order": ["image", "upload"] }, + "VAEEncode": { "widget_order": [] }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { "widget_order": [] }, + "SaveImage": { "widget_order": ["filename_prefix"] }, + "ImageStitch": { + "widget_order": [ + "direction", + "match_image_size", + "spacing_width", + "spacing_color" + ] + }, + "PreviewImage": { "widget_order": [] } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { "name": "IMAGE", "type": "IMAGE", "links": [1] }, + { "name": "MASK", "type": "MASK", "links": [] } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "VAEEncode", + "pos": [400, 0], + "size": [210, 46], + "inputs": [ + { "name": "pixels", "type": "IMAGE", "link": 1 }, + { "name": "vae", "type": "VAE", "link": null } + ], + "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [2] }], + "widgets_values": [] + }, + { + "id": 3, + "type": "KSampler", + "pos": [700, 0], + "size": [315, 262], + "inputs": [ + { "name": "model", "type": "MODEL", "link": null }, + { "name": "positive", "type": "CONDITIONING", "link": null }, + { "name": "negative", "type": "CONDITIONING", "link": null }, + { "name": "latent_image", "type": "LATENT", "link": 2 } + ], + "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [3] }], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1100, 0], + "size": [210, 46], + "inputs": [ + { "name": "samples", "type": "LATENT", "link": 3 }, + { "name": "vae", "type": "VAE", "link": null } + ], + "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [4] }], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1400, 0], + "size": [315, 270], + "inputs": [{ "name": "images", "type": "IMAGE", "link": 4 }], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 1, 0, 2, 0, "IMAGE"], + [2, 2, 0, 3, 3, "LATENT"], + [3, 3, 0, 8, 0, "LATENT"], + [4, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "now use a stitch images node to show the output next to the input" + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll stitch the loaded input and the decoded output side by side and preview the result." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "workflow_id": "2b7d9a34-5c1e-4f0a-8e6b-7d2f3a9c1b45", + "name": "Image edit" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-add-stitch", + "tool_name": "add_node", + "status": "running" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "node_id": 12, + "class_type": "ImageStitch", + "pos": [1400, 350], + "node": { + "id": 12, + "type": "ImageStitch", + "title": "Input vs output", + "pos": [1400, 350], + "size": [315, 150], + "inputs": [ + { "name": "image1", "type": "IMAGE", "link": null }, + { "name": "image2", "type": "IMAGE", "link": null } + ], + "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [] }], + "widgets_values": ["right", true, 0, "white"] + } + }, + { + "op": "add_node", + "node_id": 13, + "class_type": "PreviewImage", + "pos": [1800, 350], + "node": { + "id": 13, + "type": "PreviewImage", + "title": "Side by side", + "pos": [1800, 350], + "size": [315, 270], + "inputs": [{ "name": "images", "type": "IMAGE", "link": null }], + "outputs": [], + "widgets_values": [] + } + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-add-stitch", + "tool_name": "add_node", + "status": "success", + "duration_ms": 500 + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-wire-stitch", + "tool_name": "connect", + "status": "running" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "link_id": 5, + "from_node": 1, + "from_slot": 0, + "to_node": 12, + "to_slot": 0, + "link_type": "IMAGE" + }, + { + "op": "connect", + "link_id": 6, + "from_node": 8, + "from_slot": 0, + "to_node": 12, + "to_slot": 1, + "link_type": "IMAGE" + }, + { + "op": "connect", + "link_id": 7, + "from_node": 12, + "from_slot": 0, + "to_node": 13, + "to_slot": 0, + "link_type": "IMAGE" + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "tool_call_id": "call-wire-stitch", + "tool_name": "connect", + "status": "success", + "duration_ms": 700 + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done. An **Image Stitch** node now takes the loaded input on the left and the decoded output on the right, and a preview node shows the pair. Your save node is untouched." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "input_tokens": 6010, + "output_tokens": 280, + "total_tokens": 6290, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0 + } + } + } + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts new file mode 100644 index 000000000000..235ec38448af --- /dev/null +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -0,0 +1,137 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' + +test.describe('Agent conversation replay', { tag: '@cloud' }, () => { + test.describe('evals agent-l4-zimage-string-node-prompt', () => { + test.use({ conversationCase: 'agent-l4-zimage-string-node-prompt' }) + + test('adds a string node, sets its text, and wires it into the prompt', async ({ + agentConversation + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { + name: 'Ran 3 tool calls for 0.9 seconds' + }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows).toHaveCount(3) + await expect(toolRows.filter({ hasText: 'Add node' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Set widget' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Connect' })).toBeVisible() + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + await expect( + panel.locator('strong', { hasText: 'Prompt text' }) + ).toBeVisible() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '4 updates' + ) + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + // The wired `text` input is widget-backed and renders as a widget, + // not a slot row; the link shows on the string node's output. + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { id: '10', title: 'Prompt text', inputs: [], outputs: [true] } + ]) + }) + }) + + test.describe('evals agent-workflow-editing-05', () => { + test.use({ conversationCase: 'agent-workflow-editing-05' }) + + test('stitches the input next to the output and previews the pair', async ({ + agentConversation + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { + name: 'Ran 2 tool calls for 1.2 seconds' + }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await expect( + panel.locator('strong', { hasText: 'Image Stitch' }) + ).toBeVisible() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '3 updates' + ) + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { id: '1', title: 'LoadImage', inputs: [], outputs: [true, false] }, + { + id: '2', + title: 'VAEEncode', + inputs: [true, false], + outputs: [true] + }, + { + id: '3', + title: 'KSampler', + inputs: [false, false, false, true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, false], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '12', + title: 'Input vs output', + inputs: [true, true], + outputs: [true] + }, + { id: '13', title: 'Side by side', inputs: [true], outputs: [] } + ]) + }) + }) +}) From bdfe7667eaefd61cbae8b7916ad2e920ac83e352 Mon Sep 17 00:00:00 2001 From: cheap-drain Date: Wed, 2 Sep 2026 12:49:59 +0000 Subject: [PATCH 02/49] fix(agent): drop unused export flagged by knip AgentConversationHarness is only constructed inside its own file (the agentConversationTest fixture); the export served no external consumer and knip flagged it as an unused export. --- browser_tests/fixtures/agentConversationFixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 31ba17592f64..7b548a3928b5 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -143,7 +143,7 @@ interface GraphNodeSnapshot { * chat frames plus doc-host `doc_update` frames the CRDT follower applies to * the canvas. */ -export class AgentConversationHarness { +class AgentConversationHarness { readonly postedMessages: string[] = [] readonly panel: Locator readonly ack: AgentTurnAccepted From 7554341cc9dfceeb18c190c44280aeb2f6d05cd2 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 2 Sep 2026 06:28:34 -0700 Subject: [PATCH 03/49] fix(agent): mint the agent tab from blankGraph, not the default template agent_active_tab for an unbound workflow called createTemporary(name) with no data, which seeds defaultGraph. The follower then reconciled the doc onto template nodes with the same ids (3, 8, 9); reconcileNode keeps any link it does not displace, so template link 8 (node 4 VAE to node 8 vae) survived and VAEDecode rendered [true, true]. The replay spec for agent-workflow-editing-05 caught this in CI. Pass blankGraph so the tab really is empty, as the surrounding comment and the PR describe. The "blank named tab" unit test now asserts the minted tab holds no nodes or links (fails on the old code). The fixture comment on graphSnapshot now describes the collision hazard instead of treating the template as expected. --- browser_tests/fixtures/agentConversationFixture.ts | 8 +++++--- src/workbench/extensions/agent/AgentPanelRoot.test.ts | 3 +++ src/workbench/extensions/agent/AgentPanelRoot.vue | 9 ++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 7b548a3928b5..606634517c4e 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -231,9 +231,11 @@ class AgentConversationHarness { /** * The agent's workflow document as rendered on the canvas: every node the - * host doc owns, with its slot connectivity. Scoped to the doc's node ids - * because the tab the agent opens starts from the default template, whose - * nodes sit next to the follower's and would pin that template here. + * host doc owns, with its slot connectivity. The tab the agent opens is + * expected to be blank (AgentPanelRoot mints it from `blankGraph`); the + * doc-id filter is defensive, so a stray template node cannot pin the + * template here. Colliding ids are the real hazard: a template link into a + * doc node survives the follower's reconcile and shows up as connectivity. */ async graphSnapshot(): Promise { const docNodeIds = this.host.nodeIds() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index c111e7812003..356068ce9747 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -2311,6 +2311,9 @@ describe('AgentPanelRoot workflow binding', () => { ) const minted = hostStores.workflow.tabs.get('workflows/Video test.json') expect(minted?.filename).toBe('Video test') + // Blank, not the default template: the follower reconciles onto whatever + // the tab holds and keeps template links it does not displace. + expect(minted?.activeState).toMatchObject({ nodes: [], links: [] }) // The host minted the doc server-side; the follower fills the canvas. // Nothing loads, saves, or adopts here. expect(workflowService.saveWorkflowAs).not.toHaveBeenCalled() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index e3df37520793..0fa526f2626f 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -46,6 +46,7 @@ import { ACTOR_CONFIG } from '@/renderer/core/layout/constants' import { LayoutSource } from '@/renderer/core/layout/types' import { api } from '@/scripts/api' import { app } from '@/scripts/app' +import { blankGraph } from '@/scripts/defaultGraph' import { useAgentNodeSelectionStore } from '@/stores/agentNodeSelectionStore' import { useExecutionErrorStore } from '@/stores/executionErrorStore' import { useWorkflowTabActivityStore } from '@/stores/workflowTabActivityStore' @@ -514,6 +515,9 @@ async function onAgentActiveTab( // A new agent workflow opens as an EMPTY tab: the host minted its doc // server-side (seed-at-bind), and the follower fills the canvas through // the ordinary subscribe catch-up - no snapshot fetch, no draft apply. + // `createTemporary` without data seeds the default template, whose nodes + // share ids with typical agent docs; the follower's reconcile keeps any + // template link it does not displace, so the tab must really be blank. const creatingStartedAt = Date.now() tabActivity.setCreating(true) const remainingCreatingTime = @@ -521,7 +525,10 @@ async function onAgentActiveTab( if (remainingCreatingTime > 0) await new Promise((resolve) => setTimeout(resolve, remainingCreatingTime)) if (stale()) return - const tab = workflowStore.createTemporary(agentTabFilename(data.name)) + const tab = workflowStore.createTemporary( + agentTabFilename(data.name), + blankGraph + ) tabActivity.setCreating(false) await workflowService.openWorkflow(tab) if (stale()) { From 38f558127801a5ced64bfc34105eacfbffbb98a4 Mon Sep 17 00:00:00 2001 From: bymyself Date: Wed, 2 Sep 2026 16:31:18 +0000 Subject: [PATCH 04/49] test(agent): add recorded conversation exporter --- browser_tests/fixtures/data/agent/README.md | 55 ++++++ .../fixtures/data/agent/agentConversation.ts | 65 +++++-- scripts/agentConversationCapture.test.ts | 144 ++++++++++++++ scripts/agentConversationCapture.ts | 177 ++++++++++++++++++ 4 files changed, 422 insertions(+), 19 deletions(-) create mode 100644 browser_tests/fixtures/data/agent/README.md create mode 100644 scripts/agentConversationCapture.test.ts create mode 100644 scripts/agentConversationCapture.ts diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md new file mode 100644 index 000000000000..8ae7ba93eda7 --- /dev/null +++ b/browser_tests/fixtures/data/agent/README.md @@ -0,0 +1,55 @@ +# Agent conversation capture + +Conversation replays must use recorded cloud-agent responses. The exporter +combines two records from the same turn: + +1. Websocket `agent_*` frames, in received order. +2. The cloud backend's durable parent `agent_tool_calls.result` plus the + accepted operation IDs from its child audit rows. + +Do not write `graph_ops` by hand or relabel a synthesized response. The fixture +schema rejects `response_side: recorded` without a cloud thread/message ID and +export timestamp. + +## Capture + +Record the `/ws` frames for one eval turn and remove unrelated frame types. Keep +the original `thread_id`, `message_id`, `tool_call_id`, and ordering. Export the +matching backend rows with this query (bind `$1` to the thread ID and `$2` to +the assistant message ID): + +```sql +SELECT + parent.tool_call_id, + parent.result, + COALESCE( + json_agg(child.op_id ORDER BY child.op_index) + FILTER (WHERE child.status = 'ok' AND child.op_id IS NOT NULL), + '[]'::json + ) AS applied_op_ids +FROM agent_tool_calls AS parent +LEFT JOIN agent_tool_calls AS child ON child.parent_call_id = parent.id +WHERE parent.thread_id = $1 + AND parent.message_id = $2 + AND parent.parent_call_id IS NULL +GROUP BY parent.id +ORDER BY parent.started_at, parent.id; +``` + +Put those values in an `agent-backend-capture.v1` JSON document together with +the eval source, seed workflow, widget catalog, request, and captured frames. +The thread ID and message ID in `capture` must be the IDs used for both exports. + +## Export + +```bash +pnpm exec tsx scripts/agentConversationCapture.ts \ + /path/to/backend-capture.json \ + browser_tests/fixtures/data/agent/conversations/.json +``` + +The exporter strips turn identity from frames (the replay mints its own), +inserts each durably accepted op before its terminal tool-call frame, and fails +if any frame belongs to another turn, an accepted op is missing from the +recorded parent result, or a mutating call has no terminal frame. Then run the +conversation replay test for the case. diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 2a948ab58748..8e6a8e2a5787 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -16,7 +16,7 @@ function isRecord(value: unknown): value is Record { * (`message_id` / `thread_id`): the replay stamps those from the ack it * minted, then validates the result against `zAgentWsEvent` before sending. */ -const zRecordedWsEvent = z.object({ +export const zRecordedWsEvent = z.object({ type: z.string(), data: z.record(z.string(), z.unknown()) }) @@ -40,6 +40,17 @@ const zWidgetCatalog = z.custom( (value) => isRecord(value) && isRecord(value.types) ) +export const zAgentConversationWorkflow = z.object({ + id: z.string().uuid(), + name: z.string().min(1), + catalog: zWidgetCatalog, + seed: zWorkflowJson +}) + +export const zAgentConversationRequest = z.object({ + content: z.string().min(1) +}) + const zResponseEntry = z.discriminatedUnion('kind', [ z.object({ kind: z.literal('event'), event: zRecordedWsEvent }), z.object({ @@ -53,24 +64,40 @@ const zResponseEntry = z.discriminatedUnion('kind', [ * (`event`) interleaved with the semantic graph operations (`graph_ops`) the * doc host folds into the workflow document. */ -const zAgentConversation = z.object({ - schema_version: z.literal('agent-conversation.v1'), - source: z.object({ - repo: z.string(), - suite: z.string(), - case_id: z.string(), - response_side: z.enum(['recorded', 'synthesized']), - note: z.string().optional() - }), - workflow: z.object({ - id: z.string().uuid(), - name: z.string().min(1), - catalog: zWidgetCatalog, - seed: zWorkflowJson - }), - request: z.object({ content: z.string().min(1) }), - response: z.array(zResponseEntry).min(1) -}) +export const zAgentConversation = z + .object({ + schema_version: z.literal('agent-conversation.v1'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + response_side: z.enum(['recorded', 'synthesized']), + note: z.string().optional(), + capture: z + .object({ + backend: z.literal('Comfy-Org/cloud'), + thread_id: z.string().min(1), + message_id: z.string().min(1), + exported_at: z.string().datetime() + }) + .optional() + }), + workflow: zAgentConversationWorkflow, + request: zAgentConversationRequest, + response: z.array(zResponseEntry).min(1) + }) + .superRefine((conversation, ctx) => { + if ( + conversation.source.response_side === 'recorded' && + conversation.source.capture === undefined + ) { + ctx.addIssue({ + code: 'custom', + path: ['source', 'capture'], + message: 'recorded responses require backend capture provenance' + }) + } + }) export type AgentConversation = z.infer export function loadAgentConversation(caseId: string): AgentConversation { diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts new file mode 100644 index 000000000000..2bab1bac8e00 --- /dev/null +++ b/scripts/agentConversationCapture.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from 'vitest' + +import { zAgentConversation } from '../browser_tests/fixtures/data/agent/agentConversation' +import { exportAgentConversation } from './agentConversationCapture' + +const capture = { + schema_version: 'agent-backend-capture.v1', + source: { + repo: 'Comfy-Org/evals', + suite: 'agent', + case_id: 'recorded-case' + }, + capture: { + backend: 'Comfy-Org/cloud', + thread_id: 'thread-1', + message_id: 'message-1', + exported_at: '2026-09-02T16:00:00.000Z' + }, + workflow: { + id: '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01', + name: 'Captured workflow', + catalog: { types: {} }, + seed: { nodes: [], links: [] } + }, + request: { content: 'Add a node' }, + frames: [ + { + type: 'agent_tool_call', + data: { + thread_id: 'thread-1', + message_id: 'message-1', + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'running' + } + }, + { + type: 'agent_tool_call', + data: { + thread_id: 'thread-1', + message_id: 'message-1', + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'success' + } + } + ], + tool_calls: [ + { + tool_call_id: 'tool-1', + applied_op_ids: ['op-accepted'], + result: { + data: { + ops: [ + { op: 'add_node', op_id: 'op-accepted', node_id: 1 }, + { op: 'add_node', op_id: 'op-rejected', node_id: 2 } + ] + } + } + } + ] +} as const + +describe('exportAgentConversation', () => { + it('interleaves only durably applied backend ops before the terminal frame', () => { + const conversation = exportAgentConversation(capture) + + expect(conversation.source).toMatchObject({ + response_side: 'recorded', + capture: capture.capture + }) + expect(conversation.response).toEqual([ + { + kind: 'event', + event: { + type: 'agent_tool_call', + data: { + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'running' + } + } + }, + { + kind: 'graph_ops', + ops: [{ op: 'add_node', op_id: 'op-accepted', node_id: 1 }] + }, + { + kind: 'event', + event: { + type: 'agent_tool_call', + data: { + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'success' + } + } + } + ]) + }) + + it('refuses an accepted op missing from the recorded result', () => { + expect(() => + exportAgentConversation({ + ...capture, + tool_calls: [ + { + ...capture.tool_calls[0], + applied_op_ids: ['op-not-in-result'] + } + ] + }) + ).toThrow('accepted op op-not-in-result') + }) + + it('refuses to silently omit a mutating backend call', () => { + expect(() => + exportAgentConversation({ ...capture, frames: [capture.frames[0]] }) + ).toThrow('no terminal websocket frame for tool call(s): tool-1') + }) + + it('refuses a websocket frame from another turn', () => { + expect(() => + exportAgentConversation({ + ...capture, + frames: [ + { + ...capture.frames[0], + data: { ...capture.frames[0].data, message_id: 'another-message' } + } + ] + }) + ).toThrow('does not belong to capture thread-1/message-1') + }) + + it('refuses a recorded label without backend provenance', () => { + const conversation = exportAgentConversation(capture) + const { capture: _capture, ...source } = conversation.source + + expect(() => zAgentConversation.parse({ ...conversation, source })).toThrow( + 'recorded responses require backend capture provenance' + ) + }) +}) diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts new file mode 100644 index 000000000000..d956f16818db --- /dev/null +++ b/scripts/agentConversationCapture.ts @@ -0,0 +1,177 @@ +#!/usr/bin/env tsx + +import { readFileSync, writeFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import { + zAgentConversation, + zAgentConversationRequest, + zAgentConversationWorkflow, + zRecordedWsEvent +} from '../browser_tests/fixtures/data/agent/agentConversation' + +const zOperation = z + .record(z.string(), z.unknown()) + .refine( + (operation) => + typeof operation.op === 'string' && + (FROZEN_OPS as readonly string[]).includes(operation.op), + 'unknown graph operation' + ) + +const zToolCall = z.object({ + tool_call_id: z.string().min(1), + result: z.record(z.string(), z.unknown()), + applied_op_ids: z.array(z.string().min(1)) +}) + +const zBackendCapture = z.object({ + schema_version: z.literal('agent-backend-capture.v1'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + note: z.string().optional() + }), + capture: z.object({ + backend: z.literal('Comfy-Org/cloud'), + thread_id: z.string().min(1), + message_id: z.string().min(1), + exported_at: z.string().datetime() + }), + workflow: zAgentConversationWorkflow, + request: zAgentConversationRequest, + frames: z.array(zRecordedWsEvent).min(1), + tool_calls: z.array(zToolCall) +}) +export type AgentBackendCapture = z.input + +function operationsFromResult(toolCall: z.infer) { + const data = toolCall.result.data + if (typeof data !== 'object' || data === null || Array.isArray(data)) + return [] + + const result = data as Record + const candidates = Array.isArray(result.ops) + ? result.ops + : result.op === undefined + ? [] + : [result.op] + const operations = candidates.map((operation) => zOperation.parse(operation)) + const byId = new Map( + operations.flatMap((operation) => + typeof operation.op_id === 'string' + ? [[operation.op_id, operation] as const] + : [] + ) + ) + + return toolCall.applied_op_ids.map((opId) => { + const operation = byId.get(opId) + if (operation === undefined) { + throw new Error( + `tool call ${toolCall.tool_call_id} accepted op ${opId}, but its recorded result does not contain that op` + ) + } + return operation + }) +} + +/** + * Projects a cloud backend capture into the replay format. Websocket frames + * retain their recorded order; accepted semantic ops are inserted immediately + * before the matching terminal tool-call frame, where the live doc update was + * observed. No operation can be authored by this exporter: every emitted op + * must exist in the parent audit result and in its durable accepted-op rows. + */ +export function exportAgentConversation(input: unknown) { + const capture = zBackendCapture.parse(input) + const toolCalls = new Map( + capture.tool_calls.map((toolCall) => [toolCall.tool_call_id, toolCall]) + ) + const emittedToolCalls = new Set() + const response: Array< + | { kind: 'event'; event: z.infer } + | { kind: 'graph_ops'; ops: Array> } + > = [] + + for (const frame of capture.frames) { + if ( + frame.data.thread_id !== capture.capture.thread_id || + frame.data.message_id !== capture.capture.message_id + ) { + throw new Error( + `recorded ${frame.type} frame does not belong to capture ${capture.capture.thread_id}/${capture.capture.message_id}` + ) + } + const event = { + ...frame, + data: { ...frame.data } + } + delete event.data.thread_id + delete event.data.message_id + + if (event.type === 'agent_tool_call') { + const toolCallId = event.data.tool_call_id + const status = event.data.status + if ( + typeof toolCallId === 'string' && + status !== 'running' && + toolCalls.has(toolCallId) + ) { + const operations = operationsFromResult(toolCalls.get(toolCallId)!) + if (operations.length > 0) { + response.push({ kind: 'graph_ops', ops: operations }) + } + emittedToolCalls.add(toolCallId) + } + } + response.push({ kind: 'event', event }) + } + + const omitted = capture.tool_calls.filter( + (toolCall) => + toolCall.applied_op_ids.length > 0 && + !emittedToolCalls.has(toolCall.tool_call_id) + ) + if (omitted.length > 0) { + throw new Error( + `recorded response has no terminal websocket frame for tool call(s): ${omitted.map((call) => call.tool_call_id).join(', ')}` + ) + } + + return zAgentConversation.parse({ + schema_version: 'agent-conversation.v1', + source: { + ...capture.source, + response_side: 'recorded', + capture: capture.capture + }, + workflow: capture.workflow, + request: capture.request, + response + }) +} + +function main(args: string[]): void { + if (args.length !== 2) { + throw new Error( + 'usage: pnpm exec tsx scripts/agentConversationCapture.ts ' + ) + } + const [inputPath, outputPath] = args + const conversation = exportAgentConversation( + JSON.parse(readFileSync(inputPath, 'utf8')) + ) + writeFileSync(outputPath, `${JSON.stringify(conversation, null, 2)}\n`) +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv.slice(2)) +} From 1f447de7b6286e5843215da8a9a55a5cc633b003 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 09:06:14 -0700 Subject: [PATCH 05/49] docs(browser-tests): add replay-coverage convention for agent bug fixes Capture the turn from the standalone agent's local store, convert it to a recorded conversation fixture, add a replay case asserting the canvas-observable outcome, and prove red-on-parent/green-on-fix in the PR. --- browser_tests/README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/browser_tests/README.md b/browser_tests/README.md index 14080d2a449d..d99bc2fa2c4f 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -612,6 +612,34 @@ test.afterEach(async ({ comfyPage }) => { `debugShowCanvasOverlay()`, `debugGetCanvasDataURL()` are for local debugging only. Never commit them. +## Replay coverage for agent bug fixes + +When a fix changes how the agent's turns affect the app (graph edits, +CRDT frames, panel state), add a conversation replay case alongside the +fix so the bug stays fixed: + +1. **Capture the conversation.** Reproduce the bug's turn against the + local agent harness (`scripts/dev-agent-integration.ts`); the + standalone agent records every turn in its local SQLite store + (`agent_messages` / `agent_tool_calls`). Export the turn's rows and + convert them to a conversation JSON under + `browser_tests/fixtures/data/agent/conversations/`, marking + `response_side: 'recorded'`. Until the exporter lands, a fixture may + be hand-assembled from the recorded rows; never invent responses for + a `recorded` fixture. +2. **Add the replay case.** Drive the fixture through the conversation + replay fixture (`agentConversationFixture`), asserting the + canvas-observable outcome the bug corrupted (graph end-state or + panel state - not mock call counts, not model text). +3. **Prove it bites.** Before merging, run the case once against the + fix's parent commit (red) and once at the fix (green); paste both + run lines in the PR description. A replay case that never went red + against the bug does not count as regression coverage. + +Name the case after the fix PR (`agent-fix--.json`) and note +the fixture's source rows (thread id + seq range) in the JSON's +metadata block so the recording can be re-exported. + ## Test Data & Typed API Mocks Mock data in `fixtures/data/` exports **typed** objects that conform to From 9b234814557debc8027a1d360592a4c13fcecbda Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 09:15:06 -0700 Subject: [PATCH 06/49] test(agent): replay regression for #16611 subgraph-scope minting Conversation binds the follower to a doc seeded with a subgraph definition and instance; the edits under test are human canvas deletes (definition-free, so the mocked empty object_info cannot starve them). A root delete must mint (positive control); an interior delete must not, bounded by an ordering anchor (deleting the instance itself) and pinned by node id. Adds an additive outbound-frame capture (clientFrames/outboundOps) to the conversation fixture. --- .../fixtures/agentConversationFixture.ts | 17 +++ .../agent-fix-16611-subgraph-scope.json | 108 ++++++++++++++++++ .../agent/agentFix16611SubgraphScope.spec.ts | 73 ++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json create mode 100644 browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 606634517c4e..90e3413c5d7b 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -145,6 +145,12 @@ interface GraphNodeSnapshot { */ class AgentConversationHarness { readonly postedMessages: string[] = [] + /** + * Every frame the CLIENT sent on /ws, in order (BE-11470 A4): human-op + * minting is observable only on this side of the socket, so regression + * specs assert against these rather than app internals. + */ + readonly clientFrames: { type?: unknown; data?: unknown }[] = [] readonly panel: Locator readonly ack: AgentTurnAccepted @@ -313,10 +319,21 @@ class AgentConversationHarness { this.socket.send(JSON.stringify(frame)) } + /** The ops of every outbound `doc_ops` frame, flattened in send order. */ + outboundOps(): { op?: unknown }[] { + return this.clientFrames + .filter((frame) => frame.type === 'doc_ops') + .flatMap((frame) => { + const ops = (frame.data as { ops?: unknown } | undefined)?.ops + return Array.isArray(ops) ? (ops as { op?: unknown }[]) : [] + }) + } + private onClientFrame(raw: string | Buffer): void { const frame: unknown = JSON.parse(raw.toString()) if (typeof frame !== 'object' || frame === null) return const { type, data } = frame as { type?: unknown; data?: unknown } + this.clientFrames.push({ type, data }) if (type !== 'doc_subscribe' || typeof data !== 'object' || data === null) return const { workflow_id, state_vector_b64 } = data as { diff --git a/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json b/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json new file mode 100644 index 000000000000..9601996faab6 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json @@ -0,0 +1,108 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-fix-16611-subgraph-scope", + "response_side": "synthesized", + "note": "Regression fixture for PR #16611 (preserve root semantics for human graph edits). The conversation only binds the follower to a doc whose seed carries a subgraph definition and instance; the regression surface is the HUMAN edit the spec performs afterwards. Response synthesized: no model output is asserted." + }, + "workflow": { + "id": "a3f8c1d2-9b4e-4f7a-8c50-6e1d2b3a4c5f", + "name": "Subgraph scope", + "catalog": { + "types": { + "LoadImage": { "widget_order": ["image", "upload"] }, + "PreviewImage": { "widget_order": [] }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { "name": "IMAGE", "type": "IMAGE", "links": [] }, + { "name": "MASK", "type": "MASK", "links": [] } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d", + "title": "Inner steps", + "pos": [420, 0], + "size": [220, 100], + "inputs": [], + "outputs": [], + "widgets_values": [] + } + ], + "links": [], + "definitions": { + "subgraphs": [ + { + "id": "9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d", + "name": "Inner steps", + "inputs": [], + "outputs": [], + "nodes": [ + { + "id": 10, + "type": "PreviewImage", + "pos": [40, 40], + "size": [210, 246], + "inputs": [{ "name": "images", "type": "IMAGE", "link": null }], + "outputs": [], + "widgets_values": [] + } + ], + "links": [] + } + ] + } + } + }, + "request": { + "content": "Take a look at this workflow; I will make some edits myself." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "workflow_id": "a3f8c1d2-9b4e-4f7a-8c50-6e1d2b3a4c5f", + "name": "Subgraph scope" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I can see the workflow. Go ahead with your edits." + } + } + }, + { + "kind": "event", + "event": { "type": "agent_message_done", "data": {} } + } + ] +} diff --git a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts new file mode 100644 index 000000000000..569d29dc9846 --- /dev/null +++ b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts @@ -0,0 +1,73 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' + +/** + * Replay regression for PR #16611 (preserve root semantics for human graph + * edits). Pre-fix, a node the user deleted INSIDE a subgraph minted a + * root-scope `delete_node` wire op - the doc lost a root node it still had. + * The fix refuses to mint subgraph-interior deletes (they have no wire + * representation) while root-level deletes keep minting. + * + * The conversation only binds the follower to a doc seeded with a subgraph + * definition and instance; the edits under test are HUMAN deletes through + * the real canvas UI (deletes need no node definitions, so the mocked empty + * object_info cannot starve them). Red at base with 0a20384d8a reverted: + * the interior delete surfaces as an outbound delete op for node 10. + */ +test.describe( + 'Agent replay regression: #16611 subgraph scope', + { tag: '@cloud' }, + () => { + test.use({ conversationCase: 'agent-fix-16611-subgraph-scope' }) + + test('a human delete inside a subgraph does not mint a root wire op', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + const vueNodes = new VueNodeHelpers(page) + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + + const deletedNodeIds = () => + agentConversation + .outboundOps() + .filter((op) => op.op === 'delete_node') + .map((op) => String((op as { node_id?: unknown }).node_id)) + + // Positive control: a ROOT-level human delete must mint, proving the + // doc-bound mint path is live in this environment. + await page.locator('[data-node-id="1"]').click() + await page.keyboard.press('Delete') + await expect.poll(deletedNodeIds, { timeout: 10_000 }).toEqual(['1']) + + // The regression surface: delete a node INSIDE the seeded subgraph. + await vueNodes.enterSubgraph('2') + await page.locator('[data-node-id="10"]').click() + await page.keyboard.press('Delete') + + // Ordering anchor: exit to root and delete the subgraph instance + // itself (a root-level delete that must mint). Frames are sent in + // order, so a minted interior delete would be on the wire before this + // one - waiting for it gives the interior op every chance to surface + // before the negative below is trusted. + await page.getByTestId('subgraph-breadcrumb-back').click() + await page.locator('[data-node-id="2"]').click() + await page.keyboard.press('Delete') + await expect + .poll(() => deletedNodeIds().length, { timeout: 10_000 }) + .toBeGreaterThanOrEqual(2) + + // Exactly the two root deletes, in order; node 10 never minted. + expect(deletedNodeIds()).toEqual(['1', '2']) + }) + } +) From ed90dffbbc4790d42b6e38dc832d570703e5ae75 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 2 Sep 2026 14:46:28 +0000 Subject: [PATCH 07/49] fix(agent-crdt): treat awareness state:null as absent, not a rejected frame (#16653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Resolves an unresolved review question from [#16612](https://github.com/Comfy-Org/ComfyUI_frontend/pull/16612#discussion_r3911665011) that merged before the author answered it. `parseAwarenessState` folded a `state: null` value into the same branch as any other invalid, non-record shape (arrays, strings, numbers), rejecting the *entire* awareness frame including `actor` and `expires_at`. But the Go server's `State map[string]any` has `omitempty` and can never actually emit `null` on the wire — so this was defence-in-depth for a value the server cannot send, and it silently changed behavior from pre-#16612 `main` (where `parseRecord(null)` returned `null` and the field was dropped, but the frame was still accepted). This PR folds `null` into the same branch as `undefined` (no state present) instead of rejecting the frame. A genuinely malformed non-record shape (array, string, number) is still rejected. ## Why treat null as absent, not reject Awareness is ephemeral presence data (cursor/selection), not part of the shared CRDT doc. Dropping a best-effort state update should not also discard the actor's `expires_at`, which is the signal a consumer would use to clear stale presence for that actor. Rejecting the whole frame is strictly more destructive than necessary for a value that, per the server's own schema, means "no state." ## Evidence ``` $ pnpm exec vitest run src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts Test Files 1 passed (1) Tests 10 passed (10) $ pnpm exec vitest run src/workbench/extensions/agent/crdt Test Files 19 passed (19) Tests 207 passed (207) $ pnpm typecheck $ vue-tsc --noEmit [exited with code 0] $ pnpm exec eslint src/workbench/extensions/agent/crdt/docFrameClient.ts src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts (no output, 0 errors) ``` --- .../agent/crdt/docFrameClient.awareness.test.ts | 15 +++++++++++++++ .../extensions/agent/crdt/docFrameClient.ts | 9 ++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts b/src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts index d6d1e2986361..91aa6fb64ec0 100644 --- a/src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts +++ b/src/workbench/extensions/agent/crdt/docFrameClient.awareness.test.ts @@ -37,6 +37,21 @@ describe('awareness frame validation', () => { expect(parseServerDocFrame(awarenessFrame(['cursor', 10, 20]))).toBeNull() }) + it('treats a null state as absent rather than rejecting the frame', () => { + // The Go server's `State map[string]any` has `omitempty` and never + // actually emits `state: null`, but this is defence in depth: null + // should fold into "no state", not discard the whole frame (and with + // it actor/expires_at). discussion_r3911665011. + expect(parseServerDocFrame(awarenessFrame(null, 456))).toEqual({ + type: 'awareness', + data: { + workflowId: 'wf-1', + actor: 'human:user:tab-a', + expiresAt: 456 + } + }) + }) + it('rejects negative expires_at', () => { expect(parseServerDocFrame(awarenessFrame({}, -1))).toBeNull() }) diff --git a/src/workbench/extensions/agent/crdt/docFrameClient.ts b/src/workbench/extensions/agent/crdt/docFrameClient.ts index 648e63ff14e8..9917b4a42544 100644 --- a/src/workbench/extensions/agent/crdt/docFrameClient.ts +++ b/src/workbench/extensions/agent/crdt/docFrameClient.ts @@ -131,7 +131,14 @@ function isNonNegativeInteger(value: unknown): value is number { function parseAwarenessState( value: unknown ): Record | undefined | null { - if (value === undefined) return undefined + // The Go server's `State map[string]any` has `omitempty`, so it never + // emits `state: null` on the wire; nil/empty maps are omitted entirely, + // same as an absent field. Treat null the same as absent (no state) rather + // than rejecting the whole frame, so a value the server cannot actually + // send does not discard `actor`/`expires_at` too. A non-null, non-record + // shape (array, string, number) is still a malformed frame and rejected. + // discussion_r3911665011. + if (value === undefined || value === null) return undefined const state = parseRecord(value) if (state === null) return null From 2c34fd28cb79bba372ba4af69780643af8250055 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 09:19:13 -0700 Subject: [PATCH 08/49] test(agent): replay regression for #16285 catch-up baseline The conversation fixture reproduces the host's exact ordering (doc_subscribed(seq=N) then the catch-up doc_update(seq=N)); pre-fix the bridge dropped that catch-up as stale and a fresh follower rendered empty. The seeded graph projecting at all is the regression surface. The base carries an evolved form of the fix, so the red recipe is a recorded three-line patch restoring the pre-fix baseline write, noted in the spec header. --- .../agent-fix-16285-catchup-baseline.json | 69 +++++++++++++++++++ .../agentFix16285CatchupBaseline.spec.ts | 61 ++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json create mode 100644 browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts diff --git a/browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json b/browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json new file mode 100644 index 000000000000..70c540a3de9a --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json @@ -0,0 +1,69 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-fix-16285-catchup-baseline", + "response_side": "synthesized", + "note": "Regression fixture for PR #16285 (apply CRDT catch-up at acknowledged sequence). The host sends doc_subscribed(seq=N) and then the catch-up doc_update(seq=N); pre-fix the bridge recorded N as the applied baseline and dropped the catch-up as stale, leaving a fresh follower empty. The conversation only binds; the seeded graph projecting at all IS the regression surface. Response synthesized: no model output is asserted." + }, + "workflow": { + "id": "b7e2d4f6-1a3c-4e5b-9d80-2f4a6c8e0b1d", + "name": "Catch-up baseline", + "catalog": { + "types": { + "LoadImage": { "widget_order": ["image", "upload"] }, + "PreviewImage": { "widget_order": [] } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { "name": "IMAGE", "type": "IMAGE", "links": [1] }, + { "name": "MASK", "type": "MASK", "links": [] } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "PreviewImage", + "pos": [400, 0], + "size": [210, 246], + "inputs": [{ "name": "images", "type": "IMAGE", "link": 1 }], + "outputs": [], + "widgets_values": [] + } + ], + "links": [[1, 1, 0, 2, 0, "IMAGE"]] + } + }, + "request": { + "content": "Take a look at this workflow." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "workflow_id": "b7e2d4f6-1a3c-4e5b-9d80-2f4a6c8e0b1d", + "name": "Catch-up baseline" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { "delta": "I can see the workflow." } + } + }, + { "kind": "event", "event": { "type": "agent_message_done", "data": {} } } + ] +} diff --git a/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts b/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts new file mode 100644 index 000000000000..94fe858ab517 --- /dev/null +++ b/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts @@ -0,0 +1,61 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' + +/** + * Replay regression for PR #16285 (apply CRDT catch-up at acknowledged + * sequence). The host sends `doc_subscribed(seq=N)` and THEN the catch-up + * `doc_update(seq=N)` - the conversation fixture reproduces exactly that + * ordering with the same seq on both frames. Pre-fix, the bridge recorded N + * from the acknowledgement as the applied baseline and dropped the catch-up + * as stale, so a fresh follower rendered an EMPTY graph. The seeded graph + * projecting at all is the regression surface. + * + * Red recipe (recorded patch, no revertible commit on this base - the base + * carries an evolved form of the fix): in layoutFollowerBridge's + * onDocSubscribed, restore the pre-fix baseline write + * `this.lastSeq = subscribed.seq ?? null` on the ok path. + */ +test.describe( + 'Agent replay regression: #16285 catch-up baseline', + { tag: '@cloud' }, + () => { + test.use({ conversationCase: 'agent-fix-16285-catchup-baseline' }) + + test('the subscription catch-up projects the seeded graph', async ({ + agentConversation + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // The catch-up is the one applied update; pre-fix it is dropped as + // stale and the count stays at 0. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '1 updates' + ) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '1', + title: 'LoadImage', + inputs: [], + outputs: [true, false] + }, + { + id: '2', + title: 'PreviewImage', + inputs: [true], + outputs: [] + } + ]) + }) + } +) From 23e15ea2d201cf057972c87f0e7ced795e23f711 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 10:00:05 -0700 Subject: [PATCH 09/49] docs(browser-tests): route replay capture through the conversation exporter The recorded-conversation exporter and its capture format landed upstream; the convention now points at them instead of allowing interim hand-assembly, matching the data README's no-hand-written-ops rule. --- browser_tests/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/browser_tests/README.md b/browser_tests/README.md index d99bc2fa2c4f..737bcc0c21a0 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -623,10 +623,11 @@ fix so the bug stays fixed: standalone agent records every turn in its local SQLite store (`agent_messages` / `agent_tool_calls`). Export the turn's rows and convert them to a conversation JSON under - `browser_tests/fixtures/data/agent/conversations/`, marking - `response_side: 'recorded'`. Until the exporter lands, a fixture may - be hand-assembled from the recorded rows; never invent responses for - a `recorded` fixture. + `browser_tests/fixtures/data/agent/conversations/` with + `scripts/agentConversationCapture.ts`, marking + `response_side: 'recorded'` (see `fixtures/data/agent/README.md` for + the capture format and backend query). Never write `graph_ops` by + hand and never relabel a synthesized response as recorded. 2. **Add the replay case.** Drive the fixture through the conversation replay fixture (`agentConversationFixture`), asserting the canvas-observable outcome the bug corrupted (graph end-state or From 718962546585c436d222f4d475092f0061412a48 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 10:09:39 -0700 Subject: [PATCH 10/49] test(agent): fix 16611 exemplar interaction idioms Select via the node header (selectNode helper) with keyboard Delete - body clicks select nothing; seed positions moved clear of the floating action bar, which intercepted the header click at [0,0]. The mint assertion itself still reds: no delete_node op is observed on the wire and no refusal is logged - under diagnosis whether the human delete-mint path is live in the replay environment at all. --- .../agent-fix-16611-subgraph-scope.json | 39 ++++++++++++++----- .../agent/agentFix16611SubgraphScope.spec.ts | 6 +-- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json b/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json index 9601996faab6..44238b06adca 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json @@ -12,8 +12,12 @@ "name": "Subgraph scope", "catalog": { "types": { - "LoadImage": { "widget_order": ["image", "upload"] }, - "PreviewImage": { "widget_order": [] }, + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "PreviewImage": { + "widget_order": [] + }, "KSampler": { "widget_order": [ "seed", @@ -32,12 +36,20 @@ { "id": 1, "type": "LoadImage", - "pos": [0, 0], + "pos": [80, 420], "size": [315, 314], "inputs": [], "outputs": [ - { "name": "IMAGE", "type": "IMAGE", "links": [] }, - { "name": "MASK", "type": "MASK", "links": [] } + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } ], "widgets_values": ["example.png", "image"] }, @@ -45,7 +57,7 @@ "id": 2, "type": "9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d", "title": "Inner steps", - "pos": [420, 0], + "pos": [460, 420], "size": [220, 100], "inputs": [], "outputs": [], @@ -64,9 +76,15 @@ { "id": 10, "type": "PreviewImage", - "pos": [40, 40], + "pos": [120, 380], "size": [210, 246], - "inputs": [{ "name": "images", "type": "IMAGE", "link": null }], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": null + } + ], "outputs": [], "widgets_values": [] } @@ -102,7 +120,10 @@ }, { "kind": "event", - "event": { "type": "agent_message_done", "data": {} } + "event": { + "type": "agent_message_done", + "data": {} + } } ] } diff --git a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts index 569d29dc9846..d852b2717b74 100644 --- a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts +++ b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts @@ -45,13 +45,13 @@ test.describe( // Positive control: a ROOT-level human delete must mint, proving the // doc-bound mint path is live in this environment. - await page.locator('[data-node-id="1"]').click() + await vueNodes.selectNode('1') await page.keyboard.press('Delete') await expect.poll(deletedNodeIds, { timeout: 10_000 }).toEqual(['1']) // The regression surface: delete a node INSIDE the seeded subgraph. await vueNodes.enterSubgraph('2') - await page.locator('[data-node-id="10"]').click() + await vueNodes.selectNode('10') await page.keyboard.press('Delete') // Ordering anchor: exit to root and delete the subgraph instance @@ -60,7 +60,7 @@ test.describe( // one - waiting for it gives the interior op every chance to surface // before the negative below is trusted. await page.getByTestId('subgraph-breadcrumb-back').click() - await page.locator('[data-node-id="2"]').click() + await vueNodes.selectNode('2') await page.keyboard.press('Delete') await expect .poll(() => deletedNodeIds().length, { timeout: 10_000 }) From 04f12bebbb2fb068412babac6772a6dc67aff116 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 10:40:00 -0700 Subject: [PATCH 11/49] test: correct the 16285 red recipe to the verified two-line patch --- .../tests/agent/agentFix16285CatchupBaseline.spec.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts b/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts index 94fe858ab517..1b582c47d94f 100644 --- a/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts +++ b/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts @@ -11,10 +11,13 @@ import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFi * as stale, so a fresh follower rendered an EMPTY graph. The seeded graph * projecting at all is the regression surface. * - * Red recipe (recorded patch, no revertible commit on this base - the base - * carries an evolved form of the fix): in layoutFollowerBridge's - * onDocSubscribed, restore the pre-fix baseline write - * `this.lastSeq = subscribed.seq ?? null` on the ok path. + * Red recipe (recorded two-line patch, no revertible commit on this base - + * the base carries an evolved form of the fix with two independent + * defenses): in layoutFollowerBridge, (1) restore the pre-fix baseline + * write `this.lastSeq = subscribed.seq ?? null` on onDocSubscribed's ok + * path AND (2) disable the catch-up carve-out in onDocUpdate + * (`const isCatchUp = false`). Either line alone stays green; both + * together reproduce the pre-fix drop (status shows `0 updates`). */ test.describe( 'Agent replay regression: #16285 catch-up baseline', From e820fb7e448368c4f14de750d17bf043a2c37094 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 11:07:57 -0700 Subject: [PATCH 12/49] test: drive the 16611 exemplar at the layout store level The UI-path deletes the previous revision drove cannot see ECS-only replay seeds (litegraph-first selection), so the human edits now apply layout operations directly with the session's local user actor, exercising the 0a20384d8a scope contract exactly: root deletes mint, subgraph-interior creates and deletes never do. The conversation host now acknowledges doc_ops batches (versioned doc_ops_result) - the client's one-in-flight op sender stalls after the first mint without the ack, as the real host always sends one. --- .../fixtures/agentConversationFixture.ts | 27 +++- .../agent/agentFix16611SubgraphScope.spec.ts | 127 ++++++++++++++---- 2 files changed, 127 insertions(+), 27 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 90e3413c5d7b..603f3937b834 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -37,7 +37,7 @@ const SEND_LABEL = enMessages.agent.send const STOP_LABEL = enMessages.agent.stop interface DocFrame { - type: 'doc_subscribed' | 'doc_update' + type: 'doc_subscribed' | 'doc_update' | 'doc_ops_result' data: Record } @@ -334,6 +334,31 @@ class AgentConversationHarness { if (typeof frame !== 'object' || frame === null) return const { type, data } = frame as { type?: unknown; data?: unknown } this.clientFrames.push({ type, data }) + if (type === 'doc_ops' && typeof data === 'object' && data !== null) { + // The real host acknowledges every submitted batch; the client's op + // sender keeps one batch in flight and only transmits the next after + // the ack, so without this reply every mint after the first stalls. + const { workflow_id, ops } = data as { + workflow_id?: unknown + ops?: unknown + } + if (workflow_id === this.conversation.workflow.id && Array.isArray(ops)) { + const applied = ops + .map((op) => (op as { op_id?: unknown }).op_id) + .filter((id): id is string => typeof id === 'string') + this.send({ + type: 'doc_ops_result', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id, + ok: true, + applied, + skipped: [] + } + }) + } + return + } if (type !== 'doc_subscribe' || typeof data !== 'object' || data === null) return const { workflow_id, state_vector_b64 } = data as { diff --git a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts index d852b2717b74..092254745826 100644 --- a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts +++ b/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts @@ -1,21 +1,42 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' /** * Replay regression for PR #16611 (preserve root semantics for human graph * edits). Pre-fix, a node the user deleted INSIDE a subgraph minted a * root-scope `delete_node` wire op - the doc lost a root node it still had. - * The fix refuses to mint subgraph-interior deletes (they have no wire - * representation) while root-level deletes keep minting. + * The fix refuses to mint subgraph-interior creates/deletes (they have no + * wire representation) while root-level deletes keep minting. * - * The conversation only binds the follower to a doc seeded with a subgraph - * definition and instance; the edits under test are HUMAN deletes through - * the real canvas UI (deletes need no node definitions, so the mocked empty - * object_info cannot starve them). Red at base with 0a20384d8a reverted: - * the interior delete surfaces as an outbound delete op for node 10. + * The conversation binds the follower to a doc seeded with a subgraph + * definition and instance; the human edits are then driven at the layout + * store level with the session's own local (user) actor, mirroring the + * operations the canvas attach/detach emitters produce. Store-level driving + * is deliberate: UI-path deletes route through litegraph selection and + * cannot see ECS-only replay seeds (see the 16611 red-row diagnosis on the + * DOPPELGANGER review pad) - the mint gate contract under test is exercised + * exactly, the UI hit-testing above it is not. + * + * Red at base with 0a20384d8a reverted (reverse-apply the commit, then + * RESTART the dev server - the spec imports the layout store by module URL, + * which only resolves to the app's own instance against a module graph that + * matches disk; running against an HMR-churned server loads a second, empty + * store instance): the interior delete surfaces as an outbound + * `delete_node` for node 10 (`['1', '10', '2']`). */ + +/** The layout-store surface this spec drives, structurally typed. */ +interface LayoutStoreModule { + layoutStore: { + getNodeLayout( + graphId: string, + nodeId: string + ): { position: { x: number; y: number } } | null + applyOperation(operation: Record): void + } +} + test.describe( 'Agent replay regression: #16611 subgraph scope', { tag: '@cloud' }, @@ -28,7 +49,6 @@ test.describe( }) => { test.setTimeout(45_000) const { panel } = agentConversation - const vueNodes = new VueNodeHelpers(page) await agentConversation.sendPrompt() await agentConversation.replayResponse() @@ -36,6 +56,7 @@ test.describe( await expect(panel.getByTestId('agent-crdt-status')).toContainText( 'connected' ) + await expect(page.locator('[data-node-id="1"]')).toBeVisible() const deletedNodeIds = () => agentConversation @@ -43,25 +64,79 @@ test.describe( .filter((op) => op.op === 'delete_node') .map((op) => String((op as { node_id?: unknown }).node_id)) - // Positive control: a ROOT-level human delete must mint, proving the - // doc-bound mint path is live in this environment. - await vueNodes.selectNode('1') - await page.keyboard.press('Delete') - await expect.poll(deletedNodeIds, { timeout: 10_000 }).toEqual(['1']) + const SUBGRAPH_ID = '9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d' + const WORKFLOW_ID = agentConversation.conversation.workflow.id + const STORE_MODULE = '/src/renderer/core/layout/store/layoutStore.ts' + await page.evaluate( + async ({ subgraphId, workflowId, storeModule }) => { + const { layoutStore } = (await import( + /* @vite-ignore */ storeModule + )) as unknown as LayoutStoreModule + // The follower keys layouts under the canvas root graph id; earlier + // revisions keyed them under the doc's workflow id. Resolving by + // lookup keeps the red half (fix reverted) driving the same seeded + // layouts instead of erroring before the contract assertion. + const rootGraphId = [window.app!.graph.id, workflowId].find((id) => + layoutStore.getNodeLayout(id, '1') + ) + if (!rootGraphId) { + throw new Error( + 'seeded layout for node 1 not found under any graph' + ) + } + const meta = (nodeId: string, ownerGraphId: string) => ({ + graphId: rootGraphId, + ownerGraphId, + nodeId, + timestamp: Date.now(), + source: 'canvas' + }) - // The regression surface: delete a node INSIDE the seeded subgraph. - await vueNodes.enterSubgraph('2') - await vueNodes.selectNode('10') - await page.keyboard.press('Delete') + // Positive control: a ROOT-level delete must mint, proving the + // doc-bound mint path is live in this environment. + layoutStore.applyOperation({ + type: 'deleteNode', + ...meta('1', rootGraphId) + }) + + // The regression surface: a node INSIDE the seeded subgraph. Its + // layout attaches the way entering the subgraph attaches it + // (ownerGraphId = the subgraph), then the user deletes it. Both the + // create and the delete are unrepresentable on the wire - neither + // may mint. + layoutStore.applyOperation({ + type: 'createNode', + ...meta('10', subgraphId), + layout: { + id: '10', + position: { x: 120, y: 380 }, + size: { width: 210, height: 246 }, + zIndex: 0, + visible: true, + bounds: { x: 120, y: 380, width: 210, height: 246 } + } + }) + layoutStore.applyOperation({ + type: 'deleteNode', + ...meta('10', subgraphId) + }) + + // Ordering anchor: a second root-level delete that must mint. + // Frames are sent in order, so a minted interior delete would be on + // the wire before this one - waiting for it gives the interior op + // every chance to surface before the negative below is trusted. + layoutStore.applyOperation({ + type: 'deleteNode', + ...meta('2', rootGraphId) + }) + }, + { + subgraphId: SUBGRAPH_ID, + workflowId: WORKFLOW_ID, + storeModule: STORE_MODULE + } + ) - // Ordering anchor: exit to root and delete the subgraph instance - // itself (a root-level delete that must mint). Frames are sent in - // order, so a minted interior delete would be on the wire before this - // one - waiting for it gives the interior op every chance to surface - // before the negative below is trusted. - await page.getByTestId('subgraph-breadcrumb-back').click() - await vueNodes.selectNode('2') - await page.keyboard.press('Delete') await expect .poll(() => deletedNodeIds().length, { timeout: 10_000 }) .toBeGreaterThanOrEqual(2) From f138a9fd93fdfdf7ae01e828f7a0d0af10bdc1e1 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 18:35:16 -0700 Subject: [PATCH 13/49] test(agent): add eight recorded conversation replays Eight conversations recorded against the local agent (six with graph operations applied through the doc host, one text-only answer, one tool error), exported with the conversation exporter unchanged, and replayed with a full graph snapshot and response-panel assertions each. The fixtures carry response_side "recorded" and a capture block (thread and message ids, parent row ids, raw capture hash) so any case can be re-exported from the same source. --- .../conversations/agent-rec-batched-ops.json | 379 +++++++++++ .../agent-rec-refiner-between.json | 442 +++++++++++++ .../agent-rec-replace-prompt-encoder.json | 405 ++++++++++++ .../agent-rec-second-prompt-encoder.json | 379 +++++++++++ .../agent-rec-set-widget-existing.json | 317 +++++++++ .../agent-rec-string-node-into-prompt.json | 360 +++++++++++ .../agent-rec-text-only-answer.json | 266 ++++++++ .../conversations/agent-rec-tool-error.json | 267 ++++++++ .../agent/agentConversationReplay.spec.ts | 608 ++++++++++++++++++ 9 files changed, 3423 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json new file mode 100644 index 000000000000..eba81063518c --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json @@ -0,0 +1,379 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-batched-ops", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 62ff8e48-248c-48cf-9fed-24b21aaa141a; message d896b67c-2e87-4312-876c-2f0765b823b3; workflow 7487a6dc-a100-4641-b76b-feba99d6fd1e (seeded by throwaway turn c850dd73-373b-4367-a363-9ff4f081affe; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['26779044-2bc9-4728-b9ec-1b389475ea22', '04a41da6-1bb7-4f78-b5f6-185dc11bf1a8', 'd845093c-b8f5-4a84-a7ac-d5d194911bfc']; rows agent-rec-batched-ops.r6-a1.rows.json; raw capture sha256 50dd0603285e88b34ecd0ff1864c44364a79244b398778a29c7ccb714c6be0fb", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "62ff8e48-248c-48cf-9fed-24b21aaa141a", + "message_id": "d896b67c-2e87-4312-876c-2f0765b823b3", + "exported_at": "2026-09-03T01:10:17Z" + } + }, + "workflow": { + "id": "7487a6dc-a100-4641-b76b-feba99d6fd1e", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first, then read its graph to find the checkpoint loader's real id." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "7487a6dc-a100-4641-b76b-feba99d6fd1e" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 7, + "status": "success", + "tool_call_id": "toolu_01T3HLiCeFHR77H9rYxPaqrG", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 723, + "status": "success", + "tool_call_id": "toolu_01TtC8NC6yVXCzk3nW8mJRC6", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3495500264284537, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8d0af55a29224e089eab1547177b2cb0", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3495500264284537, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 406], + "node": { + "id": 4454558117922341, + "pos": [715, 406], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c5d06a0a0800452794e4eec01fdd27e9", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4454558117922341, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "01727acc9c784d5b85975265e1112574", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 95739764921160, + "to_node": 3495500264284537, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "70380b8a26f64d558b56a050489544c2", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3128959453071432, + "to_node": 4454558117922341, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 243, + "status": "success", + "tool_call_id": "toolu_016atkf2nc3JCG6ZAx3L5FzP", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — focus is on \"Text to image,\" and both new text-encode nodes are on the canvas with their CLIP inputs fed from the checkpoint loader, all in one batch. They're not wired into the sampler yet, so tell me what each should feed (e.g. one as the negative prompt) and I'll connect them." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1500, + "cache_read_input_tokens": 153287, + "input_tokens": 8, + "output_tokens": 517, + "total_tokens": 525 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json new file mode 100644 index 000000000000..cb0dacd6f46f --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json @@ -0,0 +1,442 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-refiner-between", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2b47ebe1-d43e-42cc-88ea-47c6110f8594; message 1ca0e0eb-4475-4d47-92c4-4842e664c85d; workflow 4c2246c4-bf06-4117-b85d-7dd86eb1142c (seeded by throwaway turn 87004117-f7b4-4aa6-9d3c-03adfa69b405; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d04fab84-761f-4d29-96b6-14acb059fbd8', '30062b8a-f0cb-4035-b978-6fb486a4a6bc', '0506f122-389e-4e3d-8384-75652a672b86', '02cc5714-4eb8-4cac-878c-d7be00829fa8', '05657151-ffc2-4aa8-be17-c0b919befbef']; rows agent-rec-refiner-between.r5-a1.rows.json; raw capture sha256 60d75f59c47625f231fc68272617204751373cbd1b99454ca21467cee479d5c9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "2b47ebe1-d43e-42cc-88ea-47c6110f8594", + "message_id": "1ca0e0eb-4475-4d47-92c4-4842e664c85d", + "exported_at": "2026-09-03T01:09:03Z" + } + }, + "workflow": { + "id": "4c2246c4-bf06-4117-b85d-7dd86eb1142c", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and look at the graph." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "4c2246c4-bf06-4117-b85d-7dd86eb1142c" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 5, + "status": "success", + "tool_call_id": "toolu_01VGCkwgpcwHirAK5wpLEWh6", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 574, + "status": "success", + "tool_call_id": "toolu_01GL3qKHcRRTkxVZeDXmoN3K", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1565, 280], + "node": { + "id": 2629930598730104, + "pos": [1565, 280], + "mode": 0, + "size": [240, 290], + "type": "KSampler", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "model", + "type": "MODEL" + }, + { + "link": null, + "name": "positive", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "negative", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "latent_image", + "type": "LATENT" + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "96148885872a4913a0a017d5f9590184", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2629930598730104, + "class_type": "KSampler", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "bdbef333346f4b71910e48bb295cc9e1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 507448049071742, + "to_node": 2629930598730104, + "to_slot": 0, + "from_node": 4, + "from_slot": 0, + "link_type": "MODEL", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e864aa7ae7234c5fae328b9852cb0e4f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2030924172213340, + "to_node": 2629930598730104, + "to_slot": 1, + "from_node": 6, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7facf6240836428289b0430655ae082a", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1914333209534315, + "to_node": 2629930598730104, + "to_slot": 3, + "from_node": 3, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "5e0988e2e87943a693a4f823cead58bb", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3005141419537112, + "to_node": 8, + "to_slot": 0, + "from_node": 2629930598730104, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "ff7b38b3eb7642edbe6e26b434813f9a", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 20, + "widget": "steps", + "node_id": 2629930598730104, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 1, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "f720afe2838649cb9a0b60a18387284c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 0.5, + "widget": "denoise", + "node_id": 2629930598730104, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 8, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "13ce136537ef462f9c4cae51dffc5c51", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 7, + "widget": "cfg", + "node_id": 2629930598730104, + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 250, + "status": "success", + "tool_call_id": "toolu_019JQMWFbRUwQJTnvgpPP3ti", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 197, + "status": "success", + "tool_call_id": "toolu_019MucYksSrwHhjJwSS8YTCu", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 197, + "status": "error", + "tool_call_id": "toolu_01CMStzBtWs89YdxxxAbe9SB", + "tool_name": "validate" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the refiner is in and wired through: first sampler's latent → refiner → VAE decoder → save. I gave it 20 steps at 0.5 denoise so it polishes rather than re-invents the image; say the word if you want that stronger or weaker.\n\nTwo things were already missing in this graph before I touched it, and they'll stop a run:\n\n- Neither sampler has a negative prompt connected, and the first sampler has no starting latent (no empty-image/size node).\n- The checkpoint it names, `sd_xl_base_1.0`, isn't installed here — no checkpoint options are showing at all.\n\nWant me to add the negative prompt and the image-size node, and pick an available checkpoint?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2949, + "cache_read_input_tokens": 232730, + "input_tokens": 12, + "output_tokens": 1622, + "total_tokens": 1634 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json new file mode 100644 index 000000000000..6a5f16ca0c51 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json @@ -0,0 +1,405 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-replace-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6ee091cb-1d46-4b7b-bcc1-5002c1908e2a; message 2d192b16-5e0b-4bdd-b0dd-d1293950f37a; workflow 2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6 (seeded by throwaway turn 4dd6f9b1-9b01-4e75-9e3e-710c63c8a158; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d9725d24-b208-456c-9fea-cbd99272a73a', 'e2dc9c25-abe9-41b8-8c0b-5fc3ad99ff09', '7153430f-a424-463c-9d5d-7603252d3f67', '5ce083d4-2c3f-44d6-8ceb-fd79d99a6375', 'bea39c10-b218-4945-bdcb-b09701c3e472', '9bc04a9e-9d43-47aa-816d-d92f98ed521c']; rows agent-rec-replace-prompt-encoder.r4-a1.rows.json; raw capture sha256 bb270e40c444456057ed4b6d188e12f452282a9af823d3a21f6c7a5c29dc234c", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "6ee091cb-1d46-4b7b-bcc1-5002c1908e2a", + "message_id": "2d192b16-5e0b-4bdd-b0dd-d1293950f37a", + "exported_at": "2026-09-03T01:07:33Z" + } + }, + "workflow": { + "id": "2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read the graph first." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01W3neXmtRXyGw31YjNTfwd4", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 602, + "status": "success", + "tool_call_id": "toolu_01KNsLCccDucdm2Bknj7JJeN", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "703a1427e3d844ba9c525c8e171342b7", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 6, + "base_version": 2, + "removed_links": [1, 2] + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 198, + "status": "success", + "tool_call_id": "toolu_018HhyebfnrUDExa4ZFTip5x", + "tool_name": "delete_node" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 4073815636872683, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7406a83a71234d63991d53dc1394327b", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4073815636872683, + "class_type": "CLIPTextEncode", + "base_version": 3 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7bbed04e5a9b4d1c984635bc738bb631", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 909865724937361, + "to_node": 4073815636872683, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 3 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "43dfb8cd293c4eb482e17a7f01e7c791", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4412673605160400, + "to_node": 3, + "to_slot": 1, + "from_node": 4073815636872683, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 3 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "aa40bf71be2e444e86d24e2da3d04c27", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier", + "widget": "text", + "node_id": 4073815636872683, + "base_version": 3 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01Jq7JbdaxsTDJXfc4R2kX3r", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01H3psK8qCtRWqQjxium2rJv", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 202, + "status": "error", + "tool_call_id": "toolu_01Jn4m6BiqVX3XTNRTQYLJ9x", + "tool_name": "validate" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the old positive prompt node is gone, and a fresh one is in its place on the \"Text to image\" tab: CLIP from the checkpoint loader in, conditioning into the sampler's positive input, prompt text carried over (\"a photo of a pier\").\n\nTwo things that were already missing before this swap and still are: the sampler has no negative prompt and no empty-latent (image size) input wired, so it won't run as-is. Want me to add those?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2851, + "cache_read_input_tokens": 272173, + "input_tokens": 14, + "output_tokens": 1015, + "total_tokens": 1029 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json new file mode 100644 index 000000000000..a41533bb0024 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json @@ -0,0 +1,379 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-second-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 70f97c01-00e9-4e8d-86bc-ff73cf7500e2; message b55b90fa-b206-4c7b-9696-79fcb22ab8db; workflow 087617ce-c168-4bb8-86b2-05492db91a8d (seeded by throwaway turn 1bdaa2f6-3560-4405-92ec-bbe4c8f5370f; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['8e612681-1ad2-4fdb-8395-1d3e68566c7f', '156eed82-d0f4-40d1-994a-9a3c864169e0', '9c7e33ca-d9b0-4623-99a0-7f5b6a095b51', '0860390b-de13-44e2-927b-ed75988f387a', 'bebaa9e9-242f-4ba5-82e8-7d7b29664009']; rows agent-rec-second-prompt-encoder.r2-a1.rows.json; raw capture sha256 09f8ce5f32c0c30a565a65f43ef572f8ff52b8fb1793de667a056dc87b1ab4a9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "70f97c01-00e9-4e8d-86bc-ff73cf7500e2", + "message_id": "b55b90fa-b206-4c7b-9696-79fcb22ab8db", + "exported_at": "2026-09-03T01:04:16Z" + } + }, + "workflow": { + "id": "087617ce-c168-4bb8-86b2-05492db91a8d", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and wire its conditioning into the KSampler's negative input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "087617ce-c168-4bb8-86b2-05492db91a8d" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01RxsV9XAFq5tdPvcjqxzf7X", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 867, + "status": "success", + "tool_call_id": "toolu_01JUkZjF5v8B1g5B4oiPUbhh", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2904574201837211, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "30fc2175ec794e10bd7ba38aa43a063c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2904574201837211, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4b4bf9278ba840028f7ca4a4c03a7648", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 967330603592352, + "to_node": 2904574201837211, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "5469e1e4858b4565a0a0244011298a93", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2932910430668226, + "to_node": 3, + "to_slot": 2, + "from_node": 2904574201837211, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e11c18ea33a1402396bf6743884f9efa", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "blurry, low quality, watermark", + "widget": "text", + "node_id": 2904574201837211, + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 225, + "status": "success", + "tool_call_id": "toolu_012JnLUhqxiargezRkF2zY4J", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 209, + "status": "success", + "tool_call_id": "toolu_01Wn2YU8i3efNRfoQNgBRN9d", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 256, + "status": "error", + "tool_call_id": "toolu_011Cqp8r4x5nnjChpoJNnTg1", + "tool_name": "validate" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — that tab now has a second text encode node reading from the checkpoint's CLIP, feeding the sampler's negative input. I gave it a starter negative prompt (\"blurry, low quality, \") — change the text to whatever you like.\n\nTwo pre-existing issues in that graph, unrelated to this change: the sampler still has nothing wired into its latent input (it needs an empty latent to set the image size), and the checkpoint it points at isn't installed here. Want me to add the empty latent and pick an available checkpoint?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2524, + "cache_read_input_tokens": 232347, + "input_tokens": 12, + "output_tokens": 854, + "total_tokens": 866 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json new file mode 100644 index 000000000000..1e6d3c940cce --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json @@ -0,0 +1,317 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-set-widget-existing", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 42ed6743-f0f3-4bdd-99f2-852657590361; message 52c2229a-b77b-4596-b921-dcbeec3ba16d; workflow 99036750-1b1b-4ba9-8a47-bc81358f8b09 (seeded by throwaway turn bb6e501d-056e-4e80-a7fa-0938dc18c684; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['6b97831c-40e0-4105-93de-666025526b3f', '31482711-0a5e-4650-95f0-dc4422bf94b1', '6ea3de75-0d0a-4fbc-9b1b-33d9a317b5b2', '33133dbc-9880-46b8-8793-9636015db2ba']; rows agent-rec-set-widget-existing.r3-a1.rows.json; raw capture sha256 7b61b8cc5530dd548a3fa632459a553d5d6fb6d4687181cca45e2f1027a8e81b", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "42ed6743-f0f3-4bdd-99f2-852657590361", + "message_id": "52c2229a-b77b-4596-b921-dcbeec3ba16d", + "exported_at": "2026-09-03T01:06:06Z" + } + }, + "workflow": { + "id": "99036750-1b1b-4ba9-8a47-bc81358f8b09", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "99036750-1b1b-4ba9-8a47-bc81358f8b09" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 4, + "status": "success", + "tool_call_id": "toolu_01RRsWS3QvX5bmVH9wZJcJAL", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 618, + "status": "success", + "tool_call_id": "toolu_011KWGg8bY3qf8pZQCigaA5b", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "211d808e180b4d82926799ede49a7b41", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 30, + "widget": "steps", + "node_id": 3, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 7, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b4a6980f6bfa400d9b7d59630ca3adef", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 5, + "widget": "cfg", + "node_id": 3, + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 196, + "status": "success", + "tool_call_id": "toolu_01Fto3Lreshdk2LVPjcgHaUn", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 348, + "status": "success", + "tool_call_id": "toolu_012xgj4F53Nyavkc86wk379c", + "tool_name": "list_slots" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — you're on the \"Text to image\" tab, with steps now 30 and cfg 5. Everything else is untouched." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1557, + "cache_read_input_tokens": 192283, + "input_tokens": 10, + "output_tokens": 420, + "total_tokens": 430 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json new file mode 100644 index 000000000000..f9aefc16b061 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json @@ -0,0 +1,360 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-string-node-into-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread cb0eb98e-919d-4082-bccf-0b72aaa8baea; message 2ece02a3-a880-4632-b154-03462cd3aa64; workflow 38d9e9c7-e0fb-48f2-8f9a-391f7afba70c (seeded by throwaway turn 271d7739-d3bc-425c-b1ec-aaa4161e969b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['eb68d851-1fce-4a04-9ae8-88c37e6fd174', '6b77837c-c36b-4867-b003-b54c679e6ed0', '70c4c139-0bfa-4916-a84d-42c04ccc2b22', '2d6e1781-b0c0-469f-af4a-129bf6564d56', '5d89cbad-eebd-40d0-a759-062687ce1ada']; rows agent-rec-string-node-into-prompt.r1-a1.rows.json; raw capture sha256 eab6f497cbed85b5cfb6a73bc0ddd5df077c07a29b6113ef46e528b260f74932", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "cb0eb98e-919d-4082-bccf-0b72aaa8baea", + "message_id": "2ece02a3-a880-4632-b154-03462cd3aa64", + "exported_at": "2026-09-03T01:00:58Z" + } + }, + "workflow": { + "id": "38d9e9c7-e0fb-48f2-8f9a-391f7afba70c", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a PrimitiveStringMultiline node with the text 'a photo of a pier at sunset' and wire its output into the positive prompt's text input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "38d9e9c7-e0fb-48f2-8f9a-391f7afba70c" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_018Brjqc2b4FPd4i4Vq2mX5G", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 609, + "status": "success", + "tool_call_id": "toolu_01JoeZB9LHeoCe9CUbpP6Gew", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 205, + "status": "success", + "tool_call_id": "toolu_01BkfK6eCmKdd1fm4jNL27Yw", + "tool_name": "show_node" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [80, 120], + "node": { + "id": 1896929324796827, + "pos": [80, 120], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "2c9a6b7dfaeb4051bb178630885ede81", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1896929324796827, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4d6129913682457399beb255048d12a8", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier at sunset", + "widget": "value", + "node_id": 1896929324796827, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "56c9e65d3aea42c5b3a809cd0e5f8563", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 50487610085703, + "to_node": 6, + "to_slot": 1, + "from_node": 1896929324796827, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 217, + "status": "success", + "tool_call_id": "toolu_017R8ze2T1Mnbxe2zonbMJ6a", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 186, + "status": "success", + "tool_call_id": "toolu_019LTDkGn5iwWv8CDTj9ie8v", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the \"Text to image\" tab now has a text node reading \"a photo of a pier at sunset\" feeding the positive prompt.\n\nOne thing to flag before you run it: the sampler's negative prompt and latent image inputs are both empty, so the workflow won't run as-is. Want me to add an empty latent (with a size) and a negative prompt node to complete it?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2197, + "cache_read_input_tokens": 231797, + "input_tokens": 12, + "output_tokens": 675, + "total_tokens": 687 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json new file mode 100644 index 000000000000..d5bf778cf0a7 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json @@ -0,0 +1,266 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-text-only-answer", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running STANDALONE (AGENT_STANDALONE=true, SQLite) via scripts/dev-agent-integration.ts (PR 16643) at http://127.0.0.1:6207 (frames: ws http://127.0.0.1:6207/api/agent/events); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread e821b43c-f65e-4c4d-975c-76ecf4bfad9b; message 5d3c44f5-b880-42c9-8e4d-46b19f600c15; workflow edb2a004-8c29-48b8-bfd6-3c45effe82dd (seeded by throwaway turn 2e6c4667-5da1-42e3-a8f8-2506a5e6facc; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['29fcbef7-aaf9-4ea0-a34d-792a3f1b369a', 'a914f925-df3a-4f1b-84b4-3fe41a3048b4']; rows agent-rec-text-only-answer.r7-a3.agent.db; raw capture sha256 7ba7d66c074d6689145ef39b7108a7b5e8079b10d0fe4bb0425024f6c28881b0", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "e821b43c-f65e-4c4d-975c-76ecf4bfad9b", + "message_id": "5d3c44f5-b880-42c9-8e4d-46b19f600c15", + "exported_at": "2026-09-03T00:49:49Z" + } + }, + "workflow": { + "id": "edb2a004-8c29-48b8-bfd6-3c45effe82dd", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then tell me what this workflow does." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read what's there." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "edb2a004-8c29-48b8-bfd6-3c45effe82dd" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 2, + "status": "success", + "tool_call_id": "toolu_01Giap3Gbnx9T7ryXJ1z3cPY", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 649, + "status": "success", + "tool_call_id": "toolu_011QhzTDAPLJvdUS4ZLUNAUQ", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now. It's a basic text-to-image setup built on Stable Diffusion XL: it takes a written description — currently \"a photo of a pier\" — and generates an image from it at 20 steps, then saves the result.\n\nOne thing worth flagging: it's not runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt (what to avoid) and no canvas size/blank image for it to start from. Want me to wire those in so you can run it?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 977, + "cache_read_input_tokens": 114270, + "input_tokens": 6, + "output_tokens": 356, + "total_tokens": 362 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json new file mode 100644 index 000000000000..8e0a0c34f79a --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json @@ -0,0 +1,267 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-tool-error", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running STANDALONE (AGENT_STANDALONE=true, SQLite) via scripts/dev-agent-integration.ts (PR 16643) at http://127.0.0.1:6207 (frames: ws http://127.0.0.1:6207/api/agent/events); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 94cec02e-de43-4d42-889f-b899d89a80f7; message a9b5cf3a-441e-4f0a-a6a3-9c3e2f164bef; workflow c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17 (seeded by throwaway turn fad6ef22-88ee-4df6-bd4d-b542b396ac4d; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['39a8b2a7-ddf6-4895-894d-592903605ff4', 'e9b6e0f3-77c8-4686-b62e-a4a54a2a8a27']; rows agent-rec-tool-error.r8-a1.agent.db; raw capture sha256 d1f8d5251494fcadd1ad1a06a524b9739470a398cc027fb24fb612dca69ce4d8", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "94cec02e-de43-4d42-889f-b899d89a80f7", + "message_id": "a9b5cf3a-441e-4f0a-a6a3-9c3e2f164bef", + "exported_at": "2026-09-03T00:57:22Z" + } + }, + "workflow": { + "id": "c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 1, + "status": "success", + "tool_call_id": "toolu_01AeJR983cpdg2RLtC6keWCu", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 658, + "error_code": "node_not_found", + "status": "error", + "tool_call_id": "toolu_0196pmxhdJcMYPkE5PdiLc2t", + "tool_name": "add_node" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm now on the **Text to image** tab, but the node couldn't be added — there's no node class called \"DoesNotExist\" in the catalog, and there aren't any close matches to it either. Your canvas is unchanged.\n\nIf you tell me what that node was meant to do, I'll find the real one and wire it in." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 755, + "cache_read_input_tokens": 114297, + "input_tokens": 6, + "output_tokens": 288, + "total_tokens": 294 + } + } + } + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 235ec38448af..996ac2330316 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -134,4 +134,612 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { ]) }) }) + + test.describe('recorded agent-rec-text-only-answer', () => { + test.use({ conversationCase: 'agent-rec-text-only-answer' }) + + test('switches to the seeded tab and answers in text without touching the graph', async ({ + agentConversation + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // A passing group collapses at turn end; the header count is the only + // stable part of its name (the duration is the recording's). + const toolGroup = panel.getByRole('button', { name: /^Ran 2 tool calls/ }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Print workflow' })).toBeVisible() + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + // An assistant text part rendered; the model prose itself is never pinned. + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // Zero graph_ops entries: the catch-up is the one applied update. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '1 updates' + ) + + // The seed is untouched: five nodes with the seed's own wiring. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } + ]) + }) + }) + + test.describe('recorded agent-rec-tool-error', () => { + test.use({ conversationCase: 'agent-rec-tool-error' }) + + test('keeps a failed add_node group open and leaves the graph untouched', async ({ + agentConversation + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // The failed group stays open at turn end: its rows are visible with NO + // expand click, and aria-expanded is read off the group's trigger button. + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Add node' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect( + panel.getByRole('button', { name: /^Ran 2 tool calls/ }) + ).toHaveAttribute('aria-expanded', 'true') + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // Zero graph_ops entries: the catch-up is the one applied update. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '1 updates' + ) + + // No new node: the seed's five nodes with the seed's own wiring. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } + ]) + }) + }) + + test.describe('recorded agent-rec-string-node-into-prompt', () => { + test.use({ conversationCase: 'agent-rec-string-node-into-prompt' }) + + test('adds a recorded string node, sets its text, and wires it into the positive prompt', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { name: /^Ran 5 tool calls/ }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows).toHaveCount(5) + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Show node' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // One graph_ops entry (add_node + set_widget + connect) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + + // The recorded node id, its text widget value on the unfiltered DOM. + const stringNode = page.locator('[data-node-id="1896929324796827"]') + await expect(stringNode).toBeVisible() + await expect(stringNode.getByLabel('value', { exact: true })).toHaveValue( + 'a photo of a pier at sunset' + ) + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + // The wired `text` input is widget-backed and renders as a widget, + // not a slot row; the link shows on the string node's output. + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '1896929324796827', + title: 'PrimitiveStringMultiline', + inputs: [], + outputs: [true] + } + ]) + }) + }) + + test.describe('recorded agent-rec-second-prompt-encoder', () => { + test.use({ conversationCase: 'agent-rec-second-prompt-encoder' }) + + test('adds a recorded second encoder wired from CLIP into the negative input', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // The recording's closing `validate` call errored (the seed's checkpoint + // is not installed on the recording host), so the group is a failed one: + // it stays open at turn end and its rows are visible with NO click. + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Validate' })).toBeVisible() + await expect( + panel.getByRole('button', { name: /^Ran 5 tool calls/ }) + ).toHaveAttribute('aria-expanded', 'true') + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // One graph_ops entry (add_node + set_widget + connect x2) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + await expect( + page.locator('[data-node-id="2904574201837211"]') + ).toBeVisible() + + // The sampler's negative input (index 2) flips false -> true; the new + // encoder is wired on both sides. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, true, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '2904574201837211', + title: 'CLIPTextEncode', + inputs: [true], + outputs: [true] + } + ]) + }) + }) + + test.describe('recorded agent-rec-set-widget-existing', () => { + test.use({ conversationCase: 'agent-rec-set-widget-existing' }) + + test('applies recorded set_widget ops to an existing node and adds nothing', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { name: /^Ran 4 tool calls/ }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows).toHaveCount(4) + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'List slots' })).toBeVisible() + + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // One graph_ops entry (two set_widget ops) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + + // The widget values on the unfiltered DOM: the aria-label sits on the + // number widget's container, so descend to its input. + const sampler = page.locator('[data-node-id="3"]') + await expect( + sampler.getByLabel('steps', { exact: true }).locator('input') + ).toHaveValue('30') + await expect( + sampler.getByLabel('cfg', { exact: true }).locator('input') + ).toHaveValue('5') + + // No node added or removed: the seed's five, wired as seeded. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } + ]) + }) + }) + + test.describe('recorded agent-rec-replace-prompt-encoder', () => { + test.use({ conversationCase: 'agent-rec-replace-prompt-encoder' }) + + test('deletes the recorded prompt node and rewires a fresh encoder in its place', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // The recording's closing `validate` errored on the host's missing + // checkpoint, so the group stays open: rows are visible with NO click. + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Delete node' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect( + panel.getByRole('button', { name: /^Ran 6 tool calls/ }) + ).toHaveAttribute('aria-expanded', 'true') + + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // Two graph_ops entries (the delete, then the add + wiring) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '3 updates' + ) + + // The deleted node is gone from the unfiltered DOM, not merely filtered + // out of the snapshot. + await expect(page.locator('[data-node-id="6"]')).toHaveCount(0) + await expect( + page.locator('[data-node-id="4073815636872683"]') + ).toBeVisible() + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '4073815636872683', + title: 'CLIPTextEncode', + inputs: [true], + outputs: [true] + } + ]) + }) + }) + + test.describe('recorded agent-rec-refiner-between', () => { + test.use({ conversationCase: 'agent-rec-refiner-between' }) + + test('adds a recorded refiner sampler between the first sampler and the decoder', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // The recording's closing `validate` errored on the host's missing + // checkpoint, so the group stays open: rows visible with NO click. + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect( + panel.getByRole('button', { name: /^Ran 5 tool calls/ }) + ).toHaveAttribute('aria-expanded', 'true') + + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // One graph_ops entry (add + four connects + three set_widget) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + await expect( + page.locator('[data-node-id="2629930598730104"]') + ).toBeVisible() + + // The refiner is wired on both sides: its latent_image input (index 3) + // takes the first sampler, and its own LATENT output feeds the decoder, + // whose samples input keeps its connection by last-writer-wins. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '2629930598730104', + title: 'KSampler', + inputs: [true, true, false, true], + outputs: [true] + } + ]) + }) + }) + + test.describe('recorded agent-rec-batched-ops', () => { + test.use({ conversationCase: 'agent-rec-batched-ops' }) + + test('applies two recorded adds and two connects as one batched entry', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { name: /^Ran 3 tool calls/ }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows).toHaveCount(3) + // One apply call, so its row carries no coalescing badge: two adjacent + // apply calls would render as a single "Apply ops" row with "x2". + const applyRow = toolRows.filter({ hasText: 'Apply ops' }) + await expect(applyRow).toHaveCount(1) + // The coalescing badge renders as a leading multiplication sign + // (ToolCallCard.vue renders it only when count > 1). + await expect(applyRow).not.toContainText('\u00d7') + + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // ONE graph_ops entry for all four ops, plus the catch-up: two unbatched + // apply calls would read "3 updates". + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + + await expect( + page.locator('[data-node-id="3495500264284537"]') + ).toBeVisible() + await expect( + page.locator('[data-node-id="4454558117922341"]') + ).toBeVisible() + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, false, false], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '3495500264284537', + title: 'CLIPTextEncode', + inputs: [true], + outputs: [false] + }, + { + id: '4454558117922341', + title: 'CLIPTextEncode', + inputs: [true], + outputs: [false] + } + ]) + }) + }) }) From 3c898f1f010e43ea2ae26372f558d34ba28c6634 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Wed, 2 Sep 2026 18:39:44 -0700 Subject: [PATCH 14/49] test(agent): name the replay regressions after the behavior The regression specs and fixtures were named after their fix PR numbers; they now carry the behavior they protect (catch-up baseline, subgraph mint scope) with the PR cited in the spec header and the fixture note. The README convention follows. --- browser_tests/README.md | 3 ++- ...6285-catchup-baseline.json => agent-catchup-baseline.json} | 2 +- ...611-subgraph-scope.json => agent-subgraph-mint-scope.json} | 2 +- ...85CatchupBaseline.spec.ts => agentCatchupBaseline.spec.ts} | 4 ++-- ...11SubgraphScope.spec.ts => agentSubgraphMintScope.spec.ts} | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) rename browser_tests/fixtures/data/agent/conversations/{agent-fix-16285-catchup-baseline.json => agent-catchup-baseline.json} (97%) rename browser_tests/fixtures/data/agent/conversations/{agent-fix-16611-subgraph-scope.json => agent-subgraph-mint-scope.json} (98%) rename browser_tests/tests/agent/{agentFix16285CatchupBaseline.spec.ts => agentCatchupBaseline.spec.ts} (94%) rename browser_tests/tests/agent/{agentFix16611SubgraphScope.spec.ts => agentSubgraphMintScope.spec.ts} (97%) diff --git a/browser_tests/README.md b/browser_tests/README.md index 737bcc0c21a0..756454f5ee4a 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -637,7 +637,8 @@ fix so the bug stays fixed: run lines in the PR description. A replay case that never went red against the bug does not count as regression coverage. -Name the case after the fix PR (`agent-fix--.json`) and note +Name the case after the behavior it protects (`agent-.json`, +with the fix PR cited in the spec header and the fixture's `source.note`) and note the fixture's source rows (thread id + seq range) in the JSON's metadata block so the recording can be re-exported. diff --git a/browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json b/browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json similarity index 97% rename from browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json rename to browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json index 70c540a3de9a..dd7274b8f676 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-fix-16285-catchup-baseline.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json @@ -3,7 +3,7 @@ "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", - "case_id": "agent-fix-16285-catchup-baseline", + "case_id": "agent-catchup-baseline", "response_side": "synthesized", "note": "Regression fixture for PR #16285 (apply CRDT catch-up at acknowledged sequence). The host sends doc_subscribed(seq=N) and then the catch-up doc_update(seq=N); pre-fix the bridge recorded N as the applied baseline and dropped the catch-up as stale, leaving a fresh follower empty. The conversation only binds; the seeded graph projecting at all IS the regression surface. Response synthesized: no model output is asserted." }, diff --git a/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json b/browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json similarity index 98% rename from browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json rename to browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json index 44238b06adca..0bc81fcca34d 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-fix-16611-subgraph-scope.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json @@ -3,7 +3,7 @@ "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", - "case_id": "agent-fix-16611-subgraph-scope", + "case_id": "agent-subgraph-mint-scope", "response_side": "synthesized", "note": "Regression fixture for PR #16611 (preserve root semantics for human graph edits). The conversation only binds the follower to a doc whose seed carries a subgraph definition and instance; the regression surface is the HUMAN edit the spec performs afterwards. Response synthesized: no model output is asserted." }, diff --git a/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts b/browser_tests/tests/agent/agentCatchupBaseline.spec.ts similarity index 94% rename from browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts rename to browser_tests/tests/agent/agentCatchupBaseline.spec.ts index 1b582c47d94f..e229eea354da 100644 --- a/browser_tests/tests/agent/agentFix16285CatchupBaseline.spec.ts +++ b/browser_tests/tests/agent/agentCatchupBaseline.spec.ts @@ -20,10 +20,10 @@ import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFi * together reproduce the pre-fix drop (status shows `0 updates`). */ test.describe( - 'Agent replay regression: #16285 catch-up baseline', + 'Agent replay regression: catch-up baseline (PR #16285)', { tag: '@cloud' }, () => { - test.use({ conversationCase: 'agent-fix-16285-catchup-baseline' }) + test.use({ conversationCase: 'agent-catchup-baseline' }) test('the subscription catch-up projects the seeded graph', async ({ agentConversation diff --git a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts similarity index 97% rename from browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts rename to browser_tests/tests/agent/agentSubgraphMintScope.spec.ts index 092254745826..839288b2a2d4 100644 --- a/browser_tests/tests/agent/agentFix16611SubgraphScope.spec.ts +++ b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts @@ -38,10 +38,10 @@ interface LayoutStoreModule { } test.describe( - 'Agent replay regression: #16611 subgraph scope', + 'Agent replay regression: subgraph mint scope (PR #16611)', { tag: '@cloud' }, () => { - test.use({ conversationCase: 'agent-fix-16611-subgraph-scope' }) + test.use({ conversationCase: 'agent-subgraph-mint-scope' }) test('a human delete inside a subgraph does not mint a root wire op', async ({ agentConversation, From 3d41e1cab9a63a2b6133fc608f1ab409ae4447a6 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 04:54:24 -0700 Subject: [PATCH 15/49] test(agent): wait for the applied thread id before replaying sendPrompt returned as soon as the browser's POST was captured, before the page had applied the acknowledgement from the mocked response. The replay frames are stamped with the fixture's thread id and the active-tab handler drops them until the session's thread id matches, so a slow response (seen under tracing) left the follower unsubscribed and the case timed out. The fixture now waits for the id the session persists right after applying it. --- .../fixtures/agentConversationFixture.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 603f3937b834..7788c5967b21 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -26,6 +26,8 @@ import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' const TURN_ID = '0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5f60' +/** Mirrors THREAD_STORAGE_KEY in useAgentSession.ts (module-private there). */ +const THREAD_STORAGE_KEY = 'Comfy.Agent.ThreadId' const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' const HOST_ACTOR = 'agent:comfy' const DOC_PROTOCOL_VERSION = 1 @@ -213,6 +215,19 @@ class AgentConversationHarness { await this.panel.getByRole('button', { name: SEND_LABEL }).click() await expect.poll(() => this.postedMessages.length).toBeGreaterThan(0) expect(this.postedMessages[0]).toContain(content) + // The page applies the acknowledgement (thread id, then the user turn) + // only once the mocked POST resolves, which can land after the request + // is captured above. Replay frames are stamped with THREAD_ID and the + // active-tab handler drops them until the thread id matches, so wait for + // the id the session persists right after applying it. + await expect + .poll(() => + this.page.evaluate( + (key) => localStorage.getItem(key), + THREAD_STORAGE_KEY + ) + ) + .toBe(THREAD_ID) } async replayResponse(): Promise { From 7b838a4bfd529e5e6dc3d6c264f593c9e1e87637 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 05:05:06 -0700 Subject: [PATCH 16/49] test(agent): record the two eval-suite cases on the local agent The two eval prompts (stitch the output beside the input; add a text node feeding the prompt and run) replayed from hand-written agent responses. They are now recorded on the local agent behind the tab switch the replay binds on and exported with the conversation exporter unchanged; the committed synthesized fixtures stay untouched. The second recording also added the sampler's missing negative encoder and empty latent, so its seed catalog carries EmptyLatentImage. --- .../agent-rec-workflow-editing-05.json | 412 ++++++++++++++ .../agent-rec-zimage-string-node-prompt.json | 526 ++++++++++++++++++ .../agent/agentConversationReplay.spec.ts | 172 ++++++ 3 files changed, 1110 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json new file mode 100644 index 000000000000..58e46d41c4ac --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json @@ -0,0 +1,412 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-workflow-editing-05", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread d22255f9-36b3-4ec4-a99e-1a14f92f9b87; message ed9a4ce1-9fcb-4393-8b8b-900f60103eba; workflow a1474551-fbbf-428b-9d89-a41ac0031d1e (seeded by throwaway turn 31737c38-71ba-4866-a9cf-947df44fb49c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['fdfd9665-8318-403d-82a7-a6764d3be23b', '8ac84dde-0912-4fac-b4d2-24ff247442e6', '2e0155b2-2eed-4e72-8718-f56fe9561c1c', '8463f6d0-03dd-4b90-8ed0-8d33bd59723d', 'edfd4073-5440-4296-b075-c98ef506bcc6', '8c177691-44ea-4534-a1d8-378c58cfe4fe', '00922b53-e98d-4645-81db-1b3eee576273']; rows agent-rec-workflow-editing-05.e5-a1.rows.json; raw capture sha256 596f4c002229ad778ec76d5972d6a285255300e4b816a80b31bfaa1fc80fa215", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "d22255f9-36b3-4ec4-a99e-1a14f92f9b87", + "message_id": "ed9a4ce1-9fcb-4393-8b8b-900f60103eba", + "exported_at": "2026-09-03T11:59:52Z" + } + }, + "workflow": { + "id": "a1474551-fbbf-428b-9d89-a41ac0031d1e", + "name": "Image edit", + "catalog": { + "types": { + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "VAEEncode": { + "widget_order": [] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "ImageStitch": { + "widget_order": [ + "direction", + "match_image_size", + "spacing_width", + "spacing_color" + ] + }, + "PreviewImage": { + "widget_order": [] + } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [1] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "VAEEncode", + "pos": [400, 0], + "size": [210, 46], + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 1 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2] + } + ], + "widgets_values": [] + }, + { + "id": 3, + "type": "KSampler", + "pos": [700, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [3] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1100, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 3 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [4] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1400, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 1, 0, 2, 0, "IMAGE"], + [2, 2, 0, 3, 3, "LATENT"], + [3, 3, 0, 8, 0, "LATENT"], + [4, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look at what's there." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Image edit", + "workflow_id": "a1474551-fbbf-428b-9d89-a41ac0031d1e" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01D5FjSPRaApDH2iwX9YriXZ", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 494, + "status": "success", + "tool_call_id": "toolu_01MWii7Rz882gmf1pTE5pVTh", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 191, + "status": "success", + "tool_call_id": "toolu_01SuNZEL9D1wQAhhY5h85Emm", + "tool_name": "find_nodes" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1710, 280], + "node": { + "id": 1380502390045812, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], + "type": "ImageStitch", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, + "widgets_values": ["right", true, 0, "white"] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "1ae5761c56814cd59b90d5b34915d10d", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1380502390045812, + "class_type": "ImageStitch", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c1fbc369e27348d994ff6cdceaf0ccb3", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4338446108286126, + "to_node": 1380502390045812, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "dff26a2ef5734d949b53391a7c61601b", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2548323397064582, + "to_node": 1380502390045812, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 209, + "status": "success", + "tool_call_id": "toolu_01HVWPwEhd4nMt5CJzpvF9kn", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01NDFSRkMRCkQ4KqhzHcH7EN", + "tool_name": "list_slots" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "2613854b97924cf394936f6425e1b0ff", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4049842834470829, + "to_node": 9, + "to_slot": 0, + "from_node": 1380502390045812, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 193, + "status": "success", + "tool_call_id": "toolu_01NEEaQs9cNn6qcJ1LgfZQCh", + "tool_name": "connect" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 197, + "status": "success", + "tool_call_id": "toolu_018ghX2WHT3XNERbndFewc2g", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — on the \"Image edit\" tab, the saved picture is now the original and the edited result side by side, original on the left, matched to the same height with no gap between them.\n\nOne thing to flag: this workflow was already incomplete before I touched it — it has no model or VAE loader and no prompt text, so the sampler, encoder and decoder have nothing feeding them. It won't run until those are added. Want me to fill that in?" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 3222, + "cache_read_input_tokens": 312979, + "input_tokens": 16, + "output_tokens": 1175, + "total_tokens": 1191 + } + } + } + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json new file mode 100644 index 000000000000..f36fece794b2 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json @@ -0,0 +1,526 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-zimage-string-node-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 98f33b92-6517-478e-a348-19db814e2685; message 5543eebd-40ed-408c-8574-f6a0e2d1e1c9; workflow 0150feeb-61a0-4dd2-b3c4-df8e743e31d4 (seeded by throwaway turn a9b183b7-9006-44ed-88ed-6c7f7cae1e7b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['e74f78e1-874e-4673-aaf5-0ef9b45ece1e', 'a3fac78f-e0a5-43af-b43d-979f4a0c0772', '6e9347a0-2025-4d03-9b95-ce9e68cd8727', '3f61fdd8-3a0c-4f04-8697-23ac2c398b34', '0520e93c-8a51-4ce5-ab5a-c80eb49d8d0b', 'd82a0aad-c81c-4776-938a-e0bf02056ff8', 'cc97e012-c484-40f9-a864-a97c9ba2e4bb', 'bcadb3c6-f385-4a1f-ae9f-b99ff9b4e0bd', 'f93556bb-a8bb-4ba2-9f72-d32056a80587']; rows agent-rec-zimage-string-node-prompt.zi-a2.rows.json; raw capture sha256 844318cc2db8e402e576d48e64336a4733455f053bb6eceea41d3a01349465e9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "98f33b92-6517-478e-a348-19db814e2685", + "message_id": "5543eebd-40ed-408c-8574-f6a0e2d1e1c9", + "exported_at": "2026-09-03T12:01:27Z" + } + }, + "workflow": { + "id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 11, + "status": "success", + "tool_call_id": "toolu_01EjSa5ZT7qaRRqCEBFwUYLY", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 594, + "status": "success", + "tool_call_id": "toolu_011b7ZgsTFUVYggGLKAPwttu", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 185, + "status": "success", + "tool_call_id": "toolu_013REBJ3bgPMbgKaXSLAmkAG", + "tool_name": "find_nodes" + } + } + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [395, 280], + "node": { + "id": 2346297545678642, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b2a640a31f1946119e4389cfbfa38610", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2346297545678642, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "731fc9d4faf84ae0840e0b3a45a03a48", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 2346297545678642, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "9e0615aa944d426db3e5932b7ed2de8f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1868944393408444, + "to_node": 6, + "to_slot": 1, + "from_node": 2346297545678642, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 1068057559864246, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "6521c0a17c584541bcd3539250cf877c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1068057559864246, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "68344eaa0b5c4590bfa572951a13458f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "", + "widget": "text", + "node_id": 1068057559864246, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5d0567473004a4e9cb96a8ef0c251e1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 769581307347504, + "to_node": 1068057559864246, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "37b1548fe5d943b2b2738353d96892f1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2229783680013788, + "to_node": 3, + "to_slot": 2, + "from_node": 1068057559864246, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 4476795995918869, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "833c3d83b4bb4c88a0e47d1448573f81", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4476795995918869, + "class_type": "EmptyLatentImage", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "23cb809c1f8c4cafb067880b01ee9a04", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1532394889807820, + "to_node": 3, + "to_slot": 3, + "from_node": 4476795995918869, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + } + ] + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 208, + "status": "success", + "tool_call_id": "toolu_01Bob4FLcTWSWKt9cetrSQcU", + "tool_name": "apply_ops" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_014CR2Q7xLRCmDXEaydcbf4s", + "tool_name": "validate" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 184, + "status": "success", + "tool_call_id": "toolu_01LR9BRaBnxH5hdBvg91GQiE", + "tool_name": "show_node" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 236, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01FgBW9iiC8boKQAu6mmYZFt", + "tool_name": "list_model_picks" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 667, + "status": "success", + "tool_call_id": "toolu_015pTnqS23ko4wetU427Ji3Z", + "tool_name": "list_generate_models" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 243, + "status": "success", + "tool_call_id": "toolu_01QwFYGv31NWSPfeZXcvd6Ge", + "tool_name": "print_workflow" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the text node is on your canvas and wired into the prompt, with \"a red bicycle on a pier at dusk\" in it. I also filled in the two inputs the sampler was missing (an empty negative prompt and a 512×512 blank canvas) so the graph is complete.\n\nI couldn't run it, though: this workflow uses the SDXL base checkpoint, and this server currently has **no checkpoint files installed at all** — so the run would fail immediately. That's an environment issue, not a wiring one.\n\nTwo ways forward, your call:\n- Install/download a checkpoint on the server, and I'll run this graph exactly as it is.\n- Or I can build the same prompt on one of the hosted image models available here (Flux, Ideogram, Stability, Recraft, and others) in a **new tab**, leaving this workflow untouched — that one I can run right now." + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 4471, + "cache_read_input_tokens": 395107, + "input_tokens": 20, + "output_tokens": 3071, + "total_tokens": 3091 + } + } + } + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 996ac2330316..967165788fd9 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -742,4 +742,176 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { ]) }) }) + + // The two eval-suite cases, re-recorded on the local agent (the committed + // synthesized fixtures above stay as they are). The recorded prompt is the + // eval prompt behind the tab switch the replay binds on. + test.describe('recorded agent-rec-workflow-editing-05', () => { + test.use({ conversationCase: 'agent-rec-workflow-editing-05' }) + + test('stitches the recorded output beside the input and routes it to the save', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const toolGroup = panel.getByRole('button', { name: /^Ran 7 tool calls/ }) + await expect(toolGroup).toBeVisible() + await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') + await toolGroup.click() + const toolRows = panel.getByRole('listitem') + await expect(toolRows).toHaveCount(7) + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'List slots' })).toBeVisible() + + await expect( + panel.getByRole('button', { name: 'Open Image edit' }) + ).toBeVisible() + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // Two graph_ops entries (apply_ops, then the follow-up connect) plus the + // catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '3 updates' + ) + await expect( + page.locator('[data-node-id="1380502390045812"]') + ).toBeVisible() + + // The stitch takes the source image and the decoded image; its output + // replaces the decoder's link into the save node (one link per input). + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { id: '1', title: 'LoadImage', inputs: [], outputs: [true, false] }, + { + id: '2', + title: 'VAEEncode', + inputs: [true, false], + outputs: [true] + }, + { + id: '3', + title: 'KSampler', + inputs: [false, false, false, true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, false], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + { + id: '1380502390045812', + title: 'ImageStitch', + inputs: [true, true], + outputs: [true] + } + ]) + }) + }) + + test.describe('recorded agent-rec-zimage-string-node-prompt', () => { + test.use({ conversationCase: 'agent-rec-zimage-string-node-prompt' }) + + test('adds the recorded text node and the inputs the sampler was missing', async ({ + agentConversation, + page + }) => { + test.setTimeout(45_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + // The recording's `validate` and `list_model_picks` calls errored (no + // checkpoint on the recording host), so the group stays open at turn end. + const toolRows = panel.getByRole('listitem') + await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() + await expect(toolRows.filter({ hasText: 'Validate' })).toBeVisible() + await expect( + panel.getByRole('button', { name: /^Ran 9 tool calls/ }) + ).toHaveAttribute('aria-expanded', 'true') + + await expect( + panel.getByRole('button', { name: 'Open Text to image' }) + ).toBeVisible() + await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + // One graph_ops entry (nine ops) plus the catch-up. + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + '2 updates' + ) + const stringNode = page.locator('[data-node-id="2346297545678642"]') + await expect(stringNode).toBeVisible() + await expect(stringNode.getByLabel('value', { exact: true })).toHaveValue( + 'a red bicycle on a pier at dusk' + ) + + // Every sampler input is now fed: the recorded turn also added a + // negative encoder and an empty latent. + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual([ + { + id: '3', + title: 'KSampler', + inputs: [true, true, true, true], + outputs: [true] + }, + { + id: '4', + title: 'CheckpointLoaderSimple', + inputs: [], + outputs: [true, true, true] + }, + { + id: '6', + title: 'Positive prompt', + inputs: [true], + outputs: [true] + }, + { + id: '8', + title: 'VAEDecode', + inputs: [true, true], + outputs: [true] + }, + { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, + // Nodes render in ascending id order. + { + id: '1068057559864246', + title: 'CLIPTextEncode', + inputs: [true], + outputs: [true] + }, + { + id: '2346297545678642', + title: 'PrimitiveStringMultiline', + inputs: [], + outputs: [true] + }, + { + id: '4476795995918869', + title: 'EmptyLatentImage', + inputs: [], + outputs: [true] + } + ]) + }) + }) }) From 9e39c6d6c06d2a74c99fbf2748bae71df1d7af23 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 05:11:41 -0700 Subject: [PATCH 17/49] test(agent): credit the snapshot sort, not the renderer, for pin order --- browser_tests/tests/agent/agentConversationReplay.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 967165788fd9..c40695647068 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -892,7 +892,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { outputs: [true] }, { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - // Nodes render in ascending id order. + // graphSnapshot sorts nodes by numeric id. { id: '1068057559864246', title: 'CLIPTextEncode', From 081552655f7e01b68f07ba50ef698f498286c699 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 05:22:56 -0700 Subject: [PATCH 18/49] test(agent): replace the synthesized eval fixtures with recordings The two eval-suite cases now carry the conversation the local agent actually produced for their prompts (exported with the conversation exporter unchanged) under the same case ids, and their replays assert that recording. The hand-written PoC responses and their describes are gone; no fixture is labelled synthesized any more except the two constructed regression seeds, which say so. --- .../agent-l4-zimage-string-node-prompt.json | 417 +++++++++++--- .../agent-rec-workflow-editing-05.json | 412 -------------- .../agent-rec-zimage-string-node-prompt.json | 526 ------------------ .../agent-workflow-editing-05.json | 319 ++++++++--- .../agent/agentConversationReplay.spec.ts | 146 +---- 5 files changed, 567 insertions(+), 1253 deletions(-) delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json index afad82463e76..3ace5c45f25a 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -1,19 +1,29 @@ { "schema_version": "agent-conversation.v1", "source": { - "repo": "Comfy-Org/evals", + "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", "case_id": "agent-l4-zimage-string-node-prompt", - "response_side": "synthesized", - "note": "Response synthesized from the case's expected outcome (string node added, text set, STRING wired into the prompt input). The eval seed is a Z-Image subgraph; this replay uses a flat text-to-image seed so the wiring lands on a plain CLIPTextEncode. The case's trailing run is not replayed." + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 98f33b92-6517-478e-a348-19db814e2685; message 5543eebd-40ed-408c-8574-f6a0e2d1e1c9; workflow 0150feeb-61a0-4dd2-b3c4-df8e743e31d4 (seeded by throwaway turn a9b183b7-9006-44ed-88ed-6c7f7cae1e7b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['e74f78e1-874e-4673-aaf5-0ef9b45ece1e', 'a3fac78f-e0a5-43af-b43d-979f4a0c0772', '6e9347a0-2025-4d03-9b95-ce9e68cd8727', '3f61fdd8-3a0c-4f04-8697-23ac2c398b34', '0520e93c-8a51-4ce5-ab5a-c80eb49d8d0b', 'd82a0aad-c81c-4776-938a-e0bf02056ff8', 'cc97e012-c484-40f9-a864-a97c9ba2e4bb', 'bcadb3c6-f385-4a1f-ae9f-b99ff9b4e0bd', 'f93556bb-a8bb-4ba2-9f72-d32056a80587']; rows agent-rec-zimage-string-node-prompt.zi-a2.rows.json; raw capture sha256 844318cc2db8e402e576d48e64336a4733455f053bb6eceea41d3a01349465e9 Replaces the synthesized PoC fixture of the same case id: same eval prompt, recorded on the local agent behind the tab switch the replay binds on.", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "98f33b92-6517-478e-a348-19db814e2685", + "message_id": "5543eebd-40ed-408c-8574-f6a0e2d1e1c9", + "exported_at": "2026-09-03T12:01:27Z" + } }, "workflow": { - "id": "6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01", + "id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4", "name": "Text to image", "catalog": { "types": { - "CheckpointLoaderSimple": { "widget_order": ["ckpt_name"] }, - "CLIPTextEncode": { "widget_order": ["text"] }, + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, "KSampler": { "widget_order": [ "seed", @@ -25,9 +35,18 @@ "denoise" ] }, - "VAEDecode": { "widget_order": [] }, - "SaveImage": { "widget_order": ["filename_prefix"] }, - "PrimitiveStringMultiline": { "widget_order": ["value"] } + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } } }, "seed": { @@ -39,9 +58,21 @@ "size": [315, 98], "inputs": [], "outputs": [ - { "name": "MODEL", "type": "MODEL", "links": [3] }, - { "name": "CLIP", "type": "CLIP", "links": [1] }, - { "name": "VAE", "type": "VAE", "links": [5] } + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } ], "widgets_values": ["sd_xl_base_1.0.safetensors"] }, @@ -52,16 +83,26 @@ "pos": [400, 0], "size": [400, 200], "inputs": [ - { "name": "clip", "type": "CLIP", "link": 1 }, + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, { "name": "text", "type": "STRING", - "widget": { "name": "text" }, + "widget": { + "name": "text" + }, "link": null } ], "outputs": [ - { "name": "CONDITIONING", "type": "CONDITIONING", "links": [2] } + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } ], "widgets_values": ["a photo of a pier"] }, @@ -71,12 +112,34 @@ "pos": [850, 0], "size": [315, 262], "inputs": [ - { "name": "model", "type": "MODEL", "link": 3 }, - { "name": "positive", "type": "CONDITIONING", "link": 2 }, - { "name": "negative", "type": "CONDITIONING", "link": null }, - { "name": "latent_image", "type": "LATENT", "link": null } + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } ], - "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [4] }], "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] }, { @@ -85,10 +148,24 @@ "pos": [1200, 0], "size": [210, 46], "inputs": [ - { "name": "samples", "type": "LATENT", "link": 4 }, - { "name": "vae", "type": "VAE", "link": 5 } + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } ], - "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [6] }], "widgets_values": [] }, { @@ -96,7 +173,13 @@ "type": "SaveImage", "pos": [1450, 0], "size": [315, 270], - "inputs": [{ "name": "images", "type": "IMAGE", "link": 6 }], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], "outputs": [], "widgets_values": ["ComfyUI"] } @@ -112,7 +195,7 @@ } }, "request": { - "content": "Add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." }, "response": [ { @@ -120,7 +203,7 @@ "event": { "type": "agent_thinking", "data": { - "delta": "I'll add a multiline string node, set its text, and wire its STRING output into the positive prompt's text input." + "delta": "I'll switch to that tab and take a look at what's there." } } }, @@ -129,8 +212,32 @@ "event": { "type": "agent_active_tab", "data": { - "workflow_id": "6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01", - "name": "Text to image" + "name": "Text to image", + "workflow_id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 11, + "status": "success", + "tool_call_id": "toolu_01EjSa5ZT7qaRRqCEBFwUYLY", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 594, + "status": "success", + "tool_call_id": "toolu_011b7ZgsTFUVYggGLKAPwttu", + "tool_name": "print_workflow" } } }, @@ -139,9 +246,10 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-add-string-node", - "tool_name": "add_node", - "status": "running" + "duration_ms": 185, + "status": "success", + "tool_call_id": "toolu_013REBJ3bgPMbgKaXSLAmkAG", + "tool_name": "find_nodes" } } }, @@ -150,19 +258,170 @@ "ops": [ { "op": "add_node", - "node_id": 10, - "class_type": "PrimitiveStringMultiline", - "pos": [0, 300], + "pos": [395, 280], "node": { - "id": 10, + "id": 2346297545678642, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], "type": "PrimitiveStringMultiline", - "title": "Prompt text", - "pos": [0, 300], - "size": [400, 200], + "flags": {}, + "order": 0, "inputs": [], - "outputs": [{ "name": "STRING", "type": "STRING", "links": [] }], - "widgets_values": [""] - } + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b2a640a31f1946119e4389cfbfa38610", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2346297545678642, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "731fc9d4faf84ae0840e0b3a45a03a48", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 2346297545678642, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "9e0615aa944d426db3e5932b7ed2de8f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1868944393408444, + "to_node": 6, + "to_slot": 1, + "from_node": 2346297545678642, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 1068057559864246, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "6521c0a17c584541bcd3539250cf877c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1068057559864246, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "68344eaa0b5c4590bfa572951a13458f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "", + "widget": "text", + "node_id": 1068057559864246, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5d0567473004a4e9cb96a8ef0c251e1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 769581307347504, + "to_node": 1068057559864246, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "37b1548fe5d943b2b2738353d96892f1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2229783680013788, + "to_node": 3, + "to_slot": 2, + "from_node": 1068057559864246, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 4476795995918869, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "833c3d83b4bb4c88a0e47d1448573f81", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4476795995918869, + "class_type": "EmptyLatentImage", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "23cb809c1f8c4cafb067880b01ee9a04", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1532394889807820, + "to_node": 3, + "to_slot": 3, + "from_node": 4476795995918869, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 } ] }, @@ -171,10 +430,10 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-add-string-node", - "tool_name": "add_node", + "duration_ms": 208, "status": "success", - "duration_ms": 400 + "tool_call_id": "toolu_01Bob4FLcTWSWKt9cetrSQcU", + "tool_name": "apply_ops" } } }, @@ -183,32 +442,22 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-set-prompt-text", - "tool_name": "set_widget", - "status": "running" + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_014CR2Q7xLRCmDXEaydcbf4s", + "tool_name": "validate" } } }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "set_widget", - "node_id": 10, - "widget": "value", - "value": "a red bicycle on a pier at dusk" - } - ] - }, { "kind": "event", "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-set-prompt-text", - "tool_name": "set_widget", + "duration_ms": 184, "status": "success", - "duration_ms": 200 + "tool_call_id": "toolu_01LR9BRaBnxH5hdBvg91GQiE", + "tool_name": "show_node" } } }, @@ -217,35 +466,35 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-connect-prompt", - "tool_name": "connect", - "status": "running" + "duration_ms": 236, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01FgBW9iiC8boKQAu6mmYZFt", + "tool_name": "list_model_picks" } } }, { - "kind": "graph_ops", - "ops": [ - { - "op": "connect", - "link_id": 7, - "from_node": 10, - "from_slot": 0, - "to_node": 6, - "to_slot": 1, - "link_type": "STRING" + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 667, + "status": "success", + "tool_call_id": "toolu_015pTnqS23ko4wetU427Ji3Z", + "tool_name": "list_generate_models" } - ] + } }, { "kind": "event", "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-connect-prompt", - "tool_name": "connect", + "duration_ms": 243, "status": "success", - "duration_ms": 300 + "tool_call_id": "toolu_01QwFYGv31NWSPfeZXcvd6Ge", + "tool_name": "print_workflow" } } }, @@ -254,7 +503,7 @@ "event": { "type": "agent_message_delta", "data": { - "delta": "Added a **Prompt text** node with *a red bicycle on a pier at dusk* and wired its STRING output into the positive prompt, so the prompt is now driven by that node. Say the word and I'll run it." + "delta": "Done \u2014 the text node is on your canvas and wired into the prompt, with \"a red bicycle on a pier at dusk\" in it. I also filled in the two inputs the sampler was missing (an empty negative prompt and a 512\u00d7512 blank canvas) so the graph is complete.\n\nI couldn't run it, though: this workflow uses the SDXL base checkpoint, and this server currently has **no checkpoint files installed at all** \u2014 so the run would fail immediately. That's an environment issue, not a wiring one.\n\nTwo ways forward, your call:\n- Install/download a checkpoint on the server, and I'll run this graph exactly as it is.\n- Or I can build the same prompt on one of the hosted image models available here (Flux, Ideogram, Stability, Recraft, and others) in a **new tab**, leaving this workflow untouched \u2014 that one I can run right now." } } }, @@ -264,11 +513,11 @@ "type": "agent_message_done", "data": { "usage": { - "input_tokens": 5120, - "output_tokens": 310, - "total_tokens": 5430, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0 + "cache_creation_input_tokens": 4471, + "cache_read_input_tokens": 395107, + "input_tokens": 20, + "output_tokens": 3071, + "total_tokens": 3091 } } } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json deleted file mode 100644 index 58e46d41c4ac..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-workflow-editing-05.json +++ /dev/null @@ -1,412 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-workflow-editing-05", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread d22255f9-36b3-4ec4-a99e-1a14f92f9b87; message ed9a4ce1-9fcb-4393-8b8b-900f60103eba; workflow a1474551-fbbf-428b-9d89-a41ac0031d1e (seeded by throwaway turn 31737c38-71ba-4866-a9cf-947df44fb49c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['fdfd9665-8318-403d-82a7-a6764d3be23b', '8ac84dde-0912-4fac-b4d2-24ff247442e6', '2e0155b2-2eed-4e72-8718-f56fe9561c1c', '8463f6d0-03dd-4b90-8ed0-8d33bd59723d', 'edfd4073-5440-4296-b075-c98ef506bcc6', '8c177691-44ea-4534-a1d8-378c58cfe4fe', '00922b53-e98d-4645-81db-1b3eee576273']; rows agent-rec-workflow-editing-05.e5-a1.rows.json; raw capture sha256 596f4c002229ad778ec76d5972d6a285255300e4b816a80b31bfaa1fc80fa215", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "d22255f9-36b3-4ec4-a99e-1a14f92f9b87", - "message_id": "ed9a4ce1-9fcb-4393-8b8b-900f60103eba", - "exported_at": "2026-09-03T11:59:52Z" - } - }, - "workflow": { - "id": "a1474551-fbbf-428b-9d89-a41ac0031d1e", - "name": "Image edit", - "catalog": { - "types": { - "LoadImage": { - "widget_order": ["image", "upload"] - }, - "VAEEncode": { - "widget_order": [] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "ImageStitch": { - "widget_order": [ - "direction", - "match_image_size", - "spacing_width", - "spacing_color" - ] - }, - "PreviewImage": { - "widget_order": [] - } - } - }, - "seed": { - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [0, 0], - "size": [315, 314], - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [1] - }, - { - "name": "MASK", - "type": "MASK", - "links": [] - } - ], - "widgets_values": ["example.png", "image"] - }, - { - "id": 2, - "type": "VAEEncode", - "pos": [400, 0], - "size": [210, 46], - "inputs": [ - { - "name": "pixels", - "type": "IMAGE", - "link": 1 - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [2] - } - ], - "widgets_values": [] - }, - { - "id": 3, - "type": "KSampler", - "pos": [700, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": null - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": null - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 2 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [3] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1100, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 3 - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [4] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1400, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 4 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 1, 0, 2, 0, "IMAGE"], - [2, 2, 0, 3, 3, "LATENT"], - [3, 3, 0, 8, 0, "LATENT"], - [4, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch over and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Image edit", - "workflow_id": "a1474551-fbbf-428b-9d89-a41ac0031d1e" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 6, - "status": "success", - "tool_call_id": "toolu_01D5FjSPRaApDH2iwX9YriXZ", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 494, - "status": "success", - "tool_call_id": "toolu_01MWii7Rz882gmf1pTE5pVTh", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 191, - "status": "success", - "tool_call_id": "toolu_01SuNZEL9D1wQAhhY5h85Emm", - "tool_name": "find_nodes" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [1710, 280], - "node": { - "id": 1380502390045812, - "pos": [1710, 280], - "mode": 0, - "size": [240, 178], - "type": "ImageStitch", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "image1", - "type": "IMAGE" - }, - { - "link": null, - "name": "image2", - "type": "IMAGE" - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [] - } - ], - "properties": {}, - "widgets_values": ["right", true, 0, "white"] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "1ae5761c56814cd59b90d5b34915d10d", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1380502390045812, - "class_type": "ImageStitch", - "base_version": 1 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c1fbc369e27348d994ff6cdceaf0ccb3", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4338446108286126, - "to_node": 1380502390045812, - "to_slot": 0, - "from_node": 1, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "dff26a2ef5734d949b53391a7c61601b", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2548323397064582, - "to_node": 1380502390045812, - "to_slot": 1, - "from_node": 8, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 209, - "status": "success", - "tool_call_id": "toolu_01HVWPwEhd4nMt5CJzpvF9kn", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 200, - "status": "success", - "tool_call_id": "toolu_01NDFSRkMRCkQ4KqhzHcH7EN", - "tool_name": "list_slots" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "2613854b97924cf394936f6425e1b0ff", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4049842834470829, - "to_node": 9, - "to_slot": 0, - "from_node": 1380502390045812, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 193, - "status": "success", - "tool_call_id": "toolu_01NEEaQs9cNn6qcJ1LgfZQCh", - "tool_name": "connect" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 197, - "status": "success", - "tool_call_id": "toolu_018ghX2WHT3XNERbndFewc2g", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — on the \"Image edit\" tab, the saved picture is now the original and the edited result side by side, original on the left, matched to the same height with no gap between them.\n\nOne thing to flag: this workflow was already incomplete before I touched it — it has no model or VAE loader and no prompt text, so the sampler, encoder and decoder have nothing feeding them. It won't run until those are added. Want me to fill that in?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 3222, - "cache_read_input_tokens": 312979, - "input_tokens": 16, - "output_tokens": 1175, - "total_tokens": 1191 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json deleted file mode 100644 index f36fece794b2..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-zimage-string-node-prompt.json +++ /dev/null @@ -1,526 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-zimage-string-node-prompt", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 98f33b92-6517-478e-a348-19db814e2685; message 5543eebd-40ed-408c-8574-f6a0e2d1e1c9; workflow 0150feeb-61a0-4dd2-b3c4-df8e743e31d4 (seeded by throwaway turn a9b183b7-9006-44ed-88ed-6c7f7cae1e7b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['e74f78e1-874e-4673-aaf5-0ef9b45ece1e', 'a3fac78f-e0a5-43af-b43d-979f4a0c0772', '6e9347a0-2025-4d03-9b95-ce9e68cd8727', '3f61fdd8-3a0c-4f04-8697-23ac2c398b34', '0520e93c-8a51-4ce5-ab5a-c80eb49d8d0b', 'd82a0aad-c81c-4776-938a-e0bf02056ff8', 'cc97e012-c484-40f9-a864-a97c9ba2e4bb', 'bcadb3c6-f385-4a1f-ae9f-b99ff9b4e0bd', 'f93556bb-a8bb-4ba2-9f72-d32056a80587']; rows agent-rec-zimage-string-node-prompt.zi-a2.rows.json; raw capture sha256 844318cc2db8e402e576d48e64336a4733455f053bb6eceea41d3a01349465e9", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "98f33b92-6517-478e-a348-19db814e2685", - "message_id": "5543eebd-40ed-408c-8574-f6a0e2d1e1c9", - "exported_at": "2026-09-03T12:01:27Z" - } - }, - "workflow": { - "id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - }, - "EmptyLatentImage": { - "widget_order": ["width", "height", "batch_size"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 11, - "status": "success", - "tool_call_id": "toolu_01EjSa5ZT7qaRRqCEBFwUYLY", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 594, - "status": "success", - "tool_call_id": "toolu_011b7ZgsTFUVYggGLKAPwttu", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 185, - "status": "success", - "tool_call_id": "toolu_013REBJ3bgPMbgKaXSLAmkAG", - "tool_name": "find_nodes" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [395, 280], - "node": { - "id": 2346297545678642, - "pos": [395, 280], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b2a640a31f1946119e4389cfbfa38610", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2346297545678642, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "731fc9d4faf84ae0840e0b3a45a03a48", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a red bicycle on a pier at dusk", - "widget": "value", - "node_id": 2346297545678642, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "9e0615aa944d426db3e5932b7ed2de8f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1868944393408444, - "to_node": 6, - "to_slot": 1, - "from_node": 2346297545678642, - "from_slot": 0, - "link_type": "STRING", - "base_version": 2 - }, - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 1068057559864246, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "6521c0a17c584541bcd3539250cf877c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1068057559864246, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "68344eaa0b5c4590bfa572951a13458f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "", - "widget": "text", - "node_id": 1068057559864246, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "a5d0567473004a4e9cb96a8ef0c251e1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 769581307347504, - "to_node": 1068057559864246, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "37b1548fe5d943b2b2738353d96892f1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2229783680013788, - "to_node": 3, - "to_slot": 2, - "from_node": 1068057559864246, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 - }, - { - "op": "add_node", - "pos": [395, 406], - "node": { - "id": 4476795995918869, - "pos": [395, 406], - "mode": 0, - "size": [240, 134], - "type": "EmptyLatentImage", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [512, 512, 1] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "833c3d83b4bb4c88a0e47d1448573f81", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4476795995918869, - "class_type": "EmptyLatentImage", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "23cb809c1f8c4cafb067880b01ee9a04", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1532394889807820, - "to_node": 3, - "to_slot": 3, - "from_node": 4476795995918869, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 208, - "status": "success", - "tool_call_id": "toolu_01Bob4FLcTWSWKt9cetrSQcU", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "error", - "tool_call_id": "toolu_014CR2Q7xLRCmDXEaydcbf4s", - "tool_name": "validate" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 184, - "status": "success", - "tool_call_id": "toolu_01LR9BRaBnxH5hdBvg91GQiE", - "tool_name": "show_node" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 236, - "error_code": "knowledge_unavailable", - "status": "error", - "tool_call_id": "toolu_01FgBW9iiC8boKQAu6mmYZFt", - "tool_name": "list_model_picks" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 667, - "status": "success", - "tool_call_id": "toolu_015pTnqS23ko4wetU427Ji3Z", - "tool_name": "list_generate_models" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 243, - "status": "success", - "tool_call_id": "toolu_01QwFYGv31NWSPfeZXcvd6Ge", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the text node is on your canvas and wired into the prompt, with \"a red bicycle on a pier at dusk\" in it. I also filled in the two inputs the sampler was missing (an empty negative prompt and a 512×512 blank canvas) so the graph is complete.\n\nI couldn't run it, though: this workflow uses the SDXL base checkpoint, and this server currently has **no checkpoint files installed at all** — so the run would fail immediately. That's an environment issue, not a wiring one.\n\nTwo ways forward, your call:\n- Install/download a checkpoint on the server, and I'll run this graph exactly as it is.\n- Or I can build the same prompt on one of the hosted image models available here (Flux, Ideogram, Stability, Recraft, and others) in a **new tab**, leaving this workflow untouched — that one I can run right now." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 4471, - "cache_read_input_tokens": 395107, - "input_tokens": 20, - "output_tokens": 3071, - "total_tokens": 3091 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json index 4cdeb940f97f..88d386931e79 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -1,19 +1,29 @@ { "schema_version": "agent-conversation.v1", "source": { - "repo": "Comfy-Org/evals", + "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", "case_id": "agent-workflow-editing-05", - "response_side": "synthesized", - "note": "Response synthesized from the case's expected outcome: an ImageStitch node shows the generated output next to the input image. The eval case ships no seed, so a small image-to-image seed stands in." + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread d22255f9-36b3-4ec4-a99e-1a14f92f9b87; message ed9a4ce1-9fcb-4393-8b8b-900f60103eba; workflow a1474551-fbbf-428b-9d89-a41ac0031d1e (seeded by throwaway turn 31737c38-71ba-4866-a9cf-947df44fb49c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['fdfd9665-8318-403d-82a7-a6764d3be23b', '8ac84dde-0912-4fac-b4d2-24ff247442e6', '2e0155b2-2eed-4e72-8718-f56fe9561c1c', '8463f6d0-03dd-4b90-8ed0-8d33bd59723d', 'edfd4073-5440-4296-b075-c98ef506bcc6', '8c177691-44ea-4534-a1d8-378c58cfe4fe', '00922b53-e98d-4645-81db-1b3eee576273']; rows agent-rec-workflow-editing-05.e5-a1.rows.json; raw capture sha256 596f4c002229ad778ec76d5972d6a285255300e4b816a80b31bfaa1fc80fa215 Replaces the synthesized PoC fixture of the same case id: same eval prompt, recorded on the local agent behind the tab switch the replay binds on.", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "d22255f9-36b3-4ec4-a99e-1a14f92f9b87", + "message_id": "ed9a4ce1-9fcb-4393-8b8b-900f60103eba", + "exported_at": "2026-09-03T11:59:52Z" + } }, "workflow": { - "id": "2b7d9a34-5c1e-4f0a-8e6b-7d2f3a9c1b45", + "id": "a1474551-fbbf-428b-9d89-a41ac0031d1e", "name": "Image edit", "catalog": { "types": { - "LoadImage": { "widget_order": ["image", "upload"] }, - "VAEEncode": { "widget_order": [] }, + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "VAEEncode": { + "widget_order": [] + }, "KSampler": { "widget_order": [ "seed", @@ -25,8 +35,12 @@ "denoise" ] }, - "VAEDecode": { "widget_order": [] }, - "SaveImage": { "widget_order": ["filename_prefix"] }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, "ImageStitch": { "widget_order": [ "direction", @@ -35,7 +49,9 @@ "spacing_color" ] }, - "PreviewImage": { "widget_order": [] } + "PreviewImage": { + "widget_order": [] + } } }, "seed": { @@ -47,8 +63,16 @@ "size": [315, 314], "inputs": [], "outputs": [ - { "name": "IMAGE", "type": "IMAGE", "links": [1] }, - { "name": "MASK", "type": "MASK", "links": [] } + { + "name": "IMAGE", + "type": "IMAGE", + "links": [1] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } ], "widgets_values": ["example.png", "image"] }, @@ -58,10 +82,24 @@ "pos": [400, 0], "size": [210, 46], "inputs": [ - { "name": "pixels", "type": "IMAGE", "link": 1 }, - { "name": "vae", "type": "VAE", "link": null } + { + "name": "pixels", + "type": "IMAGE", + "link": 1 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2] + } ], - "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [2] }], "widgets_values": [] }, { @@ -70,12 +108,34 @@ "pos": [700, 0], "size": [315, 262], "inputs": [ - { "name": "model", "type": "MODEL", "link": null }, - { "name": "positive", "type": "CONDITIONING", "link": null }, - { "name": "negative", "type": "CONDITIONING", "link": null }, - { "name": "latent_image", "type": "LATENT", "link": 2 } + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [3] + } ], - "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [3] }], "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] }, { @@ -84,10 +144,24 @@ "pos": [1100, 0], "size": [210, 46], "inputs": [ - { "name": "samples", "type": "LATENT", "link": 3 }, - { "name": "vae", "type": "VAE", "link": null } + { + "name": "samples", + "type": "LATENT", + "link": 3 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [4] + } ], - "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [4] }], "widgets_values": [] }, { @@ -95,7 +169,13 @@ "type": "SaveImage", "pos": [1400, 0], "size": [315, 270], - "inputs": [{ "name": "images", "type": "IMAGE", "link": 4 }], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], "outputs": [], "widgets_values": ["ComfyUI"] } @@ -109,7 +189,7 @@ } }, "request": { - "content": "now use a stitch images node to show the output next to the input" + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" }, "response": [ { @@ -117,7 +197,7 @@ "event": { "type": "agent_thinking", "data": { - "delta": "I'll stitch the loaded input and the decoded output side by side and preview the result." + "delta": "I'll switch over and take a look at what's there." } } }, @@ -126,8 +206,32 @@ "event": { "type": "agent_active_tab", "data": { - "workflow_id": "2b7d9a34-5c1e-4f0a-8e6b-7d2f3a9c1b45", - "name": "Image edit" + "name": "Image edit", + "workflow_id": "a1474551-fbbf-428b-9d89-a41ac0031d1e" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01D5FjSPRaApDH2iwX9YriXZ", + "tool_name": "switch_tab" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 494, + "status": "success", + "tool_call_id": "toolu_01MWii7Rz882gmf1pTE5pVTh", + "tool_name": "print_workflow" } } }, @@ -136,9 +240,10 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-add-stitch", - "tool_name": "add_node", - "status": "running" + "duration_ms": 191, + "status": "success", + "tool_call_id": "toolu_01SuNZEL9D1wQAhhY5h85Emm", + "tool_name": "find_nodes" } } }, @@ -147,38 +252,69 @@ "ops": [ { "op": "add_node", - "node_id": 12, - "class_type": "ImageStitch", - "pos": [1400, 350], + "pos": [1710, 280], "node": { - "id": 12, + "id": 1380502390045812, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], "type": "ImageStitch", - "title": "Input vs output", - "pos": [1400, 350], - "size": [315, 150], + "flags": {}, + "order": 0, "inputs": [ - { "name": "image1", "type": "IMAGE", "link": null }, - { "name": "image2", "type": "IMAGE", "link": null } + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } ], - "outputs": [{ "name": "IMAGE", "type": "IMAGE", "links": [] }], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, "widgets_values": ["right", true, 0, "white"] - } + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "1ae5761c56814cd59b90d5b34915d10d", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1380502390045812, + "class_type": "ImageStitch", + "base_version": 1 }, { - "op": "add_node", - "node_id": 13, - "class_type": "PreviewImage", - "pos": [1800, 350], - "node": { - "id": 13, - "type": "PreviewImage", - "title": "Side by side", - "pos": [1800, 350], - "size": [315, 270], - "inputs": [{ "name": "images", "type": "IMAGE", "link": null }], - "outputs": [], - "widgets_values": [] - } + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c1fbc369e27348d994ff6cdceaf0ccb3", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4338446108286126, + "to_node": 1380502390045812, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "dff26a2ef5734d949b53391a7c61601b", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2548323397064582, + "to_node": 1380502390045812, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 } ] }, @@ -187,10 +323,10 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-add-stitch", - "tool_name": "add_node", + "duration_ms": 209, "status": "success", - "duration_ms": 500 + "tool_call_id": "toolu_01HVWPwEhd4nMt5CJzpvF9kn", + "tool_name": "apply_ops" } } }, @@ -199,9 +335,10 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-wire-stitch", - "tool_name": "connect", - "status": "running" + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01NDFSRkMRCkQ4KqhzHcH7EN", + "tool_name": "list_slots" } } }, @@ -210,30 +347,16 @@ "ops": [ { "op": "connect", - "link_id": 5, - "from_node": 1, - "from_slot": 0, - "to_node": 12, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "2613854b97924cf394936f6425e1b0ff", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4049842834470829, + "to_node": 9, "to_slot": 0, - "link_type": "IMAGE" - }, - { - "op": "connect", - "link_id": 6, - "from_node": 8, + "from_node": 1380502390045812, "from_slot": 0, - "to_node": 12, - "to_slot": 1, - "link_type": "IMAGE" - }, - { - "op": "connect", - "link_id": 7, - "from_node": 12, - "from_slot": 0, - "to_node": 13, - "to_slot": 0, - "link_type": "IMAGE" + "link_type": "IMAGE", + "base_version": 2 } ] }, @@ -242,10 +365,22 @@ "event": { "type": "agent_tool_call", "data": { - "tool_call_id": "call-wire-stitch", - "tool_name": "connect", + "duration_ms": 193, + "status": "success", + "tool_call_id": "toolu_01NEEaQs9cNn6qcJ1LgfZQCh", + "tool_name": "connect" + } + } + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 197, "status": "success", - "duration_ms": 700 + "tool_call_id": "toolu_018ghX2WHT3XNERbndFewc2g", + "tool_name": "print_workflow" } } }, @@ -254,7 +389,7 @@ "event": { "type": "agent_message_delta", "data": { - "delta": "Done. An **Image Stitch** node now takes the loaded input on the left and the decoded output on the right, and a preview node shows the pair. Your save node is untouched." + "delta": "Done \u2014 on the \"Image edit\" tab, the saved picture is now the original and the edited result side by side, original on the left, matched to the same height with no gap between them.\n\nOne thing to flag: this workflow was already incomplete before I touched it \u2014 it has no model or VAE loader and no prompt text, so the sampler, encoder and decoder have nothing feeding them. It won't run until those are added. Want me to fill that in?" } } }, @@ -264,11 +399,11 @@ "type": "agent_message_done", "data": { "usage": { - "input_tokens": 6010, - "output_tokens": 280, - "total_tokens": 6290, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0 + "cache_creation_input_tokens": 3222, + "cache_read_input_tokens": 312979, + "input_tokens": 16, + "output_tokens": 1175, + "total_tokens": 1191 } } } diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index c40695647068..52acce70605e 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -3,138 +3,6 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' test.describe('Agent conversation replay', { tag: '@cloud' }, () => { - test.describe('evals agent-l4-zimage-string-node-prompt', () => { - test.use({ conversationCase: 'agent-l4-zimage-string-node-prompt' }) - - test('adds a string node, sets its text, and wires it into the prompt', async ({ - agentConversation - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { - name: 'Ran 3 tool calls for 0.9 seconds' - }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows).toHaveCount(3) - await expect(toolRows.filter({ hasText: 'Add node' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Set widget' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Connect' })).toBeVisible() - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - await expect( - panel.locator('strong', { hasText: 'Prompt text' }) - ).toBeVisible() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '4 updates' - ) - - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - // The wired `text` input is widget-backed and renders as a widget, - // not a slot row; the link shows on the string node's output. - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { id: '10', title: 'Prompt text', inputs: [], outputs: [true] } - ]) - }) - }) - - test.describe('evals agent-workflow-editing-05', () => { - test.use({ conversationCase: 'agent-workflow-editing-05' }) - - test('stitches the input next to the output and previews the pair', async ({ - agentConversation - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { - name: 'Ran 2 tool calls for 1.2 seconds' - }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await expect( - panel.locator('strong', { hasText: 'Image Stitch' }) - ).toBeVisible() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '3 updates' - ) - - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { id: '1', title: 'LoadImage', inputs: [], outputs: [true, false] }, - { - id: '2', - title: 'VAEEncode', - inputs: [true, false], - outputs: [true] - }, - { - id: '3', - title: 'KSampler', - inputs: [false, false, false, true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, false], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '12', - title: 'Input vs output', - inputs: [true, true], - outputs: [true] - }, - { id: '13', title: 'Side by side', inputs: [true], outputs: [] } - ]) - }) - }) - test.describe('recorded agent-rec-text-only-answer', () => { test.use({ conversationCase: 'agent-rec-text-only-answer' }) @@ -743,11 +611,11 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { }) }) - // The two eval-suite cases, re-recorded on the local agent (the committed - // synthesized fixtures above stay as they are). The recorded prompt is the - // eval prompt behind the tab switch the replay binds on. - test.describe('recorded agent-rec-workflow-editing-05', () => { - test.use({ conversationCase: 'agent-rec-workflow-editing-05' }) + // The two eval-suite cases, recorded on the local agent (they replace the + // synthesized PoC fixtures). The recorded prompt is the eval prompt behind + // the tab switch the replay binds on. + test.describe('recorded agent-workflow-editing-05', () => { + test.use({ conversationCase: 'agent-workflow-editing-05' }) test('stitches the recorded output beside the input and routes it to the save', async ({ agentConversation, @@ -821,8 +689,8 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { }) }) - test.describe('recorded agent-rec-zimage-string-node-prompt', () => { - test.use({ conversationCase: 'agent-rec-zimage-string-node-prompt' }) + test.describe('recorded agent-l4-zimage-string-node-prompt', () => { + test.use({ conversationCase: 'agent-l4-zimage-string-node-prompt' }) test('adds the recorded text node and the inputs the sampler was missing', async ({ agentConversation, From a8158b8fa3ce13efbc79c30fdc2468d421d8a11f Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:01:19 -0700 Subject: [PATCH 19/49] test(agent): derive replay expectations from the fixture and replay recorded timing Every recorded conversation now runs through one replay that reads its expectations from the fixture: tool groups (split where thinking or text interrupts the calls), the update count, the added nodes, the widget values set, and the graph snapshot computed from the seed plus the ops. Re-recording a case no longer needs hand-pinned values. Recorded frames carry at_ms; the exporter emits each entry's offset from the turn's first frame and the fixture can replay those gaps (replayTiming, or AGENT_REPLAY_TIMING=recorded). Comments cut to one-line whys. --- .../fixtures/agentConversationFixture.ts | 38 +- .../fixtures/data/agent/agentConversation.ts | 25 +- .../agent/agentConversationExpectations.ts | 273 ++++++ .../tests/agent/agentCatchupBaseline.spec.ts | 20 +- .../agent/agentConversationReplay.spec.ts | 864 ++---------------- .../agent/agentSubgraphMintScope.spec.ts | 46 +- scripts/agentConversationCapture.test.ts | 24 + scripts/agentConversationCapture.ts | 16 +- 8 files changed, 452 insertions(+), 854 deletions(-) create mode 100644 browser_tests/fixtures/data/agent/agentConversationExpectations.ts diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 7788c5967b21..4c8e453ec540 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -26,7 +26,7 @@ import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' const TURN_ID = '0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5f60' -/** Mirrors THREAD_STORAGE_KEY in useAgentSession.ts (module-private there). */ +// useAgentSession.ts keeps THREAD_STORAGE_KEY module-private. const THREAD_STORAGE_KEY = 'Comfy.Agent.ThreadId' const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' const HOST_ACTOR = 'agent:comfy' @@ -165,7 +165,8 @@ class AgentConversationHarness { constructor( private readonly page: Page, - readonly conversation: AgentConversation + readonly conversation: AgentConversation, + readonly replayTiming: ReplayTiming ) { const { workflow } = conversation this.host = new HostDoc(workflow.id, workflow.seed, workflow.catalog) @@ -215,11 +216,7 @@ class AgentConversationHarness { await this.panel.getByRole('button', { name: SEND_LABEL }).click() await expect.poll(() => this.postedMessages.length).toBeGreaterThan(0) expect(this.postedMessages[0]).toContain(content) - // The page applies the acknowledgement (thread id, then the user turn) - // only once the mocked POST resolves, which can land after the request - // is captured above. Replay frames are stamped with THREAD_ID and the - // active-tab handler drops them until the thread id matches, so wait for - // the id the session persists right after applying it. + // Replay frames are dropped until the page has applied the ack's thread id. await expect .poll(() => this.page.evaluate( @@ -231,7 +228,12 @@ class AgentConversationHarness { } async replayResponse(): Promise { + const startedAt = Date.now() for (const entry of this.conversation.response) { + if (this.replayTiming === 'recorded' && entry.at_ms !== undefined) { + const wait = entry.at_ms - (Date.now() - startedAt) + if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)) + } if (entry.kind === 'event') { this.send(this.stampTurn(entry.event)) continue @@ -239,8 +241,11 @@ class AgentConversationHarness { await this.waitForSubscribe() this.send(this.host.apply(entry.ops)) } + this.replayElapsedMs = Date.now() - startedAt } + replayElapsedMs = 0 + async waitForTurnComplete(): Promise { await expect( this.panel.getByRole('button', { name: SEND_LABEL }) @@ -411,23 +416,38 @@ class AgentConversationHarness { } } +export type ReplayTiming = 'immediate' | 'recorded' + +function defaultReplayTiming(): ReplayTiming { + const value = process.env.AGENT_REPLAY_TIMING + if (value === undefined || value === 'immediate') return 'immediate' + if (value === 'recorded') return 'recorded' + throw new Error( + `AGENT_REPLAY_TIMING must be immediate or recorded, got ${value}` + ) +} + interface ConversationFixtures { /** Case id of the conversation under `fixtures/data/agent/conversations`. */ conversationCase: string + // 'recorded' replays the fixture's at_ms gaps; the default follows AGENT_REPLAY_TIMING. + replayTiming: ReplayTiming agentConversation: AgentConversationHarness } export const agentConversationTest = agentTest.extend({ conversationCase: ['', { option: true }], + replayTiming: [defaultReplayTiming(), { option: true }], agentConversation: async ( - { page, agentFlagEnabled, conversationCase }, + { page, agentFlagEnabled, conversationCase, replayTiming }, use ) => { if (conversationCase.length === 0) throw new Error('test.use({ conversationCase }) names the conversation') const harness = new AgentConversationHarness( page, - loadAgentConversation(conversationCase) + loadAgentConversation(conversationCase), + replayTiming ) await harness.boot(agentFlagEnabled) await use(harness) diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 8e6a8e2a5787..2cfc991a884f 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -1,4 +1,4 @@ -import { readFileSync } from 'node:fs' +import { readFileSync, readdirSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' @@ -18,7 +18,8 @@ function isRecord(value: unknown): value is Record { */ export const zRecordedWsEvent = z.object({ type: z.string(), - data: z.record(z.string(), z.unknown()) + data: z.record(z.string(), z.unknown()), + at_ms: z.number().int().nonnegative().optional() }) export type RecordedWsEvent = z.infer @@ -51,11 +52,15 @@ export const zAgentConversationRequest = z.object({ content: z.string().min(1) }) +// Offset from the turn's first frame, so replays can reproduce real gaps. +const zAtMs = z.number().int().nonnegative().optional() + const zResponseEntry = z.discriminatedUnion('kind', [ - z.object({ kind: z.literal('event'), event: zRecordedWsEvent }), + z.object({ kind: z.literal('event'), event: zRecordedWsEvent, at_ms: zAtMs }), z.object({ kind: z.literal('graph_ops'), - ops: z.array(zGraphOperation).min(1) + ops: z.array(zGraphOperation).min(1), + at_ms: zAtMs }) ]) @@ -100,6 +105,18 @@ export const zAgentConversation = z }) export type AgentConversation = z.infer +export function listRecordedConversations(): string[] { + const dir = fileURLToPath(new URL('./conversations/', import.meta.url)) + return readdirSync(dir) + .filter((file) => file.endsWith('.json')) + .map((file) => file.slice(0, -'.json'.length)) + .filter( + (caseId) => + loadAgentConversation(caseId).source.response_side === 'recorded' + ) + .sort() +} + export function loadAgentConversation(caseId: string): AgentConversation { const file = fileURLToPath( new URL(`./conversations/${caseId}.json`, import.meta.url) diff --git a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts new file mode 100644 index 000000000000..727f47b43304 --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts @@ -0,0 +1,273 @@ +import type { AgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +// Expectations come from the fixture itself, so re-recording a case never needs hand-pinned values. + +export interface ExpectedGraphNode { + id: string + title: string + inputs: boolean[] + outputs: boolean[] +} + +interface SeedNode { + id: number | string + type: string + title?: string + inputs?: Array<{ name: string }> + outputs?: Array<{ name: string }> +} + +interface Link { + from: string + fromSlot: number + to: string + toSlot: number +} + +interface NodeState { + type: string + title: string + inputNames: string[] + outputNames: string[] +} + +function asRecord(value: unknown): Record { + return typeof value === 'object' && value !== null + ? (value as Record) + : {} +} + +function slotNames(node: SeedNode): { + inputNames: string[] + outputNames: string[] +} { + return { + inputNames: (node.inputs ?? []).map((slot) => slot.name), + outputNames: (node.outputs ?? []).map((slot) => slot.name) + } +} + +export function expectedGraphSnapshot( + conversation: AgentConversation +): ExpectedGraphNode[] { + const seed = conversation.workflow.seed as unknown as { + nodes: SeedNode[] + links: Array<[number, number, number, number, number, string]> + } + const catalog = conversation.workflow.catalog.types as Record< + string, + { widget_order: string[] } + > + const nodes = new Map() + const byType = new Map< + string, + { inputNames: string[]; outputNames: string[] } + >() + for (const node of seed.nodes) { + const names = slotNames(node) + nodes.set(String(node.id), { + type: node.type, + title: node.title ?? node.type, + ...names + }) + byType.set(node.type, names) + } + const links = new Map() + for (const [id, from, fromSlot, to, toSlot] of seed.links) { + links.set(String(id), { + from: String(from), + fromSlot, + to: String(to), + toSlot + }) + } + const setLink = (id: string, link: Link) => { + for (const [existingId, existing] of links) { + if (existing.to === link.to && existing.toSlot === link.toSlot) + links.delete(existingId) + } + links.set(id, link) + } + + for (const entry of conversation.response) { + if (entry.kind !== 'graph_ops') continue + for (const rawOp of entry.ops) { + const op = asRecord(rawOp) + switch (op.op) { + case 'add_node': { + const node = asRecord(op.node) as unknown as SeedNode + const own = slotNames(node) + const fallback = byType.get(node.type) + nodes.set(String(node.id), { + type: node.type, + title: node.title ?? node.type, + inputNames: own.inputNames.length + ? own.inputNames + : (fallback?.inputNames ?? []), + outputNames: own.outputNames.length + ? own.outputNames + : (fallback?.outputNames ?? []) + }) + break + } + case 'delete_node': { + const id = String(op.node_id) + nodes.delete(id) + for (const [linkId, link] of links) { + if (link.from === id || link.to === id) links.delete(linkId) + } + break + } + case 'connect': + setLink(String(op.link_id), { + from: String(op.from_node), + fromSlot: Number(op.from_slot), + to: String(op.to_node), + toSlot: Number(op.to_slot) + }) + break + case 'set_widget': + break + default: + throw new Error( + `no expectation rule for recorded op ${String(op.op)}` + ) + } + } + } + + const connected = [...links.values()] + return [...nodes.entries()] + .map(([id, node]) => { + const widgets = new Set(catalog[node.type]?.widget_order ?? []) + return { + id, + title: node.title, + // Widget-backed inputs render as widgets, not slot rows. + inputs: node.inputNames + .map((name, slot) => ({ name, slot })) + .filter(({ name }) => !widgets.has(name)) + .map(({ slot }) => + connected.some((link) => link.to === id && link.toSlot === slot) + ), + outputs: node.outputNames.map((_, slot) => + connected.some((link) => link.from === id && link.fromSlot === slot) + ) + } + }) + .sort((a, b) => Number(a.id) - Number(b.id)) +} + +export interface RecordedToolCall { + name: string + ok: boolean +} + +// The panel starts a new tool group whenever thinking or text interrupts the calls. +export function recordedToolCallGroups( + conversation: AgentConversation +): RecordedToolCall[][] { + const groups: RecordedToolCall[][] = [] + let current: RecordedToolCall[] = [] + for (const entry of conversation.response) { + if (entry.kind !== 'event') continue + const { type, data } = entry.event + if (type === 'agent_tool_call') { + if (data.status !== 'running') + current.push({ + name: String(data.tool_name), + ok: data.status === 'success' + }) + continue + } + if ( + (type === 'agent_thinking' || type === 'agent_message_delta') && + current.length > 0 + ) { + groups.push(current) + current = [] + } + } + if (current.length > 0) groups.push(current) + return groups +} + +const KNOWN_TOOL_LABELS: Record = { + new_tab: 'Opened a new tab', + switch_tab: 'Switched tabs', + remember: 'Saved a preference', + forget: 'Forgot a preference' +} + +// Mirrors the label rule in ToolCallCard.vue. +export function toolRowLabel(name: string): string { + const known = Object.hasOwn(KNOWN_TOOL_LABELS, name) + ? KNOWN_TOOL_LABELS[name] + : undefined + if (known) return known + const spaced = name.replaceAll('_', ' ') + return spaced.charAt(0).toUpperCase() + spaced.slice(1) +} + +export interface RecordedWidgetValue { + nodeId: string + widget: string + value: string | number +} + +export function recordedWidgetValues( + conversation: AgentConversation +): RecordedWidgetValue[] { + const surviving = new Set( + expectedGraphSnapshot(conversation).map((node) => node.id) + ) + return conversation.response.flatMap((entry) => { + if (entry.kind !== 'graph_ops') return [] + return entry.ops.flatMap((rawOp) => { + const op = asRecord(rawOp) + if (op.op !== 'set_widget') return [] + const nodeId = String(op.node_id) + const value = op.value + if ( + !surviving.has(nodeId) || + (typeof value !== 'string' && typeof value !== 'number') + ) + return [] + return [{ nodeId, widget: String(op.widget), value }] + }) + }) +} + +export function recordedAddedNodeIds( + conversation: AgentConversation +): string[] { + const surviving = new Set( + expectedGraphSnapshot(conversation).map((node) => node.id) + ) + return conversation.response.flatMap((entry) => { + if (entry.kind !== 'graph_ops') return [] + return entry.ops.flatMap((rawOp) => { + const op = asRecord(rawOp) + if (op.op !== 'add_node') return [] + const id = String(asRecord(op.node).id) + return surviving.has(id) ? [id] : [] + }) + }) +} + +// One update per ops entry plus the subscribe catch-up. +export function expectedUpdateCount(conversation: AgentConversation): number { + return ( + conversation.response.filter((entry) => entry.kind === 'graph_ops').length + + 1 + ) +} + +export function recordedSpanMs( + conversation: AgentConversation +): number | undefined { + const offsets = conversation.response.flatMap((entry) => + entry.at_ms === undefined ? [] : [entry.at_ms] + ) + return offsets.length === 0 ? undefined : Math.max(...offsets) +} diff --git a/browser_tests/tests/agent/agentCatchupBaseline.spec.ts b/browser_tests/tests/agent/agentCatchupBaseline.spec.ts index e229eea354da..10a121d22553 100644 --- a/browser_tests/tests/agent/agentCatchupBaseline.spec.ts +++ b/browser_tests/tests/agent/agentCatchupBaseline.spec.ts @@ -2,23 +2,7 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -/** - * Replay regression for PR #16285 (apply CRDT catch-up at acknowledged - * sequence). The host sends `doc_subscribed(seq=N)` and THEN the catch-up - * `doc_update(seq=N)` - the conversation fixture reproduces exactly that - * ordering with the same seq on both frames. Pre-fix, the bridge recorded N - * from the acknowledgement as the applied baseline and dropped the catch-up - * as stale, so a fresh follower rendered an EMPTY graph. The seeded graph - * projecting at all is the regression surface. - * - * Red recipe (recorded two-line patch, no revertible commit on this base - - * the base carries an evolved form of the fix with two independent - * defenses): in layoutFollowerBridge, (1) restore the pre-fix baseline - * write `this.lastSeq = subscribed.seq ?? null` on onDocSubscribed's ok - * path AND (2) disable the catch-up carve-out in onDocUpdate - * (`const isCatchUp = false`). Either line alone stays green; both - * together reproduce the pre-fix drop (status shows `0 updates`). - */ +// Regression for #16285: pre-fix the follower dropped the catch-up sent at the ack's seq and rendered empty. test.describe( 'Agent replay regression: catch-up baseline (PR #16285)', { tag: '@cloud' }, @@ -38,8 +22,6 @@ test.describe( await expect(panel.getByTestId('agent-crdt-status')).toContainText( 'connected' ) - // The catch-up is the one applied update; pre-fix it is dropped as - // stale and the count stays at 0. await expect(panel.getByTestId('agent-crdt-status')).toContainText( '1 updates' ) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 52acce70605e..6b87106c5598 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -1,785 +1,95 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { + listRecordedConversations, + loadAgentConversation +} from '@e2e/fixtures/data/agent/agentConversation' +import { + expectedGraphSnapshot, + expectedUpdateCount, + recordedAddedNodeIds, + recordedToolCallGroups, + recordedWidgetValues, + toolRowLabel +} from '@e2e/fixtures/data/agent/agentConversationExpectations' test.describe('Agent conversation replay', { tag: '@cloud' }, () => { - test.describe('recorded agent-rec-text-only-answer', () => { - test.use({ conversationCase: 'agent-rec-text-only-answer' }) - - test('switches to the seeded tab and answers in text without touching the graph', async ({ - agentConversation - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // A passing group collapses at turn end; the header count is the only - // stable part of its name (the duration is the recording's). - const toolGroup = panel.getByRole('button', { name: /^Ran 2 tool calls/ }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Print workflow' })).toBeVisible() - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - // An assistant text part rendered; the model prose itself is never pinned. - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // Zero graph_ops entries: the catch-up is the one applied update. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '1 updates' - ) - - // The seed is untouched: five nodes with the seed's own wiring. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } - ]) - }) - }) - - test.describe('recorded agent-rec-tool-error', () => { - test.use({ conversationCase: 'agent-rec-tool-error' }) - - test('keeps a failed add_node group open and leaves the graph untouched', async ({ - agentConversation - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // The failed group stays open at turn end: its rows are visible with NO - // expand click, and aria-expanded is read off the group's trigger button. - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Add node' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect( - panel.getByRole('button', { name: /^Ran 2 tool calls/ }) - ).toHaveAttribute('aria-expanded', 'true') - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // Zero graph_ops entries: the catch-up is the one applied update. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '1 updates' - ) - - // No new node: the seed's five nodes with the seed's own wiring. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } - ]) - }) - }) - - test.describe('recorded agent-rec-string-node-into-prompt', () => { - test.use({ conversationCase: 'agent-rec-string-node-into-prompt' }) - - test('adds a recorded string node, sets its text, and wires it into the positive prompt', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { name: /^Ran 5 tool calls/ }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows).toHaveCount(5) - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Show node' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // One graph_ops entry (add_node + set_widget + connect) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - - // The recorded node id, its text widget value on the unfiltered DOM. - const stringNode = page.locator('[data-node-id="1896929324796827"]') - await expect(stringNode).toBeVisible() - await expect(stringNode.getByLabel('value', { exact: true })).toHaveValue( - 'a photo of a pier at sunset' - ) - - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - // The wired `text` input is widget-backed and renders as a widget, - // not a slot row; the link shows on the string node's output. - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '1896929324796827', - title: 'PrimitiveStringMultiline', - inputs: [], - outputs: [true] - } - ]) - }) - }) - - test.describe('recorded agent-rec-second-prompt-encoder', () => { - test.use({ conversationCase: 'agent-rec-second-prompt-encoder' }) - - test('adds a recorded second encoder wired from CLIP into the negative input', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // The recording's closing `validate` call errored (the seed's checkpoint - // is not installed on the recording host), so the group is a failed one: - // it stays open at turn end and its rows are visible with NO click. - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Validate' })).toBeVisible() - await expect( - panel.getByRole('button', { name: /^Ran 5 tool calls/ }) - ).toHaveAttribute('aria-expanded', 'true') - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // One graph_ops entry (add_node + set_widget + connect x2) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - await expect( - page.locator('[data-node-id="2904574201837211"]') - ).toBeVisible() - - // The sampler's negative input (index 2) flips false -> true; the new - // encoder is wired on both sides. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, true, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '2904574201837211', - title: 'CLIPTextEncode', - inputs: [true], - outputs: [true] - } - ]) - }) - }) - - test.describe('recorded agent-rec-set-widget-existing', () => { - test.use({ conversationCase: 'agent-rec-set-widget-existing' }) - - test('applies recorded set_widget ops to an existing node and adds nothing', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { name: /^Ran 4 tool calls/ }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows).toHaveCount(4) - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'List slots' })).toBeVisible() - - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // One graph_ops entry (two set_widget ops) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - - // The widget values on the unfiltered DOM: the aria-label sits on the - // number widget's container, so descend to its input. - const sampler = page.locator('[data-node-id="3"]') - await expect( - sampler.getByLabel('steps', { exact: true }).locator('input') - ).toHaveValue('30') - await expect( - sampler.getByLabel('cfg', { exact: true }).locator('input') - ).toHaveValue('5') - - // No node added or removed: the seed's five, wired as seeded. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] } - ]) - }) - }) - - test.describe('recorded agent-rec-replace-prompt-encoder', () => { - test.use({ conversationCase: 'agent-rec-replace-prompt-encoder' }) - - test('deletes the recorded prompt node and rewires a fresh encoder in its place', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // The recording's closing `validate` errored on the host's missing - // checkpoint, so the group stays open: rows are visible with NO click. - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Delete node' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect( - panel.getByRole('button', { name: /^Ran 6 tool calls/ }) - ).toHaveAttribute('aria-expanded', 'true') - - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // Two graph_ops entries (the delete, then the add + wiring) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '3 updates' - ) - - // The deleted node is gone from the unfiltered DOM, not merely filtered - // out of the snapshot. - await expect(page.locator('[data-node-id="6"]')).toHaveCount(0) - await expect( - page.locator('[data-node-id="4073815636872683"]') - ).toBeVisible() - - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '4073815636872683', - title: 'CLIPTextEncode', - inputs: [true], - outputs: [true] - } - ]) - }) - }) - - test.describe('recorded agent-rec-refiner-between', () => { - test.use({ conversationCase: 'agent-rec-refiner-between' }) - - test('adds a recorded refiner sampler between the first sampler and the decoder', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // The recording's closing `validate` errored on the host's missing - // checkpoint, so the group stays open: rows visible with NO click. - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect( - panel.getByRole('button', { name: /^Ran 5 tool calls/ }) - ).toHaveAttribute('aria-expanded', 'true') - - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // One graph_ops entry (add + four connects + three set_widget) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - await expect( - page.locator('[data-node-id="2629930598730104"]') - ).toBeVisible() - - // The refiner is wired on both sides: its latent_image input (index 3) - // takes the first sampler, and its own LATENT output feeds the decoder, - // whose samples input keeps its connection by last-writer-wins. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '2629930598730104', - title: 'KSampler', - inputs: [true, true, false, true], - outputs: [true] - } - ]) - }) - }) - - test.describe('recorded agent-rec-batched-ops', () => { - test.use({ conversationCase: 'agent-rec-batched-ops' }) - - test('applies two recorded adds and two connects as one batched entry', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { name: /^Ran 3 tool calls/ }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows).toHaveCount(3) - // One apply call, so its row carries no coalescing badge: two adjacent - // apply calls would render as a single "Apply ops" row with "x2". - const applyRow = toolRows.filter({ hasText: 'Apply ops' }) - await expect(applyRow).toHaveCount(1) - // The coalescing badge renders as a leading multiplication sign - // (ToolCallCard.vue renders it only when count > 1). - await expect(applyRow).not.toContainText('\u00d7') - - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // ONE graph_ops entry for all four ops, plus the catch-up: two unbatched - // apply calls would read "3 updates". - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - - await expect( - page.locator('[data-node-id="3495500264284537"]') - ).toBeVisible() - await expect( - page.locator('[data-node-id="4454558117922341"]') - ).toBeVisible() - - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, false, false], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '3495500264284537', - title: 'CLIPTextEncode', - inputs: [true], - outputs: [false] - }, - { - id: '4454558117922341', - title: 'CLIPTextEncode', - inputs: [true], - outputs: [false] - } - ]) - }) - }) - - // The two eval-suite cases, recorded on the local agent (they replace the - // synthesized PoC fixtures). The recorded prompt is the eval prompt behind - // the tab switch the replay binds on. - test.describe('recorded agent-workflow-editing-05', () => { - test.use({ conversationCase: 'agent-workflow-editing-05' }) - - test('stitches the recorded output beside the input and routes it to the save', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - const toolGroup = panel.getByRole('button', { name: /^Ran 7 tool calls/ }) - await expect(toolGroup).toBeVisible() - await expect(toolGroup).toHaveAttribute('aria-expanded', 'false') - await toolGroup.click() - const toolRows = panel.getByRole('listitem') - await expect(toolRows).toHaveCount(7) - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'List slots' })).toBeVisible() - - await expect( - panel.getByRole('button', { name: 'Open Image edit' }) - ).toBeVisible() - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // Two graph_ops entries (apply_ops, then the follow-up connect) plus the - // catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '3 updates' - ) - await expect( - page.locator('[data-node-id="1380502390045812"]') - ).toBeVisible() - - // The stitch takes the source image and the decoded image; its output - // replaces the decoder's link into the save node (one link per input). - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { id: '1', title: 'LoadImage', inputs: [], outputs: [true, false] }, - { - id: '2', - title: 'VAEEncode', - inputs: [true, false], - outputs: [true] - }, - { - id: '3', - title: 'KSampler', - inputs: [false, false, false, true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, false], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - { - id: '1380502390045812', - title: 'ImageStitch', - inputs: [true, true], - outputs: [true] - } - ]) - }) - }) - - test.describe('recorded agent-l4-zimage-string-node-prompt', () => { - test.use({ conversationCase: 'agent-l4-zimage-string-node-prompt' }) - - test('adds the recorded text node and the inputs the sampler was missing', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - // The recording's `validate` and `list_model_picks` calls errored (no - // checkpoint on the recording host), so the group stays open at turn end. - const toolRows = panel.getByRole('listitem') - await expect(toolRows.filter({ hasText: 'Switched tabs' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Apply ops' })).toBeVisible() - await expect(toolRows.filter({ hasText: 'Validate' })).toBeVisible() - await expect( - panel.getByRole('button', { name: /^Ran 9 tool calls/ }) - ).toHaveAttribute('aria-expanded', 'true') - - await expect( - panel.getByRole('button', { name: 'Open Text to image' }) - ).toBeVisible() - await expect(panel.getByTestId('markdown-stream').first()).not.toBeEmpty() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - // One graph_ops entry (nine ops) plus the catch-up. - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '2 updates' - ) - const stringNode = page.locator('[data-node-id="2346297545678642"]') - await expect(stringNode).toBeVisible() - await expect(stringNode.getByLabel('value', { exact: true })).toHaveValue( - 'a red bicycle on a pier at dusk' - ) - - // Every sampler input is now fed: the recorded turn also added a - // negative encoder and an empty latent. - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '3', - title: 'KSampler', - inputs: [true, true, true, true], - outputs: [true] - }, - { - id: '4', - title: 'CheckpointLoaderSimple', - inputs: [], - outputs: [true, true, true] - }, - { - id: '6', - title: 'Positive prompt', - inputs: [true], - outputs: [true] - }, - { - id: '8', - title: 'VAEDecode', - inputs: [true, true], - outputs: [true] - }, - { id: '9', title: 'SaveImage', inputs: [true], outputs: [] }, - // graphSnapshot sorts nodes by numeric id. - { - id: '1068057559864246', - title: 'CLIPTextEncode', - inputs: [true], - outputs: [true] - }, - { - id: '2346297545678642', - title: 'PrimitiveStringMultiline', - inputs: [], - outputs: [true] - }, - { - id: '4476795995918869', - title: 'EmptyLatentImage', - inputs: [], - outputs: [true] + for (const conversationCase of listRecordedConversations()) { + const conversation = loadAgentConversation(conversationCase) + const groups = recordedToolCallGroups(conversation) + + test.describe(`recorded ${conversationCase}`, () => { + test.use({ conversationCase }) + + test(`replays: ${conversation.request.content}`, async ({ + agentConversation, + page + }) => { + test.setTimeout(60_000) + const { panel } = agentConversation + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + const groupButtons = panel.getByRole('button', { + name: /^Ran \d+ tool call/ + }) + await expect(groupButtons).toHaveCount(groups.length) + const toolRows = panel.getByRole('listitem') + for (const [index, calls] of groups.entries()) { + const button = groupButtons.nth(index) + const succeeded = calls.every((call) => call.ok) + await expect(button).toHaveText( + new RegExp(`^Ran ${calls.length} tool call`) + ) + // A failed group stays open at turn end; a clean one collapses. + await expect(button).toHaveAttribute( + 'aria-expanded', + succeeded ? 'false' : 'true' + ) + if (succeeded) await button.click() + for (const call of calls) { + await expect( + toolRows.filter({ hasText: toolRowLabel(call.name) }).first() + ).toBeVisible() } - ]) + } + + await expect( + panel.getByRole('button', { + name: `Open ${conversation.workflow.name}` + }) + ).toBeVisible() + await expect( + panel.getByTestId('markdown-stream').first() + ).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + `${expectedUpdateCount(conversation)} updates` + ) + + for (const id of recordedAddedNodeIds(conversation)) { + await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() + } + for (const { nodeId, widget, value } of recordedWidgetValues( + conversation + )) { + const field = page + .locator(`[data-node-id="${nodeId}"]`) + .getByLabel(widget, { exact: true }) + const control = + typeof value === 'string' ? field : field.locator('input') + await expect(control).toHaveValue(String(value)) + } + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(expectedGraphSnapshot(conversation)) + }) }) - }) + } }) diff --git a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts index 839288b2a2d4..eaa26995ef5b 100644 --- a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts +++ b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts @@ -2,31 +2,9 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -/** - * Replay regression for PR #16611 (preserve root semantics for human graph - * edits). Pre-fix, a node the user deleted INSIDE a subgraph minted a - * root-scope `delete_node` wire op - the doc lost a root node it still had. - * The fix refuses to mint subgraph-interior creates/deletes (they have no - * wire representation) while root-level deletes keep minting. - * - * The conversation binds the follower to a doc seeded with a subgraph - * definition and instance; the human edits are then driven at the layout - * store level with the session's own local (user) actor, mirroring the - * operations the canvas attach/detach emitters produce. Store-level driving - * is deliberate: UI-path deletes route through litegraph selection and - * cannot see ECS-only replay seeds (see the 16611 red-row diagnosis on the - * DOPPELGANGER review pad) - the mint gate contract under test is exercised - * exactly, the UI hit-testing above it is not. - * - * Red at base with 0a20384d8a reverted (reverse-apply the commit, then - * RESTART the dev server - the spec imports the layout store by module URL, - * which only resolves to the app's own instance against a module graph that - * matches disk; running against an HMR-churned server loads a second, empty - * store instance): the interior delete surfaces as an outbound - * `delete_node` for node 10 (`['1', '10', '2']`). - */ +// Regression for #16611: a delete inside a subgraph must not mint a root-scope wire op. +// Edits go through the layout store because UI-path deletes cannot see follower-only nodes yet (FE-1996). -/** The layout-store surface this spec drives, structurally typed. */ interface LayoutStoreModule { layoutStore: { getNodeLayout( @@ -72,10 +50,7 @@ test.describe( const { layoutStore } = (await import( /* @vite-ignore */ storeModule )) as unknown as LayoutStoreModule - // The follower keys layouts under the canvas root graph id; earlier - // revisions keyed them under the doc's workflow id. Resolving by - // lookup keeps the red half (fix reverted) driving the same seeded - // layouts instead of erroring before the contract assertion. + // Layouts are keyed by the canvas root graph id, not the workflow id. const rootGraphId = [window.app!.graph.id, workflowId].find((id) => layoutStore.getNodeLayout(id, '1') ) @@ -92,18 +67,13 @@ test.describe( source: 'canvas' }) - // Positive control: a ROOT-level delete must mint, proving the - // doc-bound mint path is live in this environment. + // Positive control: a root-level delete must mint. layoutStore.applyOperation({ type: 'deleteNode', ...meta('1', rootGraphId) }) - // The regression surface: a node INSIDE the seeded subgraph. Its - // layout attaches the way entering the subgraph attaches it - // (ownerGraphId = the subgraph), then the user deletes it. Both the - // create and the delete are unrepresentable on the wire - neither - // may mint. + // Interior create and delete have no wire representation; neither may mint. layoutStore.applyOperation({ type: 'createNode', ...meta('10', subgraphId), @@ -121,10 +91,7 @@ test.describe( ...meta('10', subgraphId) }) - // Ordering anchor: a second root-level delete that must mint. - // Frames are sent in order, so a minted interior delete would be on - // the wire before this one - waiting for it gives the interior op - // every chance to surface before the negative below is trusted. + // A later root delete anchors the order: an interior mint would precede it. layoutStore.applyOperation({ type: 'deleteNode', ...meta('2', rootGraphId) @@ -141,7 +108,6 @@ test.describe( .poll(() => deletedNodeIds().length, { timeout: 10_000 }) .toBeGreaterThanOrEqual(2) - // Exactly the two root deletes, in order; node 10 never minted. expect(deletedNodeIds()).toEqual(['1', '2']) }) } diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts index 2bab1bac8e00..2c5bc1c9ae2a 100644 --- a/scripts/agentConversationCapture.test.ts +++ b/scripts/agentConversationCapture.test.ts @@ -99,6 +99,30 @@ describe('exportAgentConversation', () => { ]) }) + it('emits each entry at its offset from the first recorded frame', () => { + const [running, success] = capture.frames + const exported = exportAgentConversation({ + ...capture, + frames: [ + { ...running, at_ms: 1_700_000_001_000 }, + { ...success, at_ms: 1_700_000_001_750 } + ] + }) + expect(exported.response.map((entry) => entry.at_ms)).toEqual([0, 750, 750]) + expect( + exported.response.every( + (entry) => entry.kind !== 'event' || !('at_ms' in entry.event) + ) + ).toBe(true) + }) + + it('leaves the offset out when the frames carry no receipt time', () => { + const exported = exportAgentConversation(capture) + expect(exported.response.every((entry) => entry.at_ms === undefined)).toBe( + true + ) + }) + it('refuses an accepted op missing from the recorded result', () => { expect(() => exportAgentConversation({ diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts index d956f16818db..8f9b0f314e9e 100644 --- a/scripts/agentConversationCapture.ts +++ b/scripts/agentConversationCapture.ts @@ -94,9 +94,14 @@ export function exportAgentConversation(input: unknown) { ) const emittedToolCalls = new Set() const response: Array< - | { kind: 'event'; event: z.infer } - | { kind: 'graph_ops'; ops: Array> } + | { kind: 'event'; event: z.infer; at_ms?: number } + | { kind: 'graph_ops'; ops: Array>; at_ms?: number } > = [] + const firstAt = capture.frames[0].at_ms + const relativeAt = (frame: { at_ms?: number }) => + firstAt === undefined || frame.at_ms === undefined + ? undefined + : frame.at_ms - firstAt for (const frame of capture.frames) { if ( @@ -107,8 +112,9 @@ export function exportAgentConversation(input: unknown) { `recorded ${frame.type} frame does not belong to capture ${capture.capture.thread_id}/${capture.capture.message_id}` ) } + const at_ms = relativeAt(frame) const event = { - ...frame, + type: frame.type, data: { ...frame.data } } delete event.data.thread_id @@ -124,12 +130,12 @@ export function exportAgentConversation(input: unknown) { ) { const operations = operationsFromResult(toolCalls.get(toolCallId)!) if (operations.length > 0) { - response.push({ kind: 'graph_ops', ops: operations }) + response.push({ kind: 'graph_ops', ops: operations, at_ms }) } emittedToolCalls.add(toolCallId) } } - response.push({ kind: 'event', event }) + response.push({ kind: 'event', event, at_ms }) } const omitted = capture.tool_calls.filter( From 4da760e5c112e391d3411f593084960182f85cce Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:01:45 -0700 Subject: [PATCH 20/49] test(agent): move the recorded conversations to a stacked PR The twelve recorded conversation fixtures land in their own PR on top of this one so the replay code and the data review separately. --- .../agent-l4-zimage-string-node-prompt.json | 526 ------------------ .../conversations/agent-rec-batched-ops.json | 379 ------------- .../agent-rec-refiner-between.json | 442 --------------- .../agent-rec-replace-prompt-encoder.json | 405 -------------- .../agent-rec-second-prompt-encoder.json | 379 ------------- .../agent-rec-set-widget-existing.json | 317 ----------- .../agent-rec-string-node-into-prompt.json | 360 ------------ .../agent-rec-text-only-answer.json | 266 --------- .../conversations/agent-rec-tool-error.json | 267 --------- .../agent-workflow-editing-05.json | 412 -------------- 10 files changed, 3753 deletions(-) delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json deleted file mode 100644 index 3ace5c45f25a..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json +++ /dev/null @@ -1,526 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-l4-zimage-string-node-prompt", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 98f33b92-6517-478e-a348-19db814e2685; message 5543eebd-40ed-408c-8574-f6a0e2d1e1c9; workflow 0150feeb-61a0-4dd2-b3c4-df8e743e31d4 (seeded by throwaway turn a9b183b7-9006-44ed-88ed-6c7f7cae1e7b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['e74f78e1-874e-4673-aaf5-0ef9b45ece1e', 'a3fac78f-e0a5-43af-b43d-979f4a0c0772', '6e9347a0-2025-4d03-9b95-ce9e68cd8727', '3f61fdd8-3a0c-4f04-8697-23ac2c398b34', '0520e93c-8a51-4ce5-ab5a-c80eb49d8d0b', 'd82a0aad-c81c-4776-938a-e0bf02056ff8', 'cc97e012-c484-40f9-a864-a97c9ba2e4bb', 'bcadb3c6-f385-4a1f-ae9f-b99ff9b4e0bd', 'f93556bb-a8bb-4ba2-9f72-d32056a80587']; rows agent-rec-zimage-string-node-prompt.zi-a2.rows.json; raw capture sha256 844318cc2db8e402e576d48e64336a4733455f053bb6eceea41d3a01349465e9 Replaces the synthesized PoC fixture of the same case id: same eval prompt, recorded on the local agent behind the tab switch the replay binds on.", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "98f33b92-6517-478e-a348-19db814e2685", - "message_id": "5543eebd-40ed-408c-8574-f6a0e2d1e1c9", - "exported_at": "2026-09-03T12:01:27Z" - } - }, - "workflow": { - "id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - }, - "EmptyLatentImage": { - "widget_order": ["width", "height", "batch_size"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "0150feeb-61a0-4dd2-b3c4-df8e743e31d4" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 11, - "status": "success", - "tool_call_id": "toolu_01EjSa5ZT7qaRRqCEBFwUYLY", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 594, - "status": "success", - "tool_call_id": "toolu_011b7ZgsTFUVYggGLKAPwttu", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 185, - "status": "success", - "tool_call_id": "toolu_013REBJ3bgPMbgKaXSLAmkAG", - "tool_name": "find_nodes" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [395, 280], - "node": { - "id": 2346297545678642, - "pos": [395, 280], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b2a640a31f1946119e4389cfbfa38610", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2346297545678642, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "731fc9d4faf84ae0840e0b3a45a03a48", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a red bicycle on a pier at dusk", - "widget": "value", - "node_id": 2346297545678642, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "9e0615aa944d426db3e5932b7ed2de8f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1868944393408444, - "to_node": 6, - "to_slot": 1, - "from_node": 2346297545678642, - "from_slot": 0, - "link_type": "STRING", - "base_version": 2 - }, - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 1068057559864246, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "6521c0a17c584541bcd3539250cf877c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1068057559864246, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "68344eaa0b5c4590bfa572951a13458f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "", - "widget": "text", - "node_id": 1068057559864246, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "a5d0567473004a4e9cb96a8ef0c251e1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 769581307347504, - "to_node": 1068057559864246, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "37b1548fe5d943b2b2738353d96892f1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2229783680013788, - "to_node": 3, - "to_slot": 2, - "from_node": 1068057559864246, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 - }, - { - "op": "add_node", - "pos": [395, 406], - "node": { - "id": 4476795995918869, - "pos": [395, 406], - "mode": 0, - "size": [240, 134], - "type": "EmptyLatentImage", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [512, 512, 1] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "833c3d83b4bb4c88a0e47d1448573f81", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4476795995918869, - "class_type": "EmptyLatentImage", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "23cb809c1f8c4cafb067880b01ee9a04", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1532394889807820, - "to_node": 3, - "to_slot": 3, - "from_node": 4476795995918869, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 208, - "status": "success", - "tool_call_id": "toolu_01Bob4FLcTWSWKt9cetrSQcU", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "error", - "tool_call_id": "toolu_014CR2Q7xLRCmDXEaydcbf4s", - "tool_name": "validate" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 184, - "status": "success", - "tool_call_id": "toolu_01LR9BRaBnxH5hdBvg91GQiE", - "tool_name": "show_node" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 236, - "error_code": "knowledge_unavailable", - "status": "error", - "tool_call_id": "toolu_01FgBW9iiC8boKQAu6mmYZFt", - "tool_name": "list_model_picks" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 667, - "status": "success", - "tool_call_id": "toolu_015pTnqS23ko4wetU427Ji3Z", - "tool_name": "list_generate_models" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 243, - "status": "success", - "tool_call_id": "toolu_01QwFYGv31NWSPfeZXcvd6Ge", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done \u2014 the text node is on your canvas and wired into the prompt, with \"a red bicycle on a pier at dusk\" in it. I also filled in the two inputs the sampler was missing (an empty negative prompt and a 512\u00d7512 blank canvas) so the graph is complete.\n\nI couldn't run it, though: this workflow uses the SDXL base checkpoint, and this server currently has **no checkpoint files installed at all** \u2014 so the run would fail immediately. That's an environment issue, not a wiring one.\n\nTwo ways forward, your call:\n- Install/download a checkpoint on the server, and I'll run this graph exactly as it is.\n- Or I can build the same prompt on one of the hosted image models available here (Flux, Ideogram, Stability, Recraft, and others) in a **new tab**, leaving this workflow untouched \u2014 that one I can run right now." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 4471, - "cache_read_input_tokens": 395107, - "input_tokens": 20, - "output_tokens": 3071, - "total_tokens": 3091 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json deleted file mode 100644 index eba81063518c..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json +++ /dev/null @@ -1,379 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-batched-ops", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 62ff8e48-248c-48cf-9fed-24b21aaa141a; message d896b67c-2e87-4312-876c-2f0765b823b3; workflow 7487a6dc-a100-4641-b76b-feba99d6fd1e (seeded by throwaway turn c850dd73-373b-4367-a363-9ff4f081affe; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['26779044-2bc9-4728-b9ec-1b389475ea22', '04a41da6-1bb7-4f78-b5f6-185dc11bf1a8', 'd845093c-b8f5-4a84-a7ac-d5d194911bfc']; rows agent-rec-batched-ops.r6-a1.rows.json; raw capture sha256 50dd0603285e88b34ecd0ff1864c44364a79244b398778a29c7ccb714c6be0fb", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "62ff8e48-248c-48cf-9fed-24b21aaa141a", - "message_id": "d896b67c-2e87-4312-876c-2f0765b823b3", - "exported_at": "2026-09-03T01:10:17Z" - } - }, - "workflow": { - "id": "7487a6dc-a100-4641-b76b-feba99d6fd1e", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first, then read its graph to find the checkpoint loader's real id." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "7487a6dc-a100-4641-b76b-feba99d6fd1e" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 7, - "status": "success", - "tool_call_id": "toolu_01T3HLiCeFHR77H9rYxPaqrG", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 723, - "status": "success", - "tool_call_id": "toolu_01TtC8NC6yVXCzk3nW8mJRC6", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 3495500264284537, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "8d0af55a29224e089eab1547177b2cb0", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 3495500264284537, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "add_node", - "pos": [715, 406], - "node": { - "id": 4454558117922341, - "pos": [715, 406], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c5d06a0a0800452794e4eec01fdd27e9", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4454558117922341, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "01727acc9c784d5b85975265e1112574", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 95739764921160, - "to_node": 3495500264284537, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "70380b8a26f64d558b56a050489544c2", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3128959453071432, - "to_node": 4454558117922341, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 243, - "status": "success", - "tool_call_id": "toolu_016atkf2nc3JCG6ZAx3L5FzP", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — focus is on \"Text to image,\" and both new text-encode nodes are on the canvas with their CLIP inputs fed from the checkpoint loader, all in one batch. They're not wired into the sampler yet, so tell me what each should feed (e.g. one as the negative prompt) and I'll connect them." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 1500, - "cache_read_input_tokens": 153287, - "input_tokens": 8, - "output_tokens": 517, - "total_tokens": 525 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json deleted file mode 100644 index cb0dacd6f46f..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json +++ /dev/null @@ -1,442 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-refiner-between", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2b47ebe1-d43e-42cc-88ea-47c6110f8594; message 1ca0e0eb-4475-4d47-92c4-4842e664c85d; workflow 4c2246c4-bf06-4117-b85d-7dd86eb1142c (seeded by throwaway turn 87004117-f7b4-4aa6-9d3c-03adfa69b405; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d04fab84-761f-4d29-96b6-14acb059fbd8', '30062b8a-f0cb-4035-b978-6fb486a4a6bc', '0506f122-389e-4e3d-8384-75652a672b86', '02cc5714-4eb8-4cac-878c-d7be00829fa8', '05657151-ffc2-4aa8-be17-c0b919befbef']; rows agent-rec-refiner-between.r5-a1.rows.json; raw capture sha256 60d75f59c47625f231fc68272617204751373cbd1b99454ca21467cee479d5c9", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "2b47ebe1-d43e-42cc-88ea-47c6110f8594", - "message_id": "1ca0e0eb-4475-4d47-92c4-4842e664c85d", - "exported_at": "2026-09-03T01:09:03Z" - } - }, - "workflow": { - "id": "4c2246c4-bf06-4117-b85d-7dd86eb1142c", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and look at the graph." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "4c2246c4-bf06-4117-b85d-7dd86eb1142c" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 5, - "status": "success", - "tool_call_id": "toolu_01VGCkwgpcwHirAK5wpLEWh6", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 574, - "status": "success", - "tool_call_id": "toolu_01GL3qKHcRRTkxVZeDXmoN3K", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [1565, 280], - "node": { - "id": 2629930598730104, - "pos": [1565, 280], - "mode": 0, - "size": [240, 290], - "type": "KSampler", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "model", - "type": "MODEL" - }, - { - "link": null, - "name": "positive", - "type": "CONDITIONING" - }, - { - "link": null, - "name": "negative", - "type": "CONDITIONING" - }, - { - "link": null, - "name": "latent_image", - "type": "LATENT" - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "96148885872a4913a0a017d5f9590184", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2629930598730104, - "class_type": "KSampler", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "bdbef333346f4b71910e48bb295cc9e1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 507448049071742, - "to_node": 2629930598730104, - "to_slot": 0, - "from_node": 4, - "from_slot": 0, - "link_type": "MODEL", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "e864aa7ae7234c5fae328b9852cb0e4f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2030924172213340, - "to_node": 2629930598730104, - "to_slot": 1, - "from_node": 6, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7facf6240836428289b0430655ae082a", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1914333209534315, - "to_node": 2629930598730104, - "to_slot": 3, - "from_node": 3, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "5e0988e2e87943a693a4f823cead58bb", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3005141419537112, - "to_node": 8, - "to_slot": 0, - "from_node": 2629930598730104, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 - }, - { - "op": "set_widget", - "old": 20, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "ff7b38b3eb7642edbe6e26b434813f9a", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 20, - "widget": "steps", - "node_id": 2629930598730104, - "base_version": 2 - }, - { - "op": "set_widget", - "old": 1, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "f720afe2838649cb9a0b60a18387284c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 0.5, - "widget": "denoise", - "node_id": 2629930598730104, - "base_version": 2 - }, - { - "op": "set_widget", - "old": 8, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "13ce136537ef462f9c4cae51dffc5c51", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 7, - "widget": "cfg", - "node_id": 2629930598730104, - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 250, - "status": "success", - "tool_call_id": "toolu_019JQMWFbRUwQJTnvgpPP3ti", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 197, - "status": "success", - "tool_call_id": "toolu_019MucYksSrwHhjJwSS8YTCu", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 197, - "status": "error", - "tool_call_id": "toolu_01CMStzBtWs89YdxxxAbe9SB", - "tool_name": "validate" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the refiner is in and wired through: first sampler's latent → refiner → VAE decoder → save. I gave it 20 steps at 0.5 denoise so it polishes rather than re-invents the image; say the word if you want that stronger or weaker.\n\nTwo things were already missing in this graph before I touched it, and they'll stop a run:\n\n- Neither sampler has a negative prompt connected, and the first sampler has no starting latent (no empty-image/size node).\n- The checkpoint it names, `sd_xl_base_1.0`, isn't installed here — no checkpoint options are showing at all.\n\nWant me to add the negative prompt and the image-size node, and pick an available checkpoint?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2949, - "cache_read_input_tokens": 232730, - "input_tokens": 12, - "output_tokens": 1622, - "total_tokens": 1634 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json deleted file mode 100644 index 6a5f16ca0c51..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json +++ /dev/null @@ -1,405 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-replace-prompt-encoder", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6ee091cb-1d46-4b7b-bcc1-5002c1908e2a; message 2d192b16-5e0b-4bdd-b0dd-d1293950f37a; workflow 2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6 (seeded by throwaway turn 4dd6f9b1-9b01-4e75-9e3e-710c63c8a158; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d9725d24-b208-456c-9fea-cbd99272a73a', 'e2dc9c25-abe9-41b8-8c0b-5fc3ad99ff09', '7153430f-a424-463c-9d5d-7603252d3f67', '5ce083d4-2c3f-44d6-8ceb-fd79d99a6375', 'bea39c10-b218-4945-bdcb-b09701c3e472', '9bc04a9e-9d43-47aa-816d-d92f98ed521c']; rows agent-rec-replace-prompt-encoder.r4-a1.rows.json; raw capture sha256 bb270e40c444456057ed4b6d188e12f452282a9af823d3a21f6c7a5c29dc234c", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "6ee091cb-1d46-4b7b-bcc1-5002c1908e2a", - "message_id": "2d192b16-5e0b-4bdd-b0dd-d1293950f37a", - "exported_at": "2026-09-03T01:07:33Z" - } - }, - "workflow": { - "id": "2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and read the graph first." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "2bd97ec8-a3e0-4b3a-b3ff-f6920693b7d6" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 6, - "status": "success", - "tool_call_id": "toolu_01W3neXmtRXyGw31YjNTfwd4", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 602, - "status": "success", - "tool_call_id": "toolu_01KNsLCccDucdm2Bknj7JJeN", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "delete_node", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "703a1427e3d844ba9c525c8e171342b7", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 6, - "base_version": 2, - "removed_links": [1, 2] - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 198, - "status": "success", - "tool_call_id": "toolu_018HhyebfnrUDExa4ZFTip5x", - "tool_name": "delete_node" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 4073815636872683, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7406a83a71234d63991d53dc1394327b", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4073815636872683, - "class_type": "CLIPTextEncode", - "base_version": 3 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7bbed04e5a9b4d1c984635bc738bb631", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 909865724937361, - "to_node": 4073815636872683, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 3 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "43dfb8cd293c4eb482e17a7f01e7c791", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4412673605160400, - "to_node": 3, - "to_slot": 1, - "from_node": 4073815636872683, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 3 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "aa40bf71be2e444e86d24e2da3d04c27", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a photo of a pier", - "widget": "text", - "node_id": 4073815636872683, - "base_version": 3 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "success", - "tool_call_id": "toolu_01Jq7JbdaxsTDJXfc4R2kX3r", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 200, - "status": "success", - "tool_call_id": "toolu_01H3psK8qCtRWqQjxium2rJv", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 202, - "status": "error", - "tool_call_id": "toolu_01Jn4m6BiqVX3XTNRTQYLJ9x", - "tool_name": "validate" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the old positive prompt node is gone, and a fresh one is in its place on the \"Text to image\" tab: CLIP from the checkpoint loader in, conditioning into the sampler's positive input, prompt text carried over (\"a photo of a pier\").\n\nTwo things that were already missing before this swap and still are: the sampler has no negative prompt and no empty-latent (image size) input wired, so it won't run as-is. Want me to add those?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2851, - "cache_read_input_tokens": 272173, - "input_tokens": 14, - "output_tokens": 1015, - "total_tokens": 1029 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json deleted file mode 100644 index a41533bb0024..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json +++ /dev/null @@ -1,379 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-second-prompt-encoder", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 70f97c01-00e9-4e8d-86bc-ff73cf7500e2; message b55b90fa-b206-4c7b-9696-79fcb22ab8db; workflow 087617ce-c168-4bb8-86b2-05492db91a8d (seeded by throwaway turn 1bdaa2f6-3560-4405-92ec-bbe4c8f5370f; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['8e612681-1ad2-4fdb-8395-1d3e68566c7f', '156eed82-d0f4-40d1-994a-9a3c864169e0', '9c7e33ca-d9b0-4623-99a0-7f5b6a095b51', '0860390b-de13-44e2-927b-ed75988f387a', 'bebaa9e9-242f-4ba5-82e8-7d7b29664009']; rows agent-rec-second-prompt-encoder.r2-a1.rows.json; raw capture sha256 09f8ce5f32c0c30a565a65f43ef572f8ff52b8fb1793de667a056dc87b1ab4a9", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "70f97c01-00e9-4e8d-86bc-ff73cf7500e2", - "message_id": "b55b90fa-b206-4c7b-9696-79fcb22ab8db", - "exported_at": "2026-09-03T01:04:16Z" - } - }, - "workflow": { - "id": "087617ce-c168-4bb8-86b2-05492db91a8d", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and wire its conditioning into the KSampler's negative input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "087617ce-c168-4bb8-86b2-05492db91a8d" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 8, - "status": "success", - "tool_call_id": "toolu_01RxsV9XAFq5tdPvcjqxzf7X", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 867, - "status": "success", - "tool_call_id": "toolu_01JUkZjF5v8B1g5B4oiPUbhh", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 2904574201837211, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "30fc2175ec794e10bd7ba38aa43a063c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2904574201837211, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "4b4bf9278ba840028f7ca4a4c03a7648", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 967330603592352, - "to_node": 2904574201837211, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "5469e1e4858b4565a0a0244011298a93", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2932910430668226, - "to_node": 3, - "to_slot": 2, - "from_node": 2904574201837211, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "e11c18ea33a1402396bf6743884f9efa", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "blurry, low quality, watermark", - "widget": "text", - "node_id": 2904574201837211, - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 225, - "status": "success", - "tool_call_id": "toolu_012JnLUhqxiargezRkF2zY4J", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 209, - "status": "success", - "tool_call_id": "toolu_01Wn2YU8i3efNRfoQNgBRN9d", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 256, - "status": "error", - "tool_call_id": "toolu_011Cqp8r4x5nnjChpoJNnTg1", - "tool_name": "validate" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — that tab now has a second text encode node reading from the checkpoint's CLIP, feeding the sampler's negative input. I gave it a starter negative prompt (\"blurry, low quality, \") — change the text to whatever you like.\n\nTwo pre-existing issues in that graph, unrelated to this change: the sampler still has nothing wired into its latent input (it needs an empty latent to set the image size), and the checkpoint it points at isn't installed here. Want me to add the empty latent and pick an available checkpoint?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2524, - "cache_read_input_tokens": 232347, - "input_tokens": 12, - "output_tokens": 854, - "total_tokens": 866 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json deleted file mode 100644 index 1e6d3c940cce..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json +++ /dev/null @@ -1,317 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-set-widget-existing", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 42ed6743-f0f3-4bdd-99f2-852657590361; message 52c2229a-b77b-4596-b921-dcbeec3ba16d; workflow 99036750-1b1b-4ba9-8a47-bc81358f8b09 (seeded by throwaway turn bb6e501d-056e-4e80-a7fa-0938dc18c684; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['6b97831c-40e0-4105-93de-666025526b3f', '31482711-0a5e-4650-95f0-dc4422bf94b1', '6ea3de75-0d0a-4fbc-9b1b-33d9a317b5b2', '33133dbc-9880-46b8-8793-9636015db2ba']; rows agent-rec-set-widget-existing.r3-a1.rows.json; raw capture sha256 7b61b8cc5530dd548a3fa632459a553d5d6fb6d4687181cca45e2f1027a8e81b", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "42ed6743-f0f3-4bdd-99f2-852657590361", - "message_id": "52c2229a-b77b-4596-b921-dcbeec3ba16d", - "exported_at": "2026-09-03T01:06:06Z" - } - }, - "workflow": { - "id": "99036750-1b1b-4ba9-8a47-bc81358f8b09", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "99036750-1b1b-4ba9-8a47-bc81358f8b09" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 4, - "status": "success", - "tool_call_id": "toolu_01RRsWS3QvX5bmVH9wZJcJAL", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 618, - "status": "success", - "tool_call_id": "toolu_011KWGg8bY3qf8pZQCigaA5b", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "set_widget", - "old": 20, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "211d808e180b4d82926799ede49a7b41", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 30, - "widget": "steps", - "node_id": 3, - "base_version": 2 - }, - { - "op": "set_widget", - "old": 7, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b4a6980f6bfa400d9b7d59630ca3adef", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 5, - "widget": "cfg", - "node_id": 3, - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 196, - "status": "success", - "tool_call_id": "toolu_01Fto3Lreshdk2LVPjcgHaUn", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 348, - "status": "success", - "tool_call_id": "toolu_012xgj4F53Nyavkc86wk379c", - "tool_name": "list_slots" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — you're on the \"Text to image\" tab, with steps now 30 and cfg 5. Everything else is untouched." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 1557, - "cache_read_input_tokens": 192283, - "input_tokens": 10, - "output_tokens": 420, - "total_tokens": 430 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json deleted file mode 100644 index f9aefc16b061..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json +++ /dev/null @@ -1,360 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-string-node-into-prompt", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread cb0eb98e-919d-4082-bccf-0b72aaa8baea; message 2ece02a3-a880-4632-b154-03462cd3aa64; workflow 38d9e9c7-e0fb-48f2-8f9a-391f7afba70c (seeded by throwaway turn 271d7739-d3bc-425c-b1ec-aaa4161e969b; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['eb68d851-1fce-4a04-9ae8-88c37e6fd174', '6b77837c-c36b-4867-b003-b54c679e6ed0', '70c4c139-0bfa-4916-a84d-42c04ccc2b22', '2d6e1781-b0c0-469f-af4a-129bf6564d56', '5d89cbad-eebd-40d0-a759-062687ce1ada']; rows agent-rec-string-node-into-prompt.r1-a1.rows.json; raw capture sha256 eab6f497cbed85b5cfb6a73bc0ddd5df077c07a29b6113ef46e528b260f74932", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "cb0eb98e-919d-4082-bccf-0b72aaa8baea", - "message_id": "2ece02a3-a880-4632-b154-03462cd3aa64", - "exported_at": "2026-09-03T01:00:58Z" - } - }, - "workflow": { - "id": "38d9e9c7-e0fb-48f2-8f9a-391f7afba70c", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a PrimitiveStringMultiline node with the text 'a photo of a pier at sunset' and wire its output into the positive prompt's text input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "38d9e9c7-e0fb-48f2-8f9a-391f7afba70c" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 9, - "status": "success", - "tool_call_id": "toolu_018Brjqc2b4FPd4i4Vq2mX5G", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 609, - "status": "success", - "tool_call_id": "toolu_01JoeZB9LHeoCe9CUbpP6Gew", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 205, - "status": "success", - "tool_call_id": "toolu_01BkfK6eCmKdd1fm4jNL27Yw", - "tool_name": "show_node" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [80, 120], - "node": { - "id": 1896929324796827, - "pos": [80, 120], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "2c9a6b7dfaeb4051bb178630885ede81", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1896929324796827, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "4d6129913682457399beb255048d12a8", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a photo of a pier at sunset", - "widget": "value", - "node_id": 1896929324796827, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "56c9e65d3aea42c5b3a809cd0e5f8563", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 50487610085703, - "to_node": 6, - "to_slot": 1, - "from_node": 1896929324796827, - "from_slot": 0, - "link_type": "STRING", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 217, - "status": "success", - "tool_call_id": "toolu_017R8ze2T1Mnbxe2zonbMJ6a", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 186, - "status": "success", - "tool_call_id": "toolu_019LTDkGn5iwWv8CDTj9ie8v", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the \"Text to image\" tab now has a text node reading \"a photo of a pier at sunset\" feeding the positive prompt.\n\nOne thing to flag before you run it: the sampler's negative prompt and latent image inputs are both empty, so the workflow won't run as-is. Want me to add an empty latent (with a size) and a negative prompt node to complete it?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2197, - "cache_read_input_tokens": 231797, - "input_tokens": 12, - "output_tokens": 675, - "total_tokens": 687 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json deleted file mode 100644 index d5bf778cf0a7..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json +++ /dev/null @@ -1,266 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-text-only-answer", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running STANDALONE (AGENT_STANDALONE=true, SQLite) via scripts/dev-agent-integration.ts (PR 16643) at http://127.0.0.1:6207 (frames: ws http://127.0.0.1:6207/api/agent/events); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread e821b43c-f65e-4c4d-975c-76ecf4bfad9b; message 5d3c44f5-b880-42c9-8e4d-46b19f600c15; workflow edb2a004-8c29-48b8-bfd6-3c45effe82dd (seeded by throwaway turn 2e6c4667-5da1-42e3-a8f8-2506a5e6facc; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['29fcbef7-aaf9-4ea0-a34d-792a3f1b369a', 'a914f925-df3a-4f1b-84b4-3fe41a3048b4']; rows agent-rec-text-only-answer.r7-a3.agent.db; raw capture sha256 7ba7d66c074d6689145ef39b7108a7b5e8079b10d0fe4bb0425024f6c28881b0", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "e821b43c-f65e-4c4d-975c-76ecf4bfad9b", - "message_id": "5d3c44f5-b880-42c9-8e4d-46b19f600c15", - "exported_at": "2026-09-03T00:49:49Z" - } - }, - "workflow": { - "id": "edb2a004-8c29-48b8-bfd6-3c45effe82dd", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then tell me what this workflow does." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and read what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "edb2a004-8c29-48b8-bfd6-3c45effe82dd" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 2, - "status": "success", - "tool_call_id": "toolu_01Giap3Gbnx9T7ryXJ1z3cPY", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 649, - "status": "success", - "tool_call_id": "toolu_011QhzTDAPLJvdUS4ZLUNAUQ", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I'm on the \"Text to image\" tab now. It's a basic text-to-image setup built on Stable Diffusion XL: it takes a written description — currently \"a photo of a pier\" — and generates an image from it at 20 steps, then saves the result.\n\nOne thing worth flagging: it's not runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt (what to avoid) and no canvas size/blank image for it to start from. Want me to wire those in so you can run it?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 977, - "cache_read_input_tokens": 114270, - "input_tokens": 6, - "output_tokens": 356, - "total_tokens": 362 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json deleted file mode 100644 index 8e0a0c34f79a..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json +++ /dev/null @@ -1,267 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-tool-error", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running STANDALONE (AGENT_STANDALONE=true, SQLite) via scripts/dev-agent-integration.ts (PR 16643) at http://127.0.0.1:6207 (frames: ws http://127.0.0.1:6207/api/agent/events); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 94cec02e-de43-4d42-889f-b899d89a80f7; message a9b5cf3a-441e-4f0a-a6a3-9c3e2f164bef; workflow c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17 (seeded by throwaway turn fad6ef22-88ee-4df6-bd4d-b542b396ac4d; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['39a8b2a7-ddf6-4895-894d-592903605ff4', 'e9b6e0f3-77c8-4686-b62e-a4a54a2a8a27']; rows agent-rec-tool-error.r8-a1.agent.db; raw capture sha256 d1f8d5251494fcadd1ad1a06a524b9739470a398cc027fb24fb612dca69ce4d8", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "94cec02e-de43-4d42-889f-b899d89a80f7", - "message_id": "a9b5cf3a-441e-4f0a-a6a3-9c3e2f164bef", - "exported_at": "2026-09-03T00:57:22Z" - } - }, - "workflow": { - "id": "c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "c28a3f89-8c0e-4f0b-b53f-93e05dc6bc17" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 1, - "status": "success", - "tool_call_id": "toolu_01AeJR983cpdg2RLtC6keWCu", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 658, - "error_code": "node_not_found", - "status": "error", - "tool_call_id": "toolu_0196pmxhdJcMYPkE5PdiLc2t", - "tool_name": "add_node" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I'm now on the **Text to image** tab, but the node couldn't be added — there's no node class called \"DoesNotExist\" in the catalog, and there aren't any close matches to it either. Your canvas is unchanged.\n\nIf you tell me what that node was meant to do, I'll find the real one and wire it in." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 755, - "cache_read_input_tokens": 114297, - "input_tokens": 6, - "output_tokens": 288, - "total_tokens": 294 - } - } - } - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json deleted file mode 100644 index 88d386931e79..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json +++ /dev/null @@ -1,412 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-workflow-editing-05", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread d22255f9-36b3-4ec4-a99e-1a14f92f9b87; message ed9a4ce1-9fcb-4393-8b8b-900f60103eba; workflow a1474551-fbbf-428b-9d89-a41ac0031d1e (seeded by throwaway turn 31737c38-71ba-4866-a9cf-947df44fb49c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['fdfd9665-8318-403d-82a7-a6764d3be23b', '8ac84dde-0912-4fac-b4d2-24ff247442e6', '2e0155b2-2eed-4e72-8718-f56fe9561c1c', '8463f6d0-03dd-4b90-8ed0-8d33bd59723d', 'edfd4073-5440-4296-b075-c98ef506bcc6', '8c177691-44ea-4534-a1d8-378c58cfe4fe', '00922b53-e98d-4645-81db-1b3eee576273']; rows agent-rec-workflow-editing-05.e5-a1.rows.json; raw capture sha256 596f4c002229ad778ec76d5972d6a285255300e4b816a80b31bfaa1fc80fa215 Replaces the synthesized PoC fixture of the same case id: same eval prompt, recorded on the local agent behind the tab switch the replay binds on.", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "d22255f9-36b3-4ec4-a99e-1a14f92f9b87", - "message_id": "ed9a4ce1-9fcb-4393-8b8b-900f60103eba", - "exported_at": "2026-09-03T11:59:52Z" - } - }, - "workflow": { - "id": "a1474551-fbbf-428b-9d89-a41ac0031d1e", - "name": "Image edit", - "catalog": { - "types": { - "LoadImage": { - "widget_order": ["image", "upload"] - }, - "VAEEncode": { - "widget_order": [] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "ImageStitch": { - "widget_order": [ - "direction", - "match_image_size", - "spacing_width", - "spacing_color" - ] - }, - "PreviewImage": { - "widget_order": [] - } - } - }, - "seed": { - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [0, 0], - "size": [315, 314], - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [1] - }, - { - "name": "MASK", - "type": "MASK", - "links": [] - } - ], - "widgets_values": ["example.png", "image"] - }, - { - "id": 2, - "type": "VAEEncode", - "pos": [400, 0], - "size": [210, 46], - "inputs": [ - { - "name": "pixels", - "type": "IMAGE", - "link": 1 - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [2] - } - ], - "widgets_values": [] - }, - { - "id": 3, - "type": "KSampler", - "pos": [700, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": null - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": null - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 2 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [3] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1100, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 3 - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [4] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1400, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 4 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 1, 0, 2, 0, "IMAGE"], - [2, 2, 0, 3, 3, "LATENT"], - [3, 3, 0, 8, 0, "LATENT"], - [4, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch over and take a look at what's there." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Image edit", - "workflow_id": "a1474551-fbbf-428b-9d89-a41ac0031d1e" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 6, - "status": "success", - "tool_call_id": "toolu_01D5FjSPRaApDH2iwX9YriXZ", - "tool_name": "switch_tab" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 494, - "status": "success", - "tool_call_id": "toolu_01MWii7Rz882gmf1pTE5pVTh", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 191, - "status": "success", - "tool_call_id": "toolu_01SuNZEL9D1wQAhhY5h85Emm", - "tool_name": "find_nodes" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [1710, 280], - "node": { - "id": 1380502390045812, - "pos": [1710, 280], - "mode": 0, - "size": [240, 178], - "type": "ImageStitch", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "image1", - "type": "IMAGE" - }, - { - "link": null, - "name": "image2", - "type": "IMAGE" - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [] - } - ], - "properties": {}, - "widgets_values": ["right", true, 0, "white"] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "1ae5761c56814cd59b90d5b34915d10d", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1380502390045812, - "class_type": "ImageStitch", - "base_version": 1 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c1fbc369e27348d994ff6cdceaf0ccb3", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4338446108286126, - "to_node": 1380502390045812, - "to_slot": 0, - "from_node": 1, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "dff26a2ef5734d949b53391a7c61601b", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2548323397064582, - "to_node": 1380502390045812, - "to_slot": 1, - "from_node": 8, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 209, - "status": "success", - "tool_call_id": "toolu_01HVWPwEhd4nMt5CJzpvF9kn", - "tool_name": "apply_ops" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 200, - "status": "success", - "tool_call_id": "toolu_01NDFSRkMRCkQ4KqhzHcH7EN", - "tool_name": "list_slots" - } - } - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "2613854b97924cf394936f6425e1b0ff", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4049842834470829, - "to_node": 9, - "to_slot": 0, - "from_node": 1380502390045812, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 2 - } - ] - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 193, - "status": "success", - "tool_call_id": "toolu_01NEEaQs9cNn6qcJ1LgfZQCh", - "tool_name": "connect" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 197, - "status": "success", - "tool_call_id": "toolu_018ghX2WHT3XNERbndFewc2g", - "tool_name": "print_workflow" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done \u2014 on the \"Image edit\" tab, the saved picture is now the original and the edited result side by side, original on the left, matched to the same height with no gap between them.\n\nOne thing to flag: this workflow was already incomplete before I touched it \u2014 it has no model or VAE loader and no prompt text, so the sampler, encoder and decoder have nothing feeding them. It won't run until those are added. Want me to fill that in?" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 3222, - "cache_read_input_tokens": 312979, - "input_tokens": 16, - "output_tokens": 1175, - "total_tokens": 1191 - } - } - } - } - ] -} From f99f67981cf69e0139257fc0266b5e8b19a1194c Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:03:25 -0700 Subject: [PATCH 21/49] test(agent): add ten recorded agent conversations Recorded on the local agent stack (six graph-editing cases, a text-only answer, a tool error, and the two eval-suite prompts), exported with the conversation exporter unchanged, every frame carrying its recorded offset. The timing spec replays one of them at its recorded gaps. --- .../agent-l4-zimage-string-node-prompt.json | 550 ++++++++++++++++++ .../conversations/agent-rec-batched-ops.json | 380 ++++++++++++ .../agent-rec-refiner-between.json | 444 ++++++++++++++ .../agent-rec-replace-prompt-encoder.json | 407 +++++++++++++ .../agent-rec-second-prompt-encoder.json | 392 +++++++++++++ .../agent-rec-set-widget-existing.json | 342 +++++++++++ .../agent-rec-string-node-into-prompt.json | 373 ++++++++++++ .../agent-rec-text-only-answer.json | 275 +++++++++ .../conversations/agent-rec-tool-error.json | 276 +++++++++ .../agent-workflow-editing-05.json | 420 +++++++++++++ .../agent/agentConversationTiming.spec.ts | 31 + 11 files changed, 3890 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json create mode 100644 browser_tests/tests/agent/agentConversationTiming.spec.ts diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json new file mode 100644 index 000000000000..b759d1ac505f --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -0,0 +1,550 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-l4-zimage-string-node-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 91a905f3-b619-44e3-a6e5-5a047d8d0066; message 4293701d-cd6f-41cf-b03d-ea73df6af846; workflow 891d1304-09ae-4d51-9bea-aaa4756822d4 (seeded by throwaway turn 62bbe9f8-bcb1-4a84-9275-97384a9482f2; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ea9a0cdc-498e-4c8e-ba09-e487cbf02e12', '3893c0d2-605b-43bc-8d77-ff249b267087', '3ae90aa2-14e3-4001-a3c9-90eadef4ad86', '5656b627-7364-4759-8471-f0dedbf3b125', '7e4bd8e9-223d-4a79-985e-7bd262e8b5b9', 'c7f90263-0a57-4a48-8573-75bdb1746a6e', 'acc3faea-b08b-4e4d-8f4b-cb999a8cc6d5', 'a928d523-16e8-4d26-81ad-1b7cdd884c68', '42abc204-3da4-410c-bf83-1b206d567401']; rows agent-l4-zimage-string-node-prompt.t-a1.rows.json; raw capture sha256 f4a2076ae60f8e5eefc7799fd0a14d58d4c3829796ff89f88a0e4548c952b3fe", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "91a905f3-b619-44e3-a6e5-5a047d8d0066", + "message_id": "4293701d-cd6f-41cf-b03d-ea73df6af846", + "exported_at": "2026-09-03T12:40:57Z" + } + }, + "workflow": { + "id": "891d1304-09ae-4d51-9bea-aaa4756822d4", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look at that workflow." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "891d1304-09ae-4d51-9bea-aaa4756822d4" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01KZvQdg1MdhKgbWQsvScSpN", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 774, + "status": "success", + "tool_call_id": "toolu_015MGXQKWWD64ZWnv7jWQNLw", + "tool_name": "print_workflow" + } + }, + "at_ms": 2809 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 308, + "status": "success", + "tool_call_id": "toolu_015ngjjFSSFFFRrsQvJwbnut", + "tool_name": "find_nodes" + } + }, + "at_ms": 7587 + }, + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "The prompt node is easy — but this graph is also missing a couple of connections it needs to run, so I'll complete those too." + } + }, + "at_ms": 13907 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [395, 280], + "node": { + "id": 3876406316923056, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b41f1bc5b13f44efb10bb51847859813", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876406316923056, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "90d12d67f1a54617af277c333f9880be", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 3876406316923056, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c0b8ec72e8d64de7bcf9b45125f6c207", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1887935066439213, + "to_node": 6, + "to_slot": 1, + "from_node": 3876406316923056, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3880387162517873, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5da691f2d07486aa354c5ee7c7fb6f1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3880387162517873, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "d02e085ef78d41109f09ed846715fdfa", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "", + "widget": "text", + "node_id": 3880387162517873, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "54508721c33e478b94db088a93568f4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2636276550287764, + "to_node": 3880387162517873, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "98f104d28a664643b45551a76f4b6733", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2833535472085255, + "to_node": 3, + "to_slot": 2, + "from_node": 3880387162517873, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 3876084544359242, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "9e977850c86447acb7359b7a9b484a83", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876084544359242, + "class_type": "EmptyLatentImage", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c9e0686dff2245fa90d9e720c0133928", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2211554611670965, + "to_node": 3, + "to_slot": 3, + "from_node": 3876084544359242, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + } + ], + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 447, + "status": "success", + "tool_call_id": "toolu_0126og8icdp1EUnSmtrcSZv6", + "tool_name": "apply_ops" + } + }, + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 249, + "status": "error", + "tool_call_id": "toolu_01RQTp9JXwFc1AdgmAKBiMYm", + "tool_name": "validate" + } + }, + "at_ms": 16172 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 232, + "status": "success", + "tool_call_id": "toolu_01Lq8eZseF7AhmTd9UuxvcNb", + "tool_name": "show_node" + } + }, + "at_ms": 19599 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 248, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01VcWs7EVFuk18ct7nX9nEoi", + "tool_name": "list_model_picks" + } + }, + "at_ms": 24061 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 887, + "status": "success", + "tool_call_id": "toolu_01AB6XXLQrdF3EFbDRV9GtSx", + "tool_name": "list_generate_models" + } + }, + "at_ms": 28706 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 324, + "status": "success", + "tool_call_id": "toolu_01U2bR9WL8Pc5H35X2JERBwr", + "tool_name": "print_workflow" + } + }, + "at_ms": 38928 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all.\n\nI couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up.\n\nTwo ways forward, your call:\n- Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct.\n- Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first." + } + }, + "at_ms": 44823 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 4467, + "cache_read_input_tokens": 395126, + "input_tokens": 20, + "output_tokens": 2467, + "total_tokens": 2487 + } + } + }, + "at_ms": 44827 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json new file mode 100644 index 000000000000..0a9624b06b65 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json @@ -0,0 +1,380 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-batched-ops", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread a6fd73be-e957-45b7-a5e0-5edd091670fb; message 9f88da92-5074-4091-850b-7cbc65005cd1; workflow e539bb63-bc5d-4f97-b918-d85862c0f591 (seeded by throwaway turn 365de61d-2131-4966-af0b-fe7ccda288bf; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ae918468-c270-4d05-b0ef-31198a63cf0e', '069a03fa-3094-4fa4-821d-797672439db7', '044e431e-059e-45b1-b82d-45e25a92f2bf']; rows agent-rec-batched-ops.t-a1.rows.json; raw capture sha256 92757967c82950945ea1a183dbcfcb8d111ae768b3ea16a72348bb0e216b709e", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "a6fd73be-e957-45b7-a5e0-5edd091670fb", + "message_id": "9f88da92-5074-4091-850b-7cbc65005cd1", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "e539bb63-bc5d-4f97-b918-d85862c0f591", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "e539bb63-bc5d-4f97-b918-d85862c0f591" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 7, + "status": "success", + "tool_call_id": "toolu_01UJ5hWx9Rxkvwon9F7nMGaK", + "tool_name": "switch_tab" + } + }, + "at_ms": 2 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_011QfP8raZRXJbK5WZYpKTkv", + "tool_name": "print_workflow" + } + }, + "at_ms": 2139 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2559261621971657, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7a130c22fe934bc98e2ea2fed6bd10ad", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2559261621971657, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 406], + "node": { + "id": 2214070360209981, + "pos": [715, 406], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "00b500c587744366b85d5c71ddbc9fdf", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2214070360209981, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e9e5c529eb574496a516dd6014ffd5e6", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 879068618000012, + "to_node": 2559261621971657, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c7f256bd0cd34e92830062aeea907b4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3832075311322831, + "to_node": 2214070360209981, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + } + ], + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01PhJ8AyewzRU6PHk7aFGfbC", + "tool_name": "apply_ops" + } + }, + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt." + } + }, + "at_ms": 7784 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1437, + "cache_read_input_tokens": 153211, + "input_tokens": 8, + "output_tokens": 574, + "total_tokens": 582 + } + } + }, + "at_ms": 7786 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json new file mode 100644 index 000000000000..c69b9c4fbc17 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json @@ -0,0 +1,444 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-refiner-between", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a; message b363781f-6c9a-4087-9190-d478781bbcd2; workflow b7d1e09f-682a-48f5-a388-d74a33ad5184 (seeded by throwaway turn 52d56bcf-7eb0-4131-a287-223e0f15e2e9; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2f4c246a-f0c7-4ca4-ad48-814bc2fe9ab6', '9727d8d7-19f4-48eb-98e6-5c054d3d6cf0', '22b43a2d-5fe8-49c0-8339-e2b3b52bf184', '33953182-42ff-46a6-a0d5-313c5a55d630', '349abb83-7fed-424e-8fb5-dc94e412aa90']; rows agent-rec-refiner-between.t-a1.rows.json; raw capture sha256 242b1c822b8b7697bdcdf1448030181ef0bb1049806870e9075077ca44897768", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a", + "message_id": "b363781f-6c9a-4087-9190-d478781bbcd2", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "b7d1e09f-682a-48f5-a388-d74a33ad5184", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b7d1e09f-682a-48f5-a388-d74a33ad5184" + } + }, + "at_ms": 5 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 5, + "status": "success", + "tool_call_id": "toolu_01Bq7cRyMamrEnmo4yHuMphE", + "tool_name": "switch_tab" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 747, + "status": "success", + "tool_call_id": "toolu_01FEPdyTv3FtKtL8S6zTKj33", + "tool_name": "print_workflow" + } + }, + "at_ms": 3053 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1565, 280], + "node": { + "id": 2092962753103866, + "pos": [1565, 280], + "mode": 0, + "size": [240, 290], + "type": "KSampler", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "model", + "type": "MODEL" + }, + { + "link": null, + "name": "positive", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "negative", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "latent_image", + "type": "LATENT" + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a3b5b79791364f75b1d89391a7a87564", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2092962753103866, + "class_type": "KSampler", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5d9733c860c40798848bf92d649988f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4414751122447181, + "to_node": 2092962753103866, + "to_slot": 3, + "from_node": 3, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ced254ebc4a417589c32b5a1797452d", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 863234800411381, + "to_node": 2092962753103866, + "to_slot": 0, + "from_node": 4, + "from_slot": 0, + "link_type": "MODEL", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "57585fd740934c15835cef60c216c84c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3220756471207314, + "to_node": 2092962753103866, + "to_slot": 1, + "from_node": 6, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "bb113cee68314103861361151c93ad36", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 619699595909791, + "to_node": 8, + "to_slot": 0, + "from_node": 2092962753103866, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "set_widget", + "old": 1, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8ba837eb610e4e0db4561b789bb44953", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 0.5, + "widget": "denoise", + "node_id": 2092962753103866, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e83bac70162244fa96c7d4633dd5fccc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 20, + "widget": "steps", + "node_id": 2092962753103866, + "base_version": 2 + } + ], + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 198, + "status": "success", + "tool_call_id": "toolu_01LqYJ1bi41MTaeDXdzC9A9j", + "tool_name": "apply_ops" + } + }, + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 379, + "status": "success", + "tool_call_id": "toolu_01DAfeiUo97WETr7TRCrv4Cp", + "tool_name": "print_workflow" + } + }, + "at_ms": 14468 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_01G8xCCA9NuoEur1n8RySCWK", + "tool_name": "validate" + } + }, + "at_ms": 19339 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node.\n\nTwo things were already unfinished in this graph before I touched it, and they'll block a run:\n\n- Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler.\n- The checkpoint it points at isn't installed here, so no model file is actually available.\n\nI set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?" + } + }, + "at_ms": 29476 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2906, + "cache_read_input_tokens": 232638, + "input_tokens": 12, + "output_tokens": 1422, + "total_tokens": 1434 + } + } + }, + "at_ms": 29482 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json new file mode 100644 index 000000000000..f25d041a11d0 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json @@ -0,0 +1,407 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-replace-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2ce25f4b-d581-4348-ab9c-3f4170222f6f; message 37504547-e90d-4c14-91c7-b8f8fb65a0eb; workflow 5a6b35c4-8902-479d-87ab-8a9a07f08b90 (seeded by throwaway turn 3add9fbe-1dbb-4d5f-b255-d0fde410ed5e; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d6e50c4a-5ef2-46bb-a48d-46473a36e13f', '6471d9bc-2e6f-41f1-b2c9-b417e7b9ecea', '3bda2337-3236-4585-9803-f8cbc1625d0a', '7cba8d9d-ebe4-481f-82e9-354377d2826e', '9d9678dd-2f0c-47eb-98da-0ec3720ff91d']; rows agent-rec-replace-prompt-encoder.t-a1.rows.json; raw capture sha256 eabe622a684331aa98da2a1a797cf0f761e3c866bcce8feaf2e1083a6fb7ff26", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "2ce25f4b-d581-4348-ab9c-3f4170222f6f", + "message_id": "37504547-e90d-4c14-91c7-b8f8fb65a0eb", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 10, + "status": "success", + "tool_call_id": "toolu_01PbXSHPoAUnHhX2dqzJXiLB", + "tool_name": "switch_tab" + } + }, + "at_ms": 11 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 295, + "status": "success", + "tool_call_id": "toolu_01C2g2RACFUykeibS1je1Tbh", + "tool_name": "print_workflow" + } + }, + "at_ms": 3673 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 4181654812796082, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4c4d739392f14272939cf46a335652c8", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4181654812796082, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "f1cd3aef4b984b04a9611a53f69763ba", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3320191292493608, + "to_node": 4181654812796082, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4f4de6968f124e23a3114f8c88a1a766", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2686273519871711, + "to_node": 3, + "to_slot": 1, + "from_node": 4181654812796082, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "279c55a82f09416f9fa68e47ee11fafc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier", + "widget": "text", + "node_id": 4181654812796082, + "base_version": 2 + } + ], + "at_ms": 10451 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 223, + "status": "success", + "tool_call_id": "toolu_018Ao8cDcxmDBaz9qV7PWpGx", + "tool_name": "apply_ops" + } + }, + "at_ms": 10451 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b141d8503ee54cde94d3a4e3f6b5204f", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 6, + "base_version": 3, + "removed_links": [1] + } + ], + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 202, + "status": "success", + "tool_call_id": "toolu_01QkN5N4e8fkpBrT9aWbb4aK", + "tool_name": "delete_node" + } + }, + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01QUTSXXBfU7oShpWRkxtLzD", + "tool_name": "print_workflow" + } + }, + "at_ms": 18022 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm on the \"Text to image\" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler's positive input. I carried over the same prompt text (\"a photo of a pier\").\n\nOne thing unrelated to this swap: the sampler's negative prompt and empty latent inputs were already unconnected before I started, so the graph won't run until those are filled in. Want me to add them?" + } + }, + "at_ms": 22947 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2216, + "cache_read_input_tokens": 232121, + "input_tokens": 12, + "output_tokens": 885, + "total_tokens": 897 + } + } + }, + "at_ms": 22951 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json new file mode 100644 index 000000000000..ec5624d316c2 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json @@ -0,0 +1,392 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-second-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 654dc7e4-c2e2-4b60-9e28-cfd88839a85c; message 72291d49-42f3-4b78-8a73-80e5cb1b9d2d; workflow b2c94bcb-c02f-4456-99ce-65708c66b341 (seeded by throwaway turn c575f8f1-ee42-45fc-99c0-c285b7c3739a; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['023ee47a-fc32-4bec-8e1f-42a6327bcc0f', 'f4b41fff-7fec-41fd-bedd-a58947686567', '165127e2-793e-4ca0-bc3f-62880660f572', '11ff34fb-553c-4466-95f7-ef010c8983a9', 'a7ee8b67-dca2-4293-9c34-993983f338b6']; rows agent-rec-second-prompt-encoder.t-a1.rows.json; raw capture sha256 20ffafacee4786b5fe000c1d7cd73c23222b2d53309664eef45085c1663e01ed", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "654dc7e4-c2e2-4b60-9e28-cfd88839a85c", + "message_id": "72291d49-42f3-4b78-8a73-80e5cb1b9d2d", + "exported_at": "2026-09-03T12:40:54Z" + } + }, + "workflow": { + "id": "b2c94bcb-c02f-4456-99ce-65708c66b341", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and wire its conditioning into the KSampler's negative input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b2c94bcb-c02f-4456-99ce-65708c66b341" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_01HDMTFUx2kkyMo87wtNzmze", + "tool_name": "switch_tab" + } + }, + "at_ms": 11 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 290, + "status": "success", + "tool_call_id": "toolu_01RajshxPyUhr7ZyCEJ5ihFS", + "tool_name": "print_workflow" + } + }, + "at_ms": 2157 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2143974170219146, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "f1d3ff4affc84a7bb0e2945349e22c8b", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2143974170219146, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "d9c4827370a24567975399a1b6c842f9", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1085915564088321, + "to_node": 2143974170219146, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "fdfd2ededd164148bd7138f8a894d333", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3735142525577434, + "to_node": 3, + "to_slot": 2, + "from_node": 2143974170219146, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c872f1a5e80447768012eab52a6a384f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "blurry, low quality, watermark", + "widget": "text", + "node_id": 2143974170219146, + "base_version": 2 + } + ], + "at_ms": 7248 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 270, + "status": "success", + "tool_call_id": "toolu_013BwRtRP1QkTwF6RRQhkwsz", + "tool_name": "apply_ops" + } + }, + "at_ms": 7248 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01EmHWYrRmyiQ1tjXGHAiuSc", + "tool_name": "print_workflow" + } + }, + "at_ms": 10148 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 196, + "status": "error", + "tool_call_id": "toolu_015XiAA4HvEjNkfWtwvdhGPY", + "tool_name": "validate" + } + }, + "at_ms": 15164 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the negative prompt encoder is on your \"Text to image\" canvas, taking CLIP from the checkpoint loader and feeding the sampler's negative input (I seeded it with \"blurry, low quality, \" — change it to whatever you like).\n\nTwo things that were already there before this edit and will stop a run: the sampler has nothing wired into its latent input, and the checkpoint it points at isn't installed on this server. Want me to add the empty-latent node and switch to an available checkpoint?" + } + }, + "at_ms": 21622 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2527, + "cache_read_input_tokens": 232384, + "input_tokens": 12, + "output_tokens": 1038, + "total_tokens": 1050 + } + } + }, + "at_ms": 21627 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json new file mode 100644 index 000000000000..b73aebeaaa99 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json @@ -0,0 +1,342 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-set-widget-existing", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread b5d11187-d40e-4be1-aeab-39869516773b; message 19230873-0f07-4f85-a0ae-e9f1d740cb3a; workflow 5733b17e-39f4-40f5-aa1c-468a4f355b55 (seeded by throwaway turn d00f4278-8e50-4671-a49b-8a237f1797b4; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2383f8e0-be34-4bb1-88c5-bbe0fd34ee95', '2b9744df-c52a-4b45-b0e8-96b64642075d', 'c93c9e49-1d3d-4507-9c3d-481a0910d2c4', 'c2b5e940-a8f3-49eb-b043-e6f8360c686c', 'b538294c-6add-4ebe-b351-c09cce102a0f']; rows agent-rec-set-widget-existing.t3-a1.rows.json; raw capture sha256 6a5a92d256bb83cfd9fb95be1f182300da2361bb15a4325ce2365de6f79ad0a9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "b5d11187-d40e-4be1-aeab-39869516773b", + "message_id": "19230873-0f07-4f85-a0ae-e9f1d740cb3a", + "exported_at": "2026-09-03T12:38:52Z" + } + }, + "workflow": { + "id": "5733b17e-39f4-40f5-aa1c-468a4f355b55", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5733b17e-39f4-40f5-aa1c-468a4f355b55" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 12, + "status": "success", + "tool_call_id": "toolu_0127MABi8i6cpznwzfMSk2b6", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 498, + "status": "success", + "tool_call_id": "toolu_01DTMkDXHKWELHfKUTXUynfK", + "tool_name": "ls_nodes" + } + }, + "at_ms": 3903 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 425, + "status": "success", + "tool_call_id": "toolu_018Ck46iqTkvLjT12cDdcyH6", + "tool_name": "list_slots" + } + }, + "at_ms": 9834 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "5ee9143999c148439b6fc848bc8d309a", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 30, + "widget": "steps", + "node_id": 3, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 7, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ad6e06be4f84086bec999cfc5177816", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 5, + "widget": "cfg", + "node_id": 3, + "base_version": 2 + } + ], + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 253, + "status": "success", + "tool_call_id": "toolu_01BaTP7o2iAhJbBjQ9tChoEG", + "tool_name": "apply_ops" + } + }, + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01MLp2wvSBojXfwPYKqT7dvB", + "tool_name": "list_slots" + } + }, + "at_ms": 21478 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "You're now on the \"Text to image\" tab, with steps at 30 and cfg at 5. Nothing else was touched." + } + }, + "at_ms": 23927 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1549, + "cache_read_input_tokens": 230832, + "input_tokens": 12, + "output_tokens": 509, + "total_tokens": 521 + } + } + }, + "at_ms": 23931 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json new file mode 100644 index 000000000000..25ebb7e0d221 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json @@ -0,0 +1,373 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-string-node-into-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2ff49d22-e6e9-4fdf-9214-492e06f77dd5; message fde9fd6d-c9a8-4ed4-a524-5f993f7b5cbf; workflow 75ffed90-1d46-4b37-9bbf-abdf3d270811 (seeded by throwaway turn b53fedff-60c8-4ae1-bb6d-b68f92d7ee97; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['3ce12d96-f9cb-45a1-b33a-075387f0cb28', '66a65cf7-228e-41ee-896a-165f96364a91', '25f41c6c-ff4b-4d20-a539-a21550ec4897', 'fc46022e-836e-44c7-9e4a-8a54653bda73', '60494f00-0fc3-4dde-8dd5-ea8356e77416']; rows agent-rec-string-node-into-prompt.t-a1.rows.json; raw capture sha256 94453e3cee7f5b7765266bbee7bcf45f02e53ca11e889a6f3f28376b859c132a", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "2ff49d22-e6e9-4fdf-9214-492e06f77dd5", + "message_id": "fde9fd6d-c9a8-4ed4-a524-5f993f7b5cbf", + "exported_at": "2026-09-03T12:40:53Z" + } + }, + "workflow": { + "id": "75ffed90-1d46-4b37-9bbf-abdf3d270811", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a PrimitiveStringMultiline node with the text 'a photo of a pier at sunset' and wire its output into the positive prompt's text input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "75ffed90-1d46-4b37-9bbf-abdf3d270811" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_012df4k3MtbwuCzgUZssmJAe", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 623, + "status": "success", + "tool_call_id": "toolu_01Mo1YBLCyoku5dm2D7D6LTH", + "tool_name": "print_workflow" + } + }, + "at_ms": 2586 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 358, + "status": "success", + "tool_call_id": "toolu_01XxtrwYzdvG6VrUXaY2j33W", + "tool_name": "show_node" + } + }, + "at_ms": 7323 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [80, 120], + "node": { + "id": 4111844109938098, + "pos": [80, 120], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7b4e35b41b8b4d97aebd5b81e90b97b4", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4111844109938098, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "cf0352bcf66f4a07a81f4ce09133f1e6", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier at sunset", + "widget": "value", + "node_id": 4111844109938098, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b3ace060146b432c996e3435be9f05e7", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 883895853831334, + "to_node": 6, + "to_slot": 1, + "from_node": 4111844109938098, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + } + ], + "at_ms": 10862 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 216, + "status": "success", + "tool_call_id": "toolu_01VG1kKqa1wQJqK4828x4WnX", + "tool_name": "apply_ops" + } + }, + "at_ms": 10862 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 232, + "status": "success", + "tool_call_id": "toolu_018Eu9J62defBJuvMyb1PbEH", + "tool_name": "print_workflow" + } + }, + "at_ms": 13023 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the text \"a photo of a pier at sunset\" now feeds the positive prompt in your **Text to image** tab.\n\nHeads up: that graph was already missing a negative prompt and an empty latent going into the sampler, so it won't run as-is. Want me to add those?" + } + }, + "at_ms": 17136 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2182, + "cache_read_input_tokens": 231753, + "input_tokens": 12, + "output_tokens": 705, + "total_tokens": 717 + } + } + }, + "at_ms": 17141 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json new file mode 100644 index 000000000000..ed3c3c5b48ed --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json @@ -0,0 +1,275 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-text-only-answer", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread f36ef047-82b4-4f2b-b480-6df821df46da; message 7de6039c-c1b4-437c-871c-2f1fc67e1750; workflow d762837d-c64d-4829-9aba-05e71b64adc3 (seeded by throwaway turn c3727b7b-569e-4eda-983a-7ccfffbf4496; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['7622ee7b-b97a-4899-8931-99488b82bee5', '5547227c-9bef-4e95-9bd1-053fb658e7b4']; rows agent-rec-text-only-answer.t-a1.rows.json; raw capture sha256 24906bcf7826bc7860dd7b109ce8d243a1c6322f56a646ea53e5645cc795c387", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "f36ef047-82b4-4f2b-b480-6df821df46da", + "message_id": "7de6039c-c1b4-437c-871c-2f1fc67e1750", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "d762837d-c64d-4829-9aba-05e71b64adc3", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then tell me what this workflow does." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d762837d-c64d-4829-9aba-05e71b64adc3" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_013WLWbbrNwTqV9UguY6rEu6", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_01PjsdjzNXWZ8oDYHQAbjo5o", + "tool_name": "print_workflow" + } + }, + "at_ms": 2364 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now.\n\nIt's a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently \"a photo of a pier\" — into guidance, generates an image over 20 steps, and saves the result.\n\nOne thing to flag: it isn't runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?" + } + }, + "at_ms": 6244 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 960, + "cache_read_input_tokens": 114249, + "input_tokens": 6, + "output_tokens": 349, + "total_tokens": 355 + } + } + }, + "at_ms": 6246 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json new file mode 100644 index 000000000000..4b2c416321e5 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json @@ -0,0 +1,276 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-tool-error", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386; message a5ec5ead-1909-4322-9d62-edbb03a3ded9; workflow 5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5 (seeded by throwaway turn 788de1fc-829b-4636-b715-2879479effbc; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['39a22dd2-4aaa-4d7d-9505-779ab6386ee8', 'aeb6b2f3-8d0c-4131-a913-144322ca42c8']; rows agent-rec-tool-error.t-a1.rows.json; raw capture sha256 83bd2c2c30c00883df39527e5f482deccc5e10c34a0d6554c34bbff8d725c0e6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386", + "message_id": "a5ec5ead-1909-4322-9d62-edbb03a3ded9", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01YWm16Ejys8FJBofYwL7k9E", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 346, + "error_code": "node_not_found", + "status": "error", + "tool_call_id": "toolu_01XqvtmCXKboog1UTZ3MGNJR", + "tool_name": "add_node" + } + }, + "at_ms": 2654 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm now on the \"Text to image\" tab, but there's no node called \"DoesNotExist\" — nothing was added, and the canvas is unchanged.\n\nIf you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I'll find the real one and wire it in." + } + }, + "at_ms": 5701 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 752, + "cache_read_input_tokens": 114291, + "input_tokens": 6, + "output_tokens": 295, + "total_tokens": 301 + } + } + }, + "at_ms": 5703 + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json new file mode 100644 index 000000000000..e9722356e35d --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -0,0 +1,420 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-workflow-editing-05", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3; message b199d9ab-1800-49b2-8a43-458cda7f104d; workflow f47e55ab-2cd5-414e-90d3-23a5fc7eee22 (seeded by throwaway turn 3d61dd80-dd03-4957-8506-734392c9583c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['6206d0ef-f781-41db-81bd-4138950ccd2e', 'fb873a4b-59b4-4088-ae32-f43e0a7382c5', '473ea6be-f9c3-4a75-8776-0e7823cd5054', 'e6ec2a79-8f0b-408a-b01e-f9362ae8fde4', '47d7581e-2d55-49ea-ac3e-1d6520d80454', 'abd6c054-6b5b-49a2-8633-881078e9b07c', '3f83a246-0181-4d1a-b2b5-c1fcfac38fe0']; rows agent-workflow-editing-05.t-a1.rows.json; raw capture sha256 982353ccbf5d92ca70a7857eb7da662aa2d19b0fcebb27181a71c3a95b762658", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3", + "message_id": "b199d9ab-1800-49b2-8a43-458cda7f104d", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22", + "name": "Image edit", + "catalog": { + "types": { + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "VAEEncode": { + "widget_order": [] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "ImageStitch": { + "widget_order": [ + "direction", + "match_image_size", + "spacing_width", + "spacing_color" + ] + }, + "PreviewImage": { + "widget_order": [] + } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [1] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "VAEEncode", + "pos": [400, 0], + "size": [210, 46], + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 1 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2] + } + ], + "widgets_values": [] + }, + { + "id": 3, + "type": "KSampler", + "pos": [700, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [3] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1100, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 3 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [4] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1400, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 1, 0, 2, 0, "IMAGE"], + [2, 2, 0, 3, 3, "LATENT"], + [3, 3, 0, 8, 0, "LATENT"], + [4, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Image edit", + "workflow_id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_01RMDkomGUUpDL8wixWq9Fhe", + "tool_name": "switch_tab" + } + }, + "at_ms": 14 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 366, + "status": "success", + "tool_call_id": "toolu_01JECLyu71YJ1niLhepMB63u", + "tool_name": "print_workflow" + } + }, + "at_ms": 2905 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 194, + "status": "success", + "tool_call_id": "toolu_01Sttwp3yTjiF6LdrL5GdmVy", + "tool_name": "find_nodes" + } + }, + "at_ms": 7034 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1710, 280], + "node": { + "id": 1177948620476144, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], + "type": "ImageStitch", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, + "widgets_values": ["right", true, 0, "white"] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8e44f8dcf31e46f6bf6d487c1bec2676", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1177948620476144, + "class_type": "ImageStitch", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "792213bf2f824cfd8866f157117795a4", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3298141227647238, + "to_node": 1177948620476144, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7c91fdf62a4d437dac4ada8d3ae13661", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2572689854678654, + "to_node": 1177948620476144, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "3eaf0c9e52cb4f0994962ff4b1e341ff", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3588278739910384, + "to_node": 9, + "to_slot": 0, + "from_node": 1177948620476144, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + } + ], + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 208, + "status": "success", + "tool_call_id": "toolu_01DNrpeJQDjW2v32xme5eyDq", + "tool_name": "apply_ops" + } + }, + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 210, + "status": "success", + "tool_call_id": "toolu_01VFMK15TdDdT4zFK3w8bRqE", + "tool_name": "list_slots" + } + }, + "at_ms": 20740 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 206, + "error_code": "conflict", + "status": "error", + "tool_call_id": "toolu_01GZUZLG9RtoeGym25Hy5p5P", + "tool_name": "set_widget" + } + }, + "at_ms": 25170 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 216, + "status": "error", + "tool_call_id": "toolu_01FCgU1tuBtusNgYiPMRaUQL", + "tool_name": "validate" + } + }, + "at_ms": 29613 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved.\n\nTwo things worth knowing:\n\n- The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider.\n- The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?" + } + }, + "at_ms": 38675 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 3425, + "cache_read_input_tokens": 313028, + "input_tokens": 16, + "output_tokens": 1658, + "total_tokens": 1674 + } + } + }, + "at_ms": 38677 + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts new file mode 100644 index 000000000000..1faad07ab9aa --- /dev/null +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -0,0 +1,31 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { + expectedGraphSnapshot, + recordedSpanMs +} from '@e2e/fixtures/data/agent/agentConversationExpectations' + +test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { + test.use({ + conversationCase: 'agent-rec-set-widget-existing', + replayTiming: 'recorded' + }) + + test('replays the frames at their recorded offsets', async ({ + agentConversation + }) => { + test.setTimeout(120_000) + const span = recordedSpanMs(agentConversation.conversation) + expect(span, 'the fixture carries at_ms offsets').toBeDefined() + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(expectedGraphSnapshot(agentConversation.conversation)) + }) +}) From 5c147377caf72167dc7e762e3bc8c1268eec39b1 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:11:44 -0700 Subject: [PATCH 22/49] test(agent): read the expected graph from the applier, not a re-implementation The replay's expected graph now comes from readGraph on the host doc the pinned applier built, so link replacement and node deletion follow the real rules instead of a copy of them. Block comments in the carried harness cut to one-line whys. --- .../fixtures/agentConversationFixture.ts | 57 ++--- .../fixtures/data/agent/agentConversation.ts | 13 +- .../agent/agentConversationExpectations.ts | 220 +++++------------- .../agent/agentConversationReplay.spec.ts | 8 +- scripts/agentConversationCapture.ts | 8 +- 5 files changed, 79 insertions(+), 227 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 4c8e453ec540..13db6cdc3444 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -2,7 +2,7 @@ import type { Locator, Page, WebSocketRoute } from '@playwright/test' import { expect } from '@playwright/test' import { applyOps, mint, readGraph } from '@comfyorg/comfy-multi-player' -import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import type { GraphSnapshot,WidgetCatalog,WorkflowJSON } from '@comfyorg/comfy-multi-player' import * as Y from 'yjs' import enMessages from '@/locales/en/main.json' with { type: 'json' } @@ -51,12 +51,6 @@ function fromBase64(value: string): Uint8Array { return new Uint8Array(Buffer.from(value, 'base64')) } -/** - * The doc host's side of the CRDT channel, in-process: the seed workflow is - * minted with the same pinned applier the real host runs, each `graph_ops` - * entry is applied to it, and the resulting Yjs delta is what the follower - * receives as a `doc_update` frame. - */ class HostDoc { private readonly doc: Y.Doc private seq = 1 @@ -69,8 +63,8 @@ class HostDoc { this.doc = mint(seed, catalog) } - nodeIds(): Set { - return new Set(Object.keys(readGraph(this.doc).nodes)) + graph(): GraphSnapshot { + return readGraph(this.doc) } subscribed(): DocFrame { @@ -132,26 +126,14 @@ class HostDoc { interface GraphNodeSnapshot { id: string title: string - /** Connected state of each rendered input slot, in slot order. */ inputs: boolean[] - /** Connected state of each rendered output slot, in slot order. */ outputs: boolean[] } -/** - * Drives one recorded user prompt -> agent response exchange through the - * real agent panel: the composer posts the prompt, the mocked backend acks - * it, and the recorded response is replayed over the routed `/ws` socket as - * chat frames plus doc-host `doc_update` frames the CRDT follower applies to - * the canvas. - */ +// Runs one recorded prompt/response through the real panel over a routed /ws socket. class AgentConversationHarness { readonly postedMessages: string[] = [] - /** - * Every frame the CLIENT sent on /ws, in order (BE-11470 A4): human-op - * minting is observable only on this side of the socket, so regression - * specs assert against these rather than app internals. - */ + // Human-op minting is observable only on the client side of the socket. readonly clientFrames: { type?: unknown; data?: unknown }[] = [] readonly panel: Locator readonly ack: AgentTurnAccepted @@ -180,10 +162,7 @@ class AgentConversationHarness { async boot(agentFlag: boolean): Promise { await this.mockAgentApi() - // The app may reopen `/ws` (the connect awaits an auth token), and the - // follower only re-drives a pending subscribe on a `status` frame, which - // the real server sends on every connect. So every routed socket becomes - // the live one and gets that frame. + // The follower re-drives a pending subscribe only on a status frame, which every real connect sends. await this.page.routeWebSocket(/\/ws/, (socket) => { this.socket = socket socket.onMessage((raw) => this.onClientFrame(raw)) @@ -198,8 +177,7 @@ class AgentConversationHarness { ) }) await bootAgentApp(this.page, agentFlag, { - // Follower edits land in the ECS stores, which only the Vue node - // renderer projects onto the canvas. + // Only the Vue node renderer projects follower edits onto the canvas. settings: { 'Comfy.VueNodes.Enabled': true } }) @@ -255,16 +233,13 @@ class AgentConversationHarness { ).toHaveCount(0) } - /** - * The agent's workflow document as rendered on the canvas: every node the - * host doc owns, with its slot connectivity. The tab the agent opens is - * expected to be blank (AgentPanelRoot mints it from `blankGraph`); the - * doc-id filter is defensive, so a stray template node cannot pin the - * template here. Colliding ids are the real hazard: a template link into a - * doc node survives the follower's reconcile and shows up as connectivity. - */ + // Doc-id filter: a stray template node must not pin the template here. + hostGraph(): GraphSnapshot { + return this.host.graph() + } + async graphSnapshot(): Promise { - const docNodeIds = this.host.nodeIds() + const docNodeIds = new Set(Object.keys(this.host.graph().nodes)) const snapshot = await this.page .locator('[data-node-id]') .evaluateAll((nodes) => @@ -339,7 +314,6 @@ class AgentConversationHarness { this.socket.send(JSON.stringify(frame)) } - /** The ops of every outbound `doc_ops` frame, flattened in send order. */ outboundOps(): { op?: unknown }[] { return this.clientFrames .filter((frame) => frame.type === 'doc_ops') @@ -355,9 +329,7 @@ class AgentConversationHarness { const { type, data } = frame as { type?: unknown; data?: unknown } this.clientFrames.push({ type, data }) if (type === 'doc_ops' && typeof data === 'object' && data !== null) { - // The real host acknowledges every submitted batch; the client's op - // sender keeps one batch in flight and only transmits the next after - // the ack, so without this reply every mint after the first stalls. + // The client keeps one op batch in flight until the host acks it. const { workflow_id, ops } = data as { workflow_id?: unknown ops?: unknown @@ -428,7 +400,6 @@ function defaultReplayTiming(): ReplayTiming { } interface ConversationFixtures { - /** Case id of the conversation under `fixtures/data/agent/conversations`. */ conversationCase: string // 'recorded' replays the fixture's at_ms gaps; the default follows AGENT_REPLAY_TIMING. replayTiming: ReplayTiming diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 2cfc991a884f..23d69dfab485 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -11,11 +11,6 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value) } -/** - * A realtime agent frame as recorded, minus the turn identity - * (`message_id` / `thread_id`): the replay stamps those from the ack it - * minted, then validates the result against `zAgentWsEvent` before sending. - */ export const zRecordedWsEvent = z.object({ type: z.string(), data: z.record(z.string(), z.unknown()), @@ -23,8 +18,7 @@ export const zRecordedWsEvent = z.object({ }) export type RecordedWsEvent = z.infer -// Payload shapes are the pinned applier's contract (it rejects a malformed -// op at replay time); the fixture only pins the vocabulary. +// The applier validates op payloads at replay time; only the vocabulary is pinned here. const zGraphOperation = z.custom( (value) => isRecord(value) && @@ -64,11 +58,6 @@ const zResponseEntry = z.discriminatedUnion('kind', [ }) ]) -/** - * One user prompt and the ordered agent response it produced: chat frames - * (`event`) interleaved with the semantic graph operations (`graph_ops`) the - * doc host folds into the workflow document. - */ export const zAgentConversation = z .object({ schema_version: z.literal('agent-conversation.v1'), diff --git a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts index 727f47b43304..431df7ac482b 100644 --- a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts +++ b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts @@ -1,6 +1,6 @@ -import type { AgentConversation } from '@e2e/fixtures/data/agent/agentConversation' +import type { GraphSnapshot } from '@comfyorg/comfy-multi-player' -// Expectations come from the fixture itself, so re-recording a case never needs hand-pinned values. +import type { AgentConversation } from '@e2e/fixtures/data/agent/agentConversation' export interface ExpectedGraphNode { id: string @@ -9,7 +9,7 @@ export interface ExpectedGraphNode { outputs: boolean[] } -interface SeedNode { +interface NodeBody { id: number | string type: string title?: string @@ -17,141 +17,60 @@ interface SeedNode { outputs?: Array<{ name: string }> } -interface Link { - from: string - fromSlot: number - to: string - toSlot: number -} - -interface NodeState { - type: string - title: string - inputNames: string[] - outputNames: string[] -} - function asRecord(value: unknown): Record { return typeof value === 'object' && value !== null ? (value as Record) : {} } -function slotNames(node: SeedNode): { - inputNames: string[] - outputNames: string[] -} { - return { - inputNames: (node.inputs ?? []).map((slot) => slot.name), - outputNames: (node.outputs ?? []).map((slot) => slot.name) - } +function seedNodes(conversation: AgentConversation): NodeBody[] { + return (conversation.workflow.seed as unknown as { nodes: NodeBody[] }).nodes +} + +function addedNodes(conversation: AgentConversation): NodeBody[] { + return conversation.response.flatMap((entry) => + entry.kind === 'graph_ops' + ? entry.ops.flatMap((op) => + asRecord(op).op === 'add_node' + ? [asRecord(asRecord(op).node) as unknown as NodeBody] + : [] + ) + : [] + ) } +// The doc keeps no titles or slot names; those come from the node bodies the turn started from or added. export function expectedGraphSnapshot( - conversation: AgentConversation + conversation: AgentConversation, + graph: GraphSnapshot ): ExpectedGraphNode[] { - const seed = conversation.workflow.seed as unknown as { - nodes: SeedNode[] - links: Array<[number, number, number, number, number, string]> - } + const bodies = [...seedNodes(conversation), ...addedNodes(conversation)] + const byType = new Map(bodies.map((body) => [body.type, body])) + const byId = new Map(bodies.map((body) => [String(body.id), body])) const catalog = conversation.workflow.catalog.types as Record< string, { widget_order: string[] } > - const nodes = new Map() - const byType = new Map< - string, - { inputNames: string[]; outputNames: string[] } - >() - for (const node of seed.nodes) { - const names = slotNames(node) - nodes.set(String(node.id), { - type: node.type, - title: node.title ?? node.type, - ...names - }) - byType.set(node.type, names) - } - const links = new Map() - for (const [id, from, fromSlot, to, toSlot] of seed.links) { - links.set(String(id), { - from: String(from), - fromSlot, - to: String(to), - toSlot - }) - } - const setLink = (id: string, link: Link) => { - for (const [existingId, existing] of links) { - if (existing.to === link.to && existing.toSlot === link.toSlot) - links.delete(existingId) - } - links.set(id, link) - } - - for (const entry of conversation.response) { - if (entry.kind !== 'graph_ops') continue - for (const rawOp of entry.ops) { - const op = asRecord(rawOp) - switch (op.op) { - case 'add_node': { - const node = asRecord(op.node) as unknown as SeedNode - const own = slotNames(node) - const fallback = byType.get(node.type) - nodes.set(String(node.id), { - type: node.type, - title: node.title ?? node.type, - inputNames: own.inputNames.length - ? own.inputNames - : (fallback?.inputNames ?? []), - outputNames: own.outputNames.length - ? own.outputNames - : (fallback?.outputNames ?? []) - }) - break - } - case 'delete_node': { - const id = String(op.node_id) - nodes.delete(id) - for (const [linkId, link] of links) { - if (link.from === id || link.to === id) links.delete(linkId) - } - break - } - case 'connect': - setLink(String(op.link_id), { - from: String(op.from_node), - fromSlot: Number(op.from_slot), - to: String(op.to_node), - toSlot: Number(op.to_slot) - }) - break - case 'set_widget': - break - default: - throw new Error( - `no expectation rule for recorded op ${String(op.op)}` - ) - } - } - } - - const connected = [...links.values()] - return [...nodes.entries()] + const links = Object.values(graph.links) as Array< + [unknown, unknown, number, unknown, number, string] + > + return Object.entries(graph.nodes) .map(([id, node]) => { - const widgets = new Set(catalog[node.type]?.widget_order ?? []) + const type = String(node.type) + const body = byId.get(id) ?? byType.get(type) + const widgets = new Set(catalog[type]?.widget_order ?? []) return { id, - title: node.title, + title: byId.get(id)?.title ?? type, // Widget-backed inputs render as widgets, not slot rows. - inputs: node.inputNames - .map((name, slot) => ({ name, slot })) + inputs: (body?.inputs ?? []) + .map((slot, index) => ({ name: slot.name, index })) .filter(({ name }) => !widgets.has(name)) - .map(({ slot }) => - connected.some((link) => link.to === id && link.toSlot === slot) + .map(({ index }) => + links.some((link) => String(link[3]) === id && link[4] === index) ), - outputs: node.outputNames.map((_, slot) => - connected.some((link) => link.from === id && link.fromSlot === slot) + outputs: (body?.outputs ?? []).map((_, index) => + links.some((link) => String(link[1]) === id && link[2] === index) ) } }) @@ -201,10 +120,7 @@ const KNOWN_TOOL_LABELS: Record = { // Mirrors the label rule in ToolCallCard.vue. export function toolRowLabel(name: string): string { - const known = Object.hasOwn(KNOWN_TOOL_LABELS, name) - ? KNOWN_TOOL_LABELS[name] - : undefined - if (known) return known + if (Object.hasOwn(KNOWN_TOOL_LABELS, name)) return KNOWN_TOOL_LABELS[name] const spaced = name.replaceAll('_', ' ') return spaced.charAt(0).toUpperCase() + spaced.slice(1) } @@ -216,43 +132,32 @@ export interface RecordedWidgetValue { } export function recordedWidgetValues( - conversation: AgentConversation + conversation: AgentConversation, + graph: GraphSnapshot ): RecordedWidgetValue[] { - const surviving = new Set( - expectedGraphSnapshot(conversation).map((node) => node.id) + return conversation.response.flatMap((entry) => + entry.kind === 'graph_ops' + ? entry.ops.flatMap((rawOp) => { + const op = asRecord(rawOp) + const nodeId = String(op.node_id) + const widget = String(op.widget) + const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] + return op.op === 'set_widget' && + (typeof value === 'string' || typeof value === 'number') + ? [{ nodeId, widget, value }] + : [] + }) + : [] ) - return conversation.response.flatMap((entry) => { - if (entry.kind !== 'graph_ops') return [] - return entry.ops.flatMap((rawOp) => { - const op = asRecord(rawOp) - if (op.op !== 'set_widget') return [] - const nodeId = String(op.node_id) - const value = op.value - if ( - !surviving.has(nodeId) || - (typeof value !== 'string' && typeof value !== 'number') - ) - return [] - return [{ nodeId, widget: String(op.widget), value }] - }) - }) } export function recordedAddedNodeIds( - conversation: AgentConversation + conversation: AgentConversation, + graph: GraphSnapshot ): string[] { - const surviving = new Set( - expectedGraphSnapshot(conversation).map((node) => node.id) - ) - return conversation.response.flatMap((entry) => { - if (entry.kind !== 'graph_ops') return [] - return entry.ops.flatMap((rawOp) => { - const op = asRecord(rawOp) - if (op.op !== 'add_node') return [] - const id = String(asRecord(op.node).id) - return surviving.has(id) ? [id] : [] - }) - }) + return addedNodes(conversation) + .map((body) => String(body.id)) + .filter((id) => id in graph.nodes) } // One update per ops entry plus the subscribe catch-up. @@ -262,12 +167,3 @@ export function expectedUpdateCount(conversation: AgentConversation): number { 1 ) } - -export function recordedSpanMs( - conversation: AgentConversation -): number | undefined { - const offsets = conversation.response.flatMap((entry) => - entry.at_ms === undefined ? [] : [entry.at_ms] - ) - return offsets.length === 0 ? undefined : Math.max(...offsets) -} diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 6b87106c5598..685e261a75ee 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -72,11 +72,13 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { `${expectedUpdateCount(conversation)} updates` ) - for (const id of recordedAddedNodeIds(conversation)) { + const graph = agentConversation.hostGraph() + for (const id of recordedAddedNodeIds(conversation, graph)) { await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() } for (const { nodeId, widget, value } of recordedWidgetValues( - conversation + conversation, + graph )) { const field = page .locator(`[data-node-id="${nodeId}"]`) @@ -88,7 +90,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { await expect .poll(() => agentConversation.graphSnapshot()) - .toEqual(expectedGraphSnapshot(conversation)) + .toEqual(expectedGraphSnapshot(conversation, graph)) }) }) } diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts index 8f9b0f314e9e..183c9051841a 100644 --- a/scripts/agentConversationCapture.ts +++ b/scripts/agentConversationCapture.ts @@ -80,13 +80,7 @@ function operationsFromResult(toolCall: z.infer) { }) } -/** - * Projects a cloud backend capture into the replay format. Websocket frames - * retain their recorded order; accepted semantic ops are inserted immediately - * before the matching terminal tool-call frame, where the live doc update was - * observed. No operation can be authored by this exporter: every emitted op - * must exist in the parent audit result and in its durable accepted-op rows. - */ +// Ops are inserted before their terminal tool-call frame and must exist in the accepted-op rows; nothing is authored here. export function exportAgentConversation(input: unknown) { const capture = zBackendCapture.parse(input) const toolCalls = new Map( From b84c222806fa9f0cbcea00452f5aab7adf05207b Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:15:00 -0700 Subject: [PATCH 23/49] test(agent): pass the host graph to the timing replay --- .../tests/agent/agentConversationTiming.spec.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts index 1faad07ab9aa..62c3a4070055 100644 --- a/browser_tests/tests/agent/agentConversationTiming.spec.ts +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -1,10 +1,7 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -import { - expectedGraphSnapshot, - recordedSpanMs -} from '@e2e/fixtures/data/agent/agentConversationExpectations' +import { expectedGraphSnapshot } from '@e2e/fixtures/data/agent/agentConversationExpectations' test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { test.use({ @@ -16,7 +13,10 @@ test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { agentConversation }) => { test.setTimeout(120_000) - const span = recordedSpanMs(agentConversation.conversation) + const offsets = agentConversation.conversation.response.flatMap((entry) => + entry.at_ms === undefined ? [] : [entry.at_ms] + ) + const span = offsets.length ? Math.max(...offsets) : undefined expect(span, 'the fixture carries at_ms offsets').toBeDefined() await agentConversation.sendPrompt() @@ -26,6 +26,11 @@ test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) await expect .poll(() => agentConversation.graphSnapshot()) - .toEqual(expectedGraphSnapshot(agentConversation.conversation)) + .toEqual( + expectedGraphSnapshot( + agentConversation.conversation, + agentConversation.hostGraph() + ) + ) }) }) From 520ca7c76c10fefb4686144fc404b39aa810c4d4 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:17:41 -0700 Subject: [PATCH 24/49] test(agent): cut the remaining multi-line comments to one line --- browser_tests/tests/agent/agentSubgraphMintScope.spec.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts index eaa26995ef5b..7bff3b0a1cea 100644 --- a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts +++ b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts @@ -2,8 +2,7 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -// Regression for #16611: a delete inside a subgraph must not mint a root-scope wire op. -// Edits go through the layout store because UI-path deletes cannot see follower-only nodes yet (FE-1996). +// Regression for #16611; edits drive the layout store because UI deletes cannot see follower-only nodes (FE-1996). interface LayoutStoreModule { layoutStore: { From 793963bb87f51e876f11c085a691f23a77b084f1 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:18:55 -0700 Subject: [PATCH 25/49] test(agent): one-line the last carried comments --- src/workbench/extensions/agent/AgentPanelRoot.test.ts | 3 +-- src/workbench/extensions/agent/AgentPanelRoot.vue | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index 1b9374100b65..5da50c3476df 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -2400,8 +2400,7 @@ describe('AgentPanelRoot workflow binding', () => { ) const minted = hostStores.workflow.tabs.get('workflows/Video test.json') expect(minted?.filename).toBe('Video test') - // Blank, not the default template: the follower reconciles onto whatever - // the tab holds and keeps template links it does not displace. + // Blank, not the default template: the follower keeps template links it does not displace. expect(minted?.activeState).toMatchObject({ nodes: [], links: [] }) // The host minted the doc server-side; the follower fills the canvas. // Nothing loads, saves, or adopts here. diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index fbc35854405e..9bef53b7d5b6 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -542,9 +542,7 @@ async function onAgentActiveTab( // A new agent workflow opens as an EMPTY tab: the host minted its doc // server-side (seed-at-bind), and the follower fills the canvas through // the ordinary subscribe catch-up - no snapshot fetch, no draft apply. - // `createTemporary` without data seeds the default template, whose nodes - // share ids with typical agent docs; the follower's reconcile keeps any - // template link it does not displace, so the tab must really be blank. + // The default template's node ids collide with agent docs, so the tab must start blank. const creatingStartedAt = Date.now() tabActivity.setCreating(true) const remainingCreatingTime = From f5c2531ceafd74d626802099071ee7ac7fc9490a Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:22:04 -0700 Subject: [PATCH 26/49] fix(agent): squash the blank-tab comment to one line --- src/workbench/extensions/agent/AgentPanelRoot.vue | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 9bef53b7d5b6..e06580d80985 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -539,10 +539,7 @@ async function onAgentActiveTab( }) return } - // A new agent workflow opens as an EMPTY tab: the host minted its doc - // server-side (seed-at-bind), and the follower fills the canvas through - // the ordinary subscribe catch-up - no snapshot fetch, no draft apply. - // The default template's node ids collide with agent docs, so the tab must start blank. + // The tab starts blank: the host seeds the doc at bind and the default template's node ids collide with agent docs. const creatingStartedAt = Date.now() tabActivity.setCreating(true) const remainingCreatingTime = From 198e9109dbee434f3d27098b99dc983b81e14502 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 3 Sep 2026 13:28:16 +0000 Subject: [PATCH 27/49] [automated] Apply ESLint and Oxfmt fixes --- browser_tests/fixtures/agentConversationFixture.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 13db6cdc3444..04e5d2cb8317 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -2,7 +2,11 @@ import type { Locator, Page, WebSocketRoute } from '@playwright/test' import { expect } from '@playwright/test' import { applyOps, mint, readGraph } from '@comfyorg/comfy-multi-player' -import type { GraphSnapshot,WidgetCatalog,WorkflowJSON } from '@comfyorg/comfy-multi-player' +import type { + GraphSnapshot, + WidgetCatalog, + WorkflowJSON +} from '@comfyorg/comfy-multi-player' import * as Y from 'yjs' import enMessages from '@/locales/en/main.json' with { type: 'json' } From 5530d7e0966cfcc2945eeba1a4ac321fe4d557e8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 3 Sep 2026 13:28:20 +0000 Subject: [PATCH 28/49] [automated] Apply ESLint and Oxfmt fixes --- browser_tests/fixtures/agentConversationFixture.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 13db6cdc3444..04e5d2cb8317 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -2,7 +2,11 @@ import type { Locator, Page, WebSocketRoute } from '@playwright/test' import { expect } from '@playwright/test' import { applyOps, mint, readGraph } from '@comfyorg/comfy-multi-player' -import type { GraphSnapshot,WidgetCatalog,WorkflowJSON } from '@comfyorg/comfy-multi-player' +import type { + GraphSnapshot, + WidgetCatalog, + WorkflowJSON +} from '@comfyorg/comfy-multi-player' import * as Y from 'yjs' import enMessages from '@/locales/en/main.json' with { type: 'json' } From 80f31b5a76cda4a3c850652d0ec671e39cd7a443 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 06:48:55 -0700 Subject: [PATCH 29/49] test(agent): drop the two replay regressions already covered by unit tests The catch-up baseline (#16285) is pinned in followerSubscription.test.ts and useAgentCrdtFollower.test.ts, the subgraph mint scope (#16611) in the mint-port tests; the replay copies added nothing. --- .../conversations/agent-catchup-baseline.json | 69 ---------- .../agent-subgraph-mint-scope.json | 129 ------------------ .../tests/agent/agentCatchupBaseline.spec.ts | 46 ------- .../agent/agentSubgraphMintScope.spec.ts | 113 --------------- 4 files changed, 357 deletions(-) delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json delete mode 100644 browser_tests/tests/agent/agentCatchupBaseline.spec.ts delete mode 100644 browser_tests/tests/agent/agentSubgraphMintScope.spec.ts diff --git a/browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json b/browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json deleted file mode 100644 index dd7274b8f676..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-catchup-baseline.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-catchup-baseline", - "response_side": "synthesized", - "note": "Regression fixture for PR #16285 (apply CRDT catch-up at acknowledged sequence). The host sends doc_subscribed(seq=N) and then the catch-up doc_update(seq=N); pre-fix the bridge recorded N as the applied baseline and dropped the catch-up as stale, leaving a fresh follower empty. The conversation only binds; the seeded graph projecting at all IS the regression surface. Response synthesized: no model output is asserted." - }, - "workflow": { - "id": "b7e2d4f6-1a3c-4e5b-9d80-2f4a6c8e0b1d", - "name": "Catch-up baseline", - "catalog": { - "types": { - "LoadImage": { "widget_order": ["image", "upload"] }, - "PreviewImage": { "widget_order": [] } - } - }, - "seed": { - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [0, 0], - "size": [315, 314], - "inputs": [], - "outputs": [ - { "name": "IMAGE", "type": "IMAGE", "links": [1] }, - { "name": "MASK", "type": "MASK", "links": [] } - ], - "widgets_values": ["example.png", "image"] - }, - { - "id": 2, - "type": "PreviewImage", - "pos": [400, 0], - "size": [210, 246], - "inputs": [{ "name": "images", "type": "IMAGE", "link": 1 }], - "outputs": [], - "widgets_values": [] - } - ], - "links": [[1, 1, 0, 2, 0, "IMAGE"]] - } - }, - "request": { - "content": "Take a look at this workflow." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "workflow_id": "b7e2d4f6-1a3c-4e5b-9d80-2f4a6c8e0b1d", - "name": "Catch-up baseline" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { "delta": "I can see the workflow." } - } - }, - { "kind": "event", "event": { "type": "agent_message_done", "data": {} } } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json b/browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json deleted file mode 100644 index 0bc81fcca34d..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-subgraph-mint-scope.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-subgraph-mint-scope", - "response_side": "synthesized", - "note": "Regression fixture for PR #16611 (preserve root semantics for human graph edits). The conversation only binds the follower to a doc whose seed carries a subgraph definition and instance; the regression surface is the HUMAN edit the spec performs afterwards. Response synthesized: no model output is asserted." - }, - "workflow": { - "id": "a3f8c1d2-9b4e-4f7a-8c50-6e1d2b3a4c5f", - "name": "Subgraph scope", - "catalog": { - "types": { - "LoadImage": { - "widget_order": ["image", "upload"] - }, - "PreviewImage": { - "widget_order": [] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - } - } - }, - "seed": { - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [80, 420], - "size": [315, 314], - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [] - }, - { - "name": "MASK", - "type": "MASK", - "links": [] - } - ], - "widgets_values": ["example.png", "image"] - }, - { - "id": 2, - "type": "9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d", - "title": "Inner steps", - "pos": [460, 420], - "size": [220, 100], - "inputs": [], - "outputs": [], - "widgets_values": [] - } - ], - "links": [], - "definitions": { - "subgraphs": [ - { - "id": "9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d", - "name": "Inner steps", - "inputs": [], - "outputs": [], - "nodes": [ - { - "id": 10, - "type": "PreviewImage", - "pos": [120, 380], - "size": [210, 246], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": null - } - ], - "outputs": [], - "widgets_values": [] - } - ], - "links": [] - } - ] - } - } - }, - "request": { - "content": "Take a look at this workflow; I will make some edits myself." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "workflow_id": "a3f8c1d2-9b4e-4f7a-8c50-6e1d2b3a4c5f", - "name": "Subgraph scope" - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I can see the workflow. Go ahead with your edits." - } - } - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": {} - } - } - ] -} diff --git a/browser_tests/tests/agent/agentCatchupBaseline.spec.ts b/browser_tests/tests/agent/agentCatchupBaseline.spec.ts deleted file mode 100644 index 10a121d22553..000000000000 --- a/browser_tests/tests/agent/agentCatchupBaseline.spec.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { expect } from '@playwright/test' - -import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' - -// Regression for #16285: pre-fix the follower dropped the catch-up sent at the ack's seq and rendered empty. -test.describe( - 'Agent replay regression: catch-up baseline (PR #16285)', - { tag: '@cloud' }, - () => { - test.use({ conversationCase: 'agent-catchup-baseline' }) - - test('the subscription catch-up projects the seeded graph', async ({ - agentConversation - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - '1 updates' - ) - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual([ - { - id: '1', - title: 'LoadImage', - inputs: [], - outputs: [true, false] - }, - { - id: '2', - title: 'PreviewImage', - inputs: [true], - outputs: [] - } - ]) - }) - } -) diff --git a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts b/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts deleted file mode 100644 index 7bff3b0a1cea..000000000000 --- a/browser_tests/tests/agent/agentSubgraphMintScope.spec.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { expect } from '@playwright/test' - -import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' - -// Regression for #16611; edits drive the layout store because UI deletes cannot see follower-only nodes (FE-1996). - -interface LayoutStoreModule { - layoutStore: { - getNodeLayout( - graphId: string, - nodeId: string - ): { position: { x: number; y: number } } | null - applyOperation(operation: Record): void - } -} - -test.describe( - 'Agent replay regression: subgraph mint scope (PR #16611)', - { tag: '@cloud' }, - () => { - test.use({ conversationCase: 'agent-subgraph-mint-scope' }) - - test('a human delete inside a subgraph does not mint a root wire op', async ({ - agentConversation, - page - }) => { - test.setTimeout(45_000) - const { panel } = agentConversation - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - await expect(panel.getByTestId('agent-crdt-status')).toContainText( - 'connected' - ) - await expect(page.locator('[data-node-id="1"]')).toBeVisible() - - const deletedNodeIds = () => - agentConversation - .outboundOps() - .filter((op) => op.op === 'delete_node') - .map((op) => String((op as { node_id?: unknown }).node_id)) - - const SUBGRAPH_ID = '9d1b2c3e-5f6a-4b7c-8d90-1e2f3a4b5c6d' - const WORKFLOW_ID = agentConversation.conversation.workflow.id - const STORE_MODULE = '/src/renderer/core/layout/store/layoutStore.ts' - await page.evaluate( - async ({ subgraphId, workflowId, storeModule }) => { - const { layoutStore } = (await import( - /* @vite-ignore */ storeModule - )) as unknown as LayoutStoreModule - // Layouts are keyed by the canvas root graph id, not the workflow id. - const rootGraphId = [window.app!.graph.id, workflowId].find((id) => - layoutStore.getNodeLayout(id, '1') - ) - if (!rootGraphId) { - throw new Error( - 'seeded layout for node 1 not found under any graph' - ) - } - const meta = (nodeId: string, ownerGraphId: string) => ({ - graphId: rootGraphId, - ownerGraphId, - nodeId, - timestamp: Date.now(), - source: 'canvas' - }) - - // Positive control: a root-level delete must mint. - layoutStore.applyOperation({ - type: 'deleteNode', - ...meta('1', rootGraphId) - }) - - // Interior create and delete have no wire representation; neither may mint. - layoutStore.applyOperation({ - type: 'createNode', - ...meta('10', subgraphId), - layout: { - id: '10', - position: { x: 120, y: 380 }, - size: { width: 210, height: 246 }, - zIndex: 0, - visible: true, - bounds: { x: 120, y: 380, width: 210, height: 246 } - } - }) - layoutStore.applyOperation({ - type: 'deleteNode', - ...meta('10', subgraphId) - }) - - // A later root delete anchors the order: an interior mint would precede it. - layoutStore.applyOperation({ - type: 'deleteNode', - ...meta('2', rootGraphId) - }) - }, - { - subgraphId: SUBGRAPH_ID, - workflowId: WORKFLOW_ID, - storeModule: STORE_MODULE - } - ) - - await expect - .poll(() => deletedNodeIds().length, { timeout: 10_000 }) - .toBeGreaterThanOrEqual(2) - - expect(deletedNodeIds()).toEqual(['1', '2']) - }) - } -) From 0a71484a1ce03d819a61ba58348352ca1a51b9e0 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:20:31 -0700 Subject: [PATCH 30/49] test(agent): derive replay expectations on the harness itself The expectation helpers move onto AgentConversationHarness, which owns the conversation and the applier-built doc they read from; the panel's row label rule now comes from the product's knownTool table. The recorded-case lister tolerates a missing conversations directory so the replay spec loads on a branch that carries no recordings. --- .../agent/agentConversationExpectations.ts | 169 ------------------ 1 file changed, 169 deletions(-) delete mode 100644 browser_tests/fixtures/data/agent/agentConversationExpectations.ts diff --git a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts b/browser_tests/fixtures/data/agent/agentConversationExpectations.ts deleted file mode 100644 index 431df7ac482b..000000000000 --- a/browser_tests/fixtures/data/agent/agentConversationExpectations.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { GraphSnapshot } from '@comfyorg/comfy-multi-player' - -import type { AgentConversation } from '@e2e/fixtures/data/agent/agentConversation' - -export interface ExpectedGraphNode { - id: string - title: string - inputs: boolean[] - outputs: boolean[] -} - -interface NodeBody { - id: number | string - type: string - title?: string - inputs?: Array<{ name: string }> - outputs?: Array<{ name: string }> -} - -function asRecord(value: unknown): Record { - return typeof value === 'object' && value !== null - ? (value as Record) - : {} -} - -function seedNodes(conversation: AgentConversation): NodeBody[] { - return (conversation.workflow.seed as unknown as { nodes: NodeBody[] }).nodes -} - -function addedNodes(conversation: AgentConversation): NodeBody[] { - return conversation.response.flatMap((entry) => - entry.kind === 'graph_ops' - ? entry.ops.flatMap((op) => - asRecord(op).op === 'add_node' - ? [asRecord(asRecord(op).node) as unknown as NodeBody] - : [] - ) - : [] - ) -} - -// The doc keeps no titles or slot names; those come from the node bodies the turn started from or added. -export function expectedGraphSnapshot( - conversation: AgentConversation, - graph: GraphSnapshot -): ExpectedGraphNode[] { - const bodies = [...seedNodes(conversation), ...addedNodes(conversation)] - const byType = new Map(bodies.map((body) => [body.type, body])) - const byId = new Map(bodies.map((body) => [String(body.id), body])) - const catalog = conversation.workflow.catalog.types as Record< - string, - { widget_order: string[] } - > - const links = Object.values(graph.links) as Array< - [unknown, unknown, number, unknown, number, string] - > - return Object.entries(graph.nodes) - .map(([id, node]) => { - const type = String(node.type) - const body = byId.get(id) ?? byType.get(type) - const widgets = new Set(catalog[type]?.widget_order ?? []) - return { - id, - title: byId.get(id)?.title ?? type, - // Widget-backed inputs render as widgets, not slot rows. - inputs: (body?.inputs ?? []) - .map((slot, index) => ({ name: slot.name, index })) - .filter(({ name }) => !widgets.has(name)) - .map(({ index }) => - links.some((link) => String(link[3]) === id && link[4] === index) - ), - outputs: (body?.outputs ?? []).map((_, index) => - links.some((link) => String(link[1]) === id && link[2] === index) - ) - } - }) - .sort((a, b) => Number(a.id) - Number(b.id)) -} - -export interface RecordedToolCall { - name: string - ok: boolean -} - -// The panel starts a new tool group whenever thinking or text interrupts the calls. -export function recordedToolCallGroups( - conversation: AgentConversation -): RecordedToolCall[][] { - const groups: RecordedToolCall[][] = [] - let current: RecordedToolCall[] = [] - for (const entry of conversation.response) { - if (entry.kind !== 'event') continue - const { type, data } = entry.event - if (type === 'agent_tool_call') { - if (data.status !== 'running') - current.push({ - name: String(data.tool_name), - ok: data.status === 'success' - }) - continue - } - if ( - (type === 'agent_thinking' || type === 'agent_message_delta') && - current.length > 0 - ) { - groups.push(current) - current = [] - } - } - if (current.length > 0) groups.push(current) - return groups -} - -const KNOWN_TOOL_LABELS: Record = { - new_tab: 'Opened a new tab', - switch_tab: 'Switched tabs', - remember: 'Saved a preference', - forget: 'Forgot a preference' -} - -// Mirrors the label rule in ToolCallCard.vue. -export function toolRowLabel(name: string): string { - if (Object.hasOwn(KNOWN_TOOL_LABELS, name)) return KNOWN_TOOL_LABELS[name] - const spaced = name.replaceAll('_', ' ') - return spaced.charAt(0).toUpperCase() + spaced.slice(1) -} - -export interface RecordedWidgetValue { - nodeId: string - widget: string - value: string | number -} - -export function recordedWidgetValues( - conversation: AgentConversation, - graph: GraphSnapshot -): RecordedWidgetValue[] { - return conversation.response.flatMap((entry) => - entry.kind === 'graph_ops' - ? entry.ops.flatMap((rawOp) => { - const op = asRecord(rawOp) - const nodeId = String(op.node_id) - const widget = String(op.widget) - const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] - return op.op === 'set_widget' && - (typeof value === 'string' || typeof value === 'number') - ? [{ nodeId, widget, value }] - : [] - }) - : [] - ) -} - -export function recordedAddedNodeIds( - conversation: AgentConversation, - graph: GraphSnapshot -): string[] { - return addedNodes(conversation) - .map((body) => String(body.id)) - .filter((id) => id in graph.nodes) -} - -// One update per ops entry plus the subscribe catch-up. -export function expectedUpdateCount(conversation: AgentConversation): number { - return ( - conversation.response.filter((entry) => entry.kind === 'graph_ops').length + - 1 - ) -} From cd7593360f0a06978a20510e7ff31e028077f948 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:22:44 -0700 Subject: [PATCH 31/49] test(agent): keep the harness's helper types private --- .../fixtures/agentConversationFixture.ts | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 04e5d2cb8317..23b29ee7deee 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -26,6 +26,8 @@ import type { RecordedWsEvent } from '@e2e/fixtures/data/agent/agentConversation' import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +import { knownTool } from '@/workbench/extensions/agent/services/agent/agentToolGlyph' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' @@ -127,6 +129,44 @@ class HostDoc { } } +interface NodeBody { + id: number | string + type: string + title?: string + inputs?: Array<{ name: string }> + outputs?: Array<{ name: string }> +} + +interface RecordedToolCall { + name: string + ok: boolean +} + +interface RecordedWidgetValue { + nodeId: string + widget: string + value: string | number +} + +function asRecord(value: unknown): Record { + return typeof value === 'object' && value !== null + ? (value as Record) + : {} +} + +// The panel's row label: the known-tool table, else the humanized tool name (ToolCallCard.vue). +export function toolRowLabel(name: string): string { + const known = knownTool(name)?.labelKey + const label = known + ? enMessages.agent[ + known.replace('agent.', '') as keyof typeof enMessages.agent + ] + : undefined + if (typeof label === 'string') return label + const spaced = name.replaceAll('_', ' ') + return spaced.charAt(0).toUpperCase() + spaced.slice(1) +} + interface GraphNodeSnapshot { id: string title: string @@ -238,6 +278,123 @@ class AgentConversationHarness { } // Doc-id filter: a stray template node must not pin the template here. + private nodeBodies(): NodeBody[] { + const seed = ( + this.conversation.workflow.seed as unknown as { nodes: NodeBody[] } + ).nodes + const added = this.conversation.response.flatMap((entry) => + entry.kind === 'graph_ops' + ? entry.ops.flatMap((op) => + asRecord(op).op === 'add_node' + ? [asRecord(asRecord(op).node) as unknown as NodeBody] + : [] + ) + : [] + ) + return [...seed, ...added] + } + + // The doc keeps no titles or slot names; those come from the node bodies the turn started from or added. + expectedGraph(): GraphNodeSnapshot[] { + const graph = this.host.graph() + const bodies = this.nodeBodies() + const byType = new Map(bodies.map((body) => [body.type, body])) + const byId = new Map(bodies.map((body) => [String(body.id), body])) + const catalog = this.conversation.workflow.catalog.types as Record< + string, + { widget_order: string[] } + > + const links = Object.values(graph.links) as Array< + [unknown, unknown, number, unknown, number, string] + > + return Object.entries(graph.nodes) + .map(([id, node]) => { + const type = String(node.type) + const body = byId.get(id) ?? byType.get(type) + const widgets = new Set(catalog[type]?.widget_order ?? []) + return { + id, + title: byId.get(id)?.title ?? type, + // Widget-backed inputs render as widgets, not slot rows. + inputs: (body?.inputs ?? []) + .map((slot, index) => ({ name: slot.name, index })) + .filter(({ name }) => !widgets.has(name)) + .map(({ index }) => + links.some((link) => String(link[3]) === id && link[4] === index) + ), + outputs: (body?.outputs ?? []).map((_, index) => + links.some((link) => String(link[1]) === id && link[2] === index) + ) + } + }) + .sort((a, b) => Number(a.id) - Number(b.id)) + } + + // The panel starts a new tool group whenever thinking or text interrupts the calls. + toolCallGroups(): RecordedToolCall[][] { + const groups: RecordedToolCall[][] = [] + let current: RecordedToolCall[] = [] + for (const entry of this.conversation.response) { + if (entry.kind !== 'event') continue + const { type, data } = entry.event + if (type === 'agent_tool_call') { + if (data.status !== 'running') + current.push({ + name: String(data.tool_name), + ok: data.status === 'success' + }) + continue + } + if ( + (type === 'agent_thinking' || type === 'agent_message_delta') && + current.length > 0 + ) { + groups.push(current) + current = [] + } + } + if (current.length > 0) groups.push(current) + return groups + } + + recordedWidgetValues(): RecordedWidgetValue[] { + const graph = this.host.graph() + return this.conversation.response.flatMap((entry) => + entry.kind === 'graph_ops' + ? entry.ops.flatMap((rawOp) => { + const op = asRecord(rawOp) + const nodeId = String(op.node_id) + const widget = String(op.widget) + const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] + return op.op === 'set_widget' && + (typeof value === 'string' || typeof value === 'number') + ? [{ nodeId, widget, value }] + : [] + }) + : [] + ) + } + + addedNodeIds(): string[] { + const graph = this.host.graph() + const seedIds = new Set( + ( + this.conversation.workflow.seed as unknown as { nodes: NodeBody[] } + ).nodes.map((node) => String(node.id)) + ) + return this.nodeBodies() + .map((body) => String(body.id)) + .filter((id) => !seedIds.has(id) && id in graph.nodes) + } + + // One update per ops entry plus the subscribe catch-up. + expectedUpdateCount(): number { + return ( + this.conversation.response.filter((entry) => entry.kind === 'graph_ops') + .length + 1 + ) + } + hostGraph(): GraphSnapshot { return this.host.graph() } From b16baafcf63658a60d7bf4e2edec7f807f1ef94f Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:27:00 -0700 Subject: [PATCH 32/49] test(agent): replay spec reads its expectations from the harness --- .../fixtures/data/agent/agentConversation.ts | 3 +- .../agent/agentConversationReplay.spec.ts | 38 +++++++------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 23d69dfab485..0fd0ccc36916 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -1,4 +1,4 @@ -import { readFileSync, readdirSync } from 'node:fs' +import { existsSync, readFileSync, readdirSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' @@ -96,6 +96,7 @@ export type AgentConversation = z.infer export function listRecordedConversations(): string[] { const dir = fileURLToPath(new URL('./conversations/', import.meta.url)) + if (!existsSync(dir)) return [] return readdirSync(dir) .filter((file) => file.endsWith('.json')) .map((file) => file.slice(0, -'.json'.length)) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 685e261a75ee..575987f54384 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -1,28 +1,17 @@ import { expect } from '@playwright/test' -import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' import { - listRecordedConversations, - loadAgentConversation -} from '@e2e/fixtures/data/agent/agentConversation' -import { - expectedGraphSnapshot, - expectedUpdateCount, - recordedAddedNodeIds, - recordedToolCallGroups, - recordedWidgetValues, + agentConversationTest as test, toolRowLabel -} from '@e2e/fixtures/data/agent/agentConversationExpectations' +} from '@e2e/fixtures/agentConversationFixture' +import { listRecordedConversations } from '@e2e/fixtures/data/agent/agentConversation' test.describe('Agent conversation replay', { tag: '@cloud' }, () => { for (const conversationCase of listRecordedConversations()) { - const conversation = loadAgentConversation(conversationCase) - const groups = recordedToolCallGroups(conversation) - test.describe(`recorded ${conversationCase}`, () => { test.use({ conversationCase }) - test(`replays: ${conversation.request.content}`, async ({ + test('replays the recorded turn onto the panel and the canvas', async ({ agentConversation, page }) => { @@ -33,6 +22,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { await agentConversation.replayResponse() await agentConversation.waitForTurnComplete() + const groups = agentConversation.toolCallGroups() const groupButtons = panel.getByRole('button', { name: /^Ran \d+ tool call/ }) @@ -59,7 +49,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { await expect( panel.getByRole('button', { - name: `Open ${conversation.workflow.name}` + name: `Open ${agentConversation.conversation.workflow.name}` }) ).toBeVisible() await expect( @@ -69,17 +59,17 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { 'connected' ) await expect(panel.getByTestId('agent-crdt-status')).toContainText( - `${expectedUpdateCount(conversation)} updates` + `${agentConversation.expectedUpdateCount()} updates` ) - const graph = agentConversation.hostGraph() - for (const id of recordedAddedNodeIds(conversation, graph)) { + for (const id of agentConversation.addedNodeIds()) { await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() } - for (const { nodeId, widget, value } of recordedWidgetValues( - conversation, - graph - )) { + for (const { + nodeId, + widget, + value + } of agentConversation.recordedWidgetValues()) { const field = page .locator(`[data-node-id="${nodeId}"]`) .getByLabel(widget, { exact: true }) @@ -90,7 +80,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { await expect .poll(() => agentConversation.graphSnapshot()) - .toEqual(expectedGraphSnapshot(conversation, graph)) + .toEqual(agentConversation.expectedGraph()) }) }) } From e4931f82691db4c8c3a181392900e4efa45f06d7 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:30:22 -0700 Subject: [PATCH 33/49] test(agent): timing replay reads the expected graph from the harness --- browser_tests/tests/agent/agentConversationTiming.spec.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts index 62c3a4070055..91221a185535 100644 --- a/browser_tests/tests/agent/agentConversationTiming.spec.ts +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -1,7 +1,6 @@ import { expect } from '@playwright/test' import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' -import { expectedGraphSnapshot } from '@e2e/fixtures/data/agent/agentConversationExpectations' test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { test.use({ @@ -26,11 +25,6 @@ test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) await expect .poll(() => agentConversation.graphSnapshot()) - .toEqual( - expectedGraphSnapshot( - agentConversation.conversation, - agentConversation.hostGraph() - ) - ) + .toEqual(agentConversation.expectedGraph()) }) }) From d17304d7ac21ae0e56a00125f560bd2edf694b2f Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:45:48 -0700 Subject: [PATCH 34/49] test(agent): address review on the replay harness and exporter Node ids sort with the shared comparator so non-numeric ids keep a stable order; a widget written twice in one turn is checked at its final value; tool rows are counted per coalesced label as ToolCallGroup renders them; the exporter refuses a tool-call frame whose status is neither running, success nor error; the README names source.capture and source.note as the provenance the replay keeps. --- browser_tests/README.md | 5 +- .../fixtures/agentConversationFixture.ts | 49 +++++++++++++------ .../agent/agentConversationReplay.spec.ts | 13 ++--- scripts/agentConversationCapture.test.ts | 24 +++++++++ scripts/agentConversationCapture.ts | 5 ++ 5 files changed, 67 insertions(+), 29 deletions(-) diff --git a/browser_tests/README.md b/browser_tests/README.md index 756454f5ee4a..68dc3e9320f9 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -637,10 +637,7 @@ fix so the bug stays fixed: run lines in the PR description. A replay case that never went red against the bug does not count as regression coverage. -Name the case after the behavior it protects (`agent-.json`, -with the fix PR cited in the spec header and the fixture's `source.note`) and note -the fixture's source rows (thread id + seq range) in the JSON's -metadata block so the recording can be re-exported. +Name the case after the behavior it protects (`agent-.json`, with the fix PR cited in the spec header and the fixture's `source.note`). The exporter writes the provenance the replay keeps: `source.capture` (thread and message ids) and `source.note` (row ids and the raw capture hash), which is enough to re-export the recording. ## Test Data & Typed API Mocks diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 23b29ee7deee..efd6ccabf977 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -27,6 +27,7 @@ import type { } from '@e2e/fixtures/data/agent/agentConversation' import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' +import { compareNodeIds, toNodeId } from '@/types/nodeId' import { knownTool } from '@/workbench/extensions/agent/services/agent/agentToolGlyph' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' @@ -327,10 +328,25 @@ class AgentConversationHarness { ) } }) - .sort((a, b) => Number(a.id) - Number(b.id)) + .sort((a, b) => compareNodeIds(toNodeId(a.id), toNodeId(b.id))) } // The panel starts a new tool group whenever thinking or text interrupts the calls. + // ToolCallGroup.vue merges consecutive calls of the same tool into one row. + toolRowLabels(): Map { + const counts = new Map() + for (const group of this.toolCallGroups()) { + let previous: string | undefined + for (const call of group) { + if (call.name === previous) continue + previous = call.name + const label = toolRowLabel(call.name) + counts.set(label, (counts.get(label) ?? 0) + 1) + } + } + return counts + } + toolCallGroups(): RecordedToolCall[][] { const groups: RecordedToolCall[][] = [] let current: RecordedToolCall[] = [] @@ -357,22 +373,23 @@ class AgentConversationHarness { return groups } + // Last write wins per widget; the rendered control shows only the final value. recordedWidgetValues(): RecordedWidgetValue[] { const graph = this.host.graph() - return this.conversation.response.flatMap((entry) => - entry.kind === 'graph_ops' - ? entry.ops.flatMap((rawOp) => { - const op = asRecord(rawOp) - const nodeId = String(op.node_id) - const widget = String(op.widget) - const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] - return op.op === 'set_widget' && - (typeof value === 'string' || typeof value === 'number') - ? [{ nodeId, widget, value }] - : [] - }) - : [] - ) + const latest = new Map() + for (const entry of this.conversation.response) { + if (entry.kind !== 'graph_ops') continue + for (const rawOp of entry.ops) { + const op = asRecord(rawOp) + if (op.op !== 'set_widget') continue + const nodeId = String(op.node_id) + const widget = String(op.widget) + const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] + if (typeof value === 'string' || typeof value === 'number') + latest.set(`${nodeId}/${widget}`, { nodeId, widget, value }) + } + } + return [...latest.values()] } addedNodeIds(): string[] { @@ -422,7 +439,7 @@ class AgentConversationHarness { ) return snapshot .filter((node) => docNodeIds.has(node.id)) - .sort((a, b) => Number(a.id) - Number(b.id)) + .sort((a, b) => compareNodeIds(toNodeId(a.id), toNodeId(b.id))) } private async mockAgentApi(): Promise { diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 575987f54384..8c20bbd5a8e9 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -1,9 +1,6 @@ import { expect } from '@playwright/test' -import { - agentConversationTest as test, - toolRowLabel -} from '@e2e/fixtures/agentConversationFixture' +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' import { listRecordedConversations } from '@e2e/fixtures/data/agent/agentConversation' test.describe('Agent conversation replay', { tag: '@cloud' }, () => { @@ -40,11 +37,9 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { succeeded ? 'false' : 'true' ) if (succeeded) await button.click() - for (const call of calls) { - await expect( - toolRows.filter({ hasText: toolRowLabel(call.name) }).first() - ).toBeVisible() - } + } + for (const [label, count] of agentConversation.toolRowLabels()) { + await expect(toolRows.filter({ hasText: label })).toHaveCount(count) } await expect( diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts index 2c5bc1c9ae2a..cd917e65a830 100644 --- a/scripts/agentConversationCapture.test.ts +++ b/scripts/agentConversationCapture.test.ts @@ -123,6 +123,30 @@ describe('exportAgentConversation', () => { ) }) + it('refuses a tool-call frame with a missing status', () => { + const [running, success] = capture.frames + const { status: _status, ...withoutStatus } = success.data + expect(() => + exportAgentConversation({ + ...capture, + frames: [running, { ...success, data: withoutStatus }] + }) + ).toThrow(/status undefined/) + }) + + it('refuses a tool-call frame with an unknown status', () => { + const [running, success] = capture.frames + expect(() => + exportAgentConversation({ + ...capture, + frames: [ + running, + { ...success, data: { ...success.data, status: 'done' } } + ] + }) + ).toThrow(/status "done"/) + }) + it('refuses an accepted op missing from the recorded result', () => { expect(() => exportAgentConversation({ diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts index 183c9051841a..567e36f8c543 100644 --- a/scripts/agentConversationCapture.ts +++ b/scripts/agentConversationCapture.ts @@ -117,6 +117,11 @@ export function exportAgentConversation(input: unknown) { if (event.type === 'agent_tool_call') { const toolCallId = event.data.tool_call_id const status = event.data.status + if (status !== 'running' && status !== 'success' && status !== 'error') { + throw new Error( + `recorded agent_tool_call frame carries status ${JSON.stringify(status)}; only running, success or error are known` + ) + } if ( typeof toolCallId === 'string' && status !== 'running' && From 2dffdd4e95ddeb8f2a6781542cae0dfa9f3b06cc Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:46:13 -0700 Subject: [PATCH 35/49] test(agent): let a replay interleave human edits replayResponse accepts a callback run after each recorded entry, and the in-process doc host folds the client's own op batches into the doc as the real host does, so the expected graph includes edits a test makes while the agent's turn replays. --- .../fixtures/agentConversationFixture.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index efd6ccabf977..216eca490087 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -91,6 +91,21 @@ class HostDoc { return this.updateFrame(update, HOST_ACTOR, []) } + // The real host folds client batches into the same doc, so the expected graph must see them. + applyClient(ops: unknown[]): string[] { + const result = applyOps( + this.doc, + ops as Parameters[1], + this.catalog + ) + this.seq += 1 + return result.outcomes.flatMap((outcome, index) => + outcome.outcome === 'applied' + ? [String((ops[index] as { op_id?: unknown }).op_id ?? index)] + : [] + ) + } + apply(operations: GraphOperation[]): DocFrame { const before = Y.encodeStateVector(this.doc) const ops = mintWireOps(operations, { @@ -250,9 +265,11 @@ class AgentConversationHarness { .toBe(THREAD_ID) } - async replayResponse(): Promise { + async replayResponse( + afterEntry?: (index: number) => Promise + ): Promise { const startedAt = Date.now() - for (const entry of this.conversation.response) { + for (const [index, entry] of this.conversation.response.entries()) { if (this.replayTiming === 'recorded' && entry.at_ms !== undefined) { const wait = entry.at_ms - (Date.now() - startedAt) if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)) @@ -263,6 +280,7 @@ class AgentConversationHarness { } await this.waitForSubscribe() this.send(this.host.apply(entry.ops)) + await afterEntry?.(index) } this.replayElapsedMs = Date.now() - startedAt } @@ -513,9 +531,7 @@ class AgentConversationHarness { ops?: unknown } if (workflow_id === this.conversation.workflow.id && Array.isArray(ops)) { - const applied = ops - .map((op) => (op as { op_id?: unknown }).op_id) - .filter((id): id is string => typeof id === 'string') + const applied = this.host.applyClient(ops) this.send({ type: 'doc_ops_result', data: { From 9328b9b351932ead8711a598c10c18c76c19bffb Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:48:03 -0700 Subject: [PATCH 36/49] test(agent): replay a recorded turn with a concurrent human edit A recording of three sequential add_node calls replays with a human widget edit typed between the first and second add; the edit must be minted onto the wire, survive the later doc updates, and show in the final graph. --- .../agent-rec-three-sequential-adds.json | 412 ++++++++++++++++++ .../agentConversationConcurrentEdits.spec.ts | 59 +++ 2 files changed, 471 insertions(+) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json create mode 100644 browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json new file mode 100644 index 000000000000..04a3b7fdaa63 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json @@ -0,0 +1,412 @@ +{ + "schema_version": "agent-conversation.v1", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-three-sequential-adds", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 3710f44b-9941-4bc4-a806-5c15d00cd807; message d0d6b5dc-b189-491a-804e-f52fdb9495cb; workflow 7afd1f94-6cb5-4923-9388-18851b0fba5d (seeded by throwaway turn 9285739d-3bbe-475f-a681-6f9761c7a621; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['c27f6e4e-220a-42fc-8fd1-97c30d6d3c98', 'f020586d-9444-457f-85e9-8c96b2be6292', '3832f4c1-77c3-44a9-b890-83b82db5513a', 'df9c490c-5a9a-43b3-b6d7-931c40084eaf']; rows agent-rec-three-sequential-adds.cc-a1.rows.json; raw capture sha256 6a1dfc3c7d4961130ecc8e1b96edd76c9ec288602e38d86432b16e2ae3d342b6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "3710f44b-9941-4bc4-a806-5c15d00cd807", + "message_id": "d0d6b5dc-b189-491a-804e-f52fdb9495cb", + "exported_at": "2026-09-03T14:05:21Z" + } + }, + "workflow": { + "id": "7afd1f94-6cb5-4923-9388-18851b0fba5d", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "request": { + "content": "Switch to the tab 'Text to image', then add three nodes one at a time, each with its own separate tool call and nothing else in that call: first a PrimitiveStringMultiline, then an EmptyLatentImage, then a CLIPTextEncode. Do not connect or configure them." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch focus to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "7afd1f94-6cb5-4923-9388-18851b0fba5d" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_0199bXc7oQVbgJA4aY56VcfY", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1845, 0], + "node": { + "id": 2478057798252548, + "pos": [1845, 0], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "cf6a6fdb84ee4f7dbec557f999d2fa8f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2478057798252548, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + } + ], + "at_ms": 2686 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 964, + "status": "success", + "tool_call_id": "toolu_014VjtpNeXB9q2rBXxK9bkm9", + "tool_name": "add_node" + } + }, + "at_ms": 2686 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2165, 0], + "node": { + "id": 2772376668635982, + "pos": [2165, 0], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "48a4c2f2ea8647d1be5b0817985bda87", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2772376668635982, + "class_type": "EmptyLatentImage", + "base_version": 3 + } + ], + "at_ms": 4441 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 220, + "status": "success", + "tool_call_id": "toolu_01M6Jpuof94JLfgBij4etkaN", + "tool_name": "add_node" + } + }, + "at_ms": 4441 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2485, 0], + "node": { + "id": 2514973844700532, + "pos": [2485, 0], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "da3e136d6b224cc8838a0d9aa569ede3", + "stamp": [4, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2514973844700532, + "class_type": "CLIPTextEncode", + "base_version": 4 + } + ], + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 192, + "status": "success", + "tool_call_id": "toolu_01FzkN53SHp6ePAFbmX2H8oX", + "tool_name": "add_node" + } + }, + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "All three nodes are on the \"Text to image\" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked." + } + }, + "at_ms": 9873 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1944, + "cache_read_input_tokens": 192480, + "input_tokens": 10, + "output_tokens": 322, + "total_tokens": 332 + } + } + }, + "at_ms": 9875 + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts new file mode 100644 index 000000000000..5a94dfe7d0ef --- /dev/null +++ b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts @@ -0,0 +1,59 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' + +const HUMAN_TEXT = 'a lighthouse at dawn, typed while the agent works' + +test.describe( + 'Agent conversation replay with concurrent human edits', + { tag: '@cloud' }, + () => { + test.use({ conversationCase: 'agent-rec-three-sequential-adds' }) + + test('keeps a human widget edit made between the agent adds', async ({ + agentConversation, + page + }) => { + test.setTimeout(60_000) + const { conversation } = agentConversation + const opsEntries = conversation.response + .map((entry, index) => (entry.kind === 'graph_ops' ? index : -1)) + .filter((index) => index >= 0) + expect(opsEntries.length).toBeGreaterThanOrEqual(3) + const promptText = page + .locator('[data-node-id="6"]') + .getByLabel('text', { exact: true }) + + await agentConversation.sendPrompt() + await agentConversation.replayResponse(async (index) => { + if (index !== opsEntries[0]) return + await promptText.fill(HUMAN_TEXT) + await expect + .poll(() => + agentConversation.outboundOps().some((op) => op.op === 'set_widget') + ) + .toBe(true) + }) + await agentConversation.waitForTurnComplete() + + for (const id of agentConversation.addedNodeIds()) { + await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() + } + await expect(promptText).toHaveValue(HUMAN_TEXT) + expect( + String( + agentConversation.hostGraph().nodes['6']?.widgets && + ( + agentConversation.hostGraph().nodes['6'].widgets as Record< + string, + unknown + > + ).text + ) + ).toBe(HUMAN_TEXT) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + } +) From ada8676221f87cb05c4494ff099036237140810b Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:50:38 -0700 Subject: [PATCH 37/49] test(agent): keep the row label helper private to the harness --- browser_tests/fixtures/agentConversationFixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index 216eca490087..f53e96e69d2f 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -171,7 +171,7 @@ function asRecord(value: unknown): Record { } // The panel's row label: the known-tool table, else the humanized tool name (ToolCallCard.vue). -export function toolRowLabel(name: string): string { +function toolRowLabel(name: string): string { const known = knownTool(name)?.labelKey const label = known ? enMessages.agent[ From a5c8462bc6b0127df3398a3f7e3dd8a59cea174b Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:56:54 -0700 Subject: [PATCH 38/49] test(agent): keep this PR to the recordings --- .../agentConversationConcurrentEdits.spec.ts | 59 ------------------- .../agent/agentConversationTiming.spec.ts | 30 ---------- 2 files changed, 89 deletions(-) delete mode 100644 browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts delete mode 100644 browser_tests/tests/agent/agentConversationTiming.spec.ts diff --git a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts deleted file mode 100644 index 5a94dfe7d0ef..000000000000 --- a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { expect } from '@playwright/test' - -import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' - -const HUMAN_TEXT = 'a lighthouse at dawn, typed while the agent works' - -test.describe( - 'Agent conversation replay with concurrent human edits', - { tag: '@cloud' }, - () => { - test.use({ conversationCase: 'agent-rec-three-sequential-adds' }) - - test('keeps a human widget edit made between the agent adds', async ({ - agentConversation, - page - }) => { - test.setTimeout(60_000) - const { conversation } = agentConversation - const opsEntries = conversation.response - .map((entry, index) => (entry.kind === 'graph_ops' ? index : -1)) - .filter((index) => index >= 0) - expect(opsEntries.length).toBeGreaterThanOrEqual(3) - const promptText = page - .locator('[data-node-id="6"]') - .getByLabel('text', { exact: true }) - - await agentConversation.sendPrompt() - await agentConversation.replayResponse(async (index) => { - if (index !== opsEntries[0]) return - await promptText.fill(HUMAN_TEXT) - await expect - .poll(() => - agentConversation.outboundOps().some((op) => op.op === 'set_widget') - ) - .toBe(true) - }) - await agentConversation.waitForTurnComplete() - - for (const id of agentConversation.addedNodeIds()) { - await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() - } - await expect(promptText).toHaveValue(HUMAN_TEXT) - expect( - String( - agentConversation.hostGraph().nodes['6']?.widgets && - ( - agentConversation.hostGraph().nodes['6'].widgets as Record< - string, - unknown - > - ).text - ) - ).toBe(HUMAN_TEXT) - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual(agentConversation.expectedGraph()) - }) - } -) diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts deleted file mode 100644 index 91221a185535..000000000000 --- a/browser_tests/tests/agent/agentConversationTiming.spec.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { expect } from '@playwright/test' - -import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' - -test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { - test.use({ - conversationCase: 'agent-rec-set-widget-existing', - replayTiming: 'recorded' - }) - - test('replays the frames at their recorded offsets', async ({ - agentConversation - }) => { - test.setTimeout(120_000) - const offsets = agentConversation.conversation.response.flatMap((entry) => - entry.at_ms === undefined ? [] : [entry.at_ms] - ) - const span = offsets.length ? Math.max(...offsets) : undefined - expect(span, 'the fixture carries at_ms offsets').toBeDefined() - - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() - - expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) - await expect - .poll(() => agentConversation.graphSnapshot()) - .toEqual(agentConversation.expectedGraph()) - }) -}) From 33e4e04ff1fa78d567b3d8f98eb4870fd8c14ce6 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:03:04 -0700 Subject: [PATCH 39/49] test(agent): timing and concurrent-edit replays live with the code, defined only when their recording exists --- .../fixtures/data/agent/agentConversation.ts | 7 ++ .../agentConversationConcurrentEdits.spec.ts | 64 +++++++++++++++++++ .../agent/agentConversationTiming.spec.ts | 33 ++++++++++ 3 files changed, 104 insertions(+) create mode 100644 browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts create mode 100644 browser_tests/tests/agent/agentConversationTiming.spec.ts diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 0fd0ccc36916..31136126997f 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -107,6 +107,13 @@ export function listRecordedConversations(): string[] { .sort() } +// Specs pinned to one recording skip on a branch that carries the code but not the data. +export function hasAgentConversation(caseId: string): boolean { + return existsSync( + fileURLToPath(new URL(`./conversations/${caseId}.json`, import.meta.url)) + ) +} + export function loadAgentConversation(caseId: string): AgentConversation { const file = fileURLToPath( new URL(`./conversations/${caseId}.json`, import.meta.url) diff --git a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts new file mode 100644 index 000000000000..be045eae74ac --- /dev/null +++ b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts @@ -0,0 +1,64 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { hasAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +const HUMAN_TEXT = 'a lighthouse at dawn, typed while the agent works' + +// The recording lands with the data PR; without it there is nothing to replay. +if (hasAgentConversation('agent-rec-three-sequential-adds')) + test.describe( + 'Agent conversation replay with concurrent human edits', + { tag: '@cloud' }, + () => { + test.use({ conversationCase: 'agent-rec-three-sequential-adds' }) + + test('keeps a human widget edit made between the agent adds', async ({ + agentConversation, + page + }) => { + test.setTimeout(60_000) + const { conversation } = agentConversation + const opsEntries = conversation.response + .map((entry, index) => (entry.kind === 'graph_ops' ? index : -1)) + .filter((index) => index >= 0) + expect(opsEntries.length).toBeGreaterThanOrEqual(3) + const promptText = page + .locator('[data-node-id="6"]') + .getByLabel('text', { exact: true }) + + await agentConversation.sendPrompt() + await agentConversation.replayResponse(async (index) => { + if (index !== opsEntries[0]) return + await promptText.fill(HUMAN_TEXT) + await expect + .poll(() => + agentConversation + .outboundOps() + .some((op) => op.op === 'set_widget') + ) + .toBe(true) + }) + await agentConversation.waitForTurnComplete() + + for (const id of agentConversation.addedNodeIds()) { + await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() + } + await expect(promptText).toHaveValue(HUMAN_TEXT) + expect( + String( + agentConversation.hostGraph().nodes['6']?.widgets && + ( + agentConversation.hostGraph().nodes['6'].widgets as Record< + string, + unknown + > + ).text + ) + ).toBe(HUMAN_TEXT) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + } + ) diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts new file mode 100644 index 000000000000..fc6e180f6c0f --- /dev/null +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -0,0 +1,33 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { hasAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +// The recording lands with the data PR; without it there is nothing to replay. +if (hasAgentConversation('agent-rec-set-widget-existing')) + test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { + test.use({ + conversationCase: 'agent-rec-set-widget-existing', + replayTiming: 'recorded' + }) + + test('replays the frames at their recorded offsets', async ({ + agentConversation + }) => { + test.setTimeout(120_000) + const offsets = agentConversation.conversation.response.flatMap( + (entry) => (entry.at_ms === undefined ? [] : [entry.at_ms]) + ) + const span = offsets.length ? Math.max(...offsets) : undefined + expect(span, 'the fixture carries at_ms offsets').toBeDefined() + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + }) From cdf46cb7625cafad8f356a9fd98c78ecbe392e1e Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:11:49 -0700 Subject: [PATCH 40/49] test(agent): read replay ops through their own types and pin the coalesced row multiplier Drops the record widening over typed ops, asserts the xN badge per same-tool run, matches row labels exactly, and word-bounds the update count so 1 no longer matches 11. --- .../fixtures/agentConversationFixture.ts | 59 ++++++++++--------- .../agent/agentConversationReplay.spec.ts | 17 +++++- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index f53e96e69d2f..ccab82bd13f0 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -145,7 +145,7 @@ class HostDoc { } } -interface NodeBody { +type NodeBody = { id: number | string type: string title?: string @@ -164,12 +164,6 @@ interface RecordedWidgetValue { value: string | number } -function asRecord(value: unknown): Record { - return typeof value === 'object' && value !== null - ? (value as Record) - : {} -} - // The panel's row label: the known-tool table, else the humanized tool name (ToolCallCard.vue). function toolRowLabel(name: string): string { const known = knownTool(name)?.labelKey @@ -298,15 +292,11 @@ class AgentConversationHarness { // Doc-id filter: a stray template node must not pin the template here. private nodeBodies(): NodeBody[] { - const seed = ( - this.conversation.workflow.seed as unknown as { nodes: NodeBody[] } - ).nodes + const seed = this.conversation.workflow.seed.nodes as NodeBody[] const added = this.conversation.response.flatMap((entry) => entry.kind === 'graph_ops' ? entry.ops.flatMap((op) => - asRecord(op).op === 'add_node' - ? [asRecord(asRecord(op).node) as unknown as NodeBody] - : [] + op.op === 'add_node' ? [op.node as NodeBody] : [] ) : [] ) @@ -351,18 +341,31 @@ class AgentConversationHarness { // The panel starts a new tool group whenever thinking or text interrupts the calls. // ToolCallGroup.vue merges consecutive calls of the same tool into one row. - toolRowLabels(): Map { - const counts = new Map() + // One row per run of same-tool calls; the card shows ×N for a run longer than one. + toolRowCounts(): Array<{ label: string; times: number; rows: number }> { + const rows = new Map< + string, + { label: string; times: number; rows: number } + >() for (const group of this.toolCallGroups()) { - let previous: string | undefined + const runs: Array<{ name: string; times: number }> = [] for (const call of group) { - if (call.name === previous) continue - previous = call.name - const label = toolRowLabel(call.name) - counts.set(label, (counts.get(label) ?? 0) + 1) + const previous = runs.at(-1) + if (previous?.name === call.name) previous.times += 1 + else runs.push({ name: call.name, times: 1 }) + } + for (const { name, times } of runs) { + const key = `${name}/${times}` + const row = rows.get(key) ?? { + label: toolRowLabel(name), + times, + rows: 0 + } + row.rows += 1 + rows.set(key, row) } } - return counts + return [...rows.values()] } toolCallGroups(): RecordedToolCall[][] { @@ -397,12 +400,14 @@ class AgentConversationHarness { const latest = new Map() for (const entry of this.conversation.response) { if (entry.kind !== 'graph_ops') continue - for (const rawOp of entry.ops) { - const op = asRecord(rawOp) + for (const op of entry.ops) { if (op.op !== 'set_widget') continue const nodeId = String(op.node_id) - const widget = String(op.widget) - const value = asRecord(graph.nodes[nodeId]?.widgets)[widget] + const widget = op.widget + const widgets = graph.nodes[nodeId]?.widgets as + | Record + | undefined + const value = widgets?.[widget] if (typeof value === 'string' || typeof value === 'number') latest.set(`${nodeId}/${widget}`, { nodeId, widget, value }) } @@ -413,9 +418,7 @@ class AgentConversationHarness { addedNodeIds(): string[] { const graph = this.host.graph() const seedIds = new Set( - ( - this.conversation.workflow.seed as unknown as { nodes: NodeBody[] } - ).nodes.map((node) => String(node.id)) + this.conversation.workflow.seed.nodes.map((node) => String(node.id)) ) return this.nodeBodies() .map((body) => String(body.id)) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index 8c20bbd5a8e9..dbbd588a9dcc 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -38,8 +38,19 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { ) if (succeeded) await button.click() } - for (const [label, count] of agentConversation.toolRowLabels()) { - await expect(toolRows.filter({ hasText: label })).toHaveCount(count) + for (const { + label, + times, + rows + } of agentConversation.toolRowCounts()) { + const named = toolRows.filter({ + has: page.getByText(label, { exact: true }) + }) + await expect( + named.filter( + times > 1 ? { hasText: `×${times}` } : { hasNotText: '×' } + ) + ).toHaveCount(rows) } await expect( @@ -54,7 +65,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { 'connected' ) await expect(panel.getByTestId('agent-crdt-status')).toContainText( - `${agentConversation.expectedUpdateCount()} updates` + new RegExp(`\\b${agentConversation.expectedUpdateCount()} updates`) ) for (const id of agentConversation.addedNodeIds()) { From 8afc472d3bde95d252b6a430263c0b04877651dd Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:19:53 -0700 Subject: [PATCH 41/49] test(agent): drop two recordings whose ops the zimage case already covers --- .../agent-rec-second-prompt-encoder.json | 392 ------------------ .../agent-rec-string-node-into-prompt.json | 373 ----------------- 2 files changed, 765 deletions(-) delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json delete mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json deleted file mode 100644 index ec5624d316c2..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-second-prompt-encoder.json +++ /dev/null @@ -1,392 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-second-prompt-encoder", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 654dc7e4-c2e2-4b60-9e28-cfd88839a85c; message 72291d49-42f3-4b78-8a73-80e5cb1b9d2d; workflow b2c94bcb-c02f-4456-99ce-65708c66b341 (seeded by throwaway turn c575f8f1-ee42-45fc-99c0-c285b7c3739a; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['023ee47a-fc32-4bec-8e1f-42a6327bcc0f', 'f4b41fff-7fec-41fd-bedd-a58947686567', '165127e2-793e-4ca0-bc3f-62880660f572', '11ff34fb-553c-4466-95f7-ef010c8983a9', 'a7ee8b67-dca2-4293-9c34-993983f338b6']; rows agent-rec-second-prompt-encoder.t-a1.rows.json; raw capture sha256 20ffafacee4786b5fe000c1d7cd73c23222b2d53309664eef45085c1663e01ed", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "654dc7e4-c2e2-4b60-9e28-cfd88839a85c", - "message_id": "72291d49-42f3-4b78-8a73-80e5cb1b9d2d", - "exported_at": "2026-09-03T12:40:54Z" - } - }, - "workflow": { - "id": "b2c94bcb-c02f-4456-99ce-65708c66b341", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - }, - "EmptyLatentImage": { - "widget_order": ["width", "height", "batch_size"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and wire its conditioning into the KSampler's negative input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "b2c94bcb-c02f-4456-99ce-65708c66b341" - } - }, - "at_ms": 8 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 9, - "status": "success", - "tool_call_id": "toolu_01HDMTFUx2kkyMo87wtNzmze", - "tool_name": "switch_tab" - } - }, - "at_ms": 11 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 290, - "status": "success", - "tool_call_id": "toolu_01RajshxPyUhr7ZyCEJ5ihFS", - "tool_name": "print_workflow" - } - }, - "at_ms": 2157 - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 2143974170219146, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "f1d3ff4affc84a7bb0e2945349e22c8b", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2143974170219146, - "class_type": "CLIPTextEncode", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "d9c4827370a24567975399a1b6c842f9", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1085915564088321, - "to_node": 2143974170219146, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "fdfd2ededd164148bd7138f8a894d333", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3735142525577434, - "to_node": 3, - "to_slot": 2, - "from_node": 2143974170219146, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c872f1a5e80447768012eab52a6a384f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "blurry, low quality, watermark", - "widget": "text", - "node_id": 2143974170219146, - "base_version": 2 - } - ], - "at_ms": 7248 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 270, - "status": "success", - "tool_call_id": "toolu_013BwRtRP1QkTwF6RRQhkwsz", - "tool_name": "apply_ops" - } - }, - "at_ms": 7248 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 200, - "status": "success", - "tool_call_id": "toolu_01EmHWYrRmyiQ1tjXGHAiuSc", - "tool_name": "print_workflow" - } - }, - "at_ms": 10148 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 196, - "status": "error", - "tool_call_id": "toolu_015XiAA4HvEjNkfWtwvdhGPY", - "tool_name": "validate" - } - }, - "at_ms": 15164 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the negative prompt encoder is on your \"Text to image\" canvas, taking CLIP from the checkpoint loader and feeding the sampler's negative input (I seeded it with \"blurry, low quality, \" — change it to whatever you like).\n\nTwo things that were already there before this edit and will stop a run: the sampler has nothing wired into its latent input, and the checkpoint it points at isn't installed on this server. Want me to add the empty-latent node and switch to an available checkpoint?" - } - }, - "at_ms": 21622 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2527, - "cache_read_input_tokens": 232384, - "input_tokens": 12, - "output_tokens": 1038, - "total_tokens": 1050 - } - } - }, - "at_ms": 21627 - } - ] -} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json deleted file mode 100644 index 25ebb7e0d221..000000000000 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-string-node-into-prompt.json +++ /dev/null @@ -1,373 +0,0 @@ -{ - "schema_version": "agent-conversation.v1", - "source": { - "repo": "Comfy-Org/ComfyUI_frontend", - "suite": "agent", - "case_id": "agent-rec-string-node-into-prompt", - "response_side": "recorded", - "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2ff49d22-e6e9-4fdf-9214-492e06f77dd5; message fde9fd6d-c9a8-4ed4-a524-5f993f7b5cbf; workflow 75ffed90-1d46-4b37-9bbf-abdf3d270811 (seeded by throwaway turn b53fedff-60c8-4ae1-bb6d-b68f92d7ee97; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['3ce12d96-f9cb-45a1-b33a-075387f0cb28', '66a65cf7-228e-41ee-896a-165f96364a91', '25f41c6c-ff4b-4d20-a539-a21550ec4897', 'fc46022e-836e-44c7-9e4a-8a54653bda73', '60494f00-0fc3-4dde-8dd5-ea8356e77416']; rows agent-rec-string-node-into-prompt.t-a1.rows.json; raw capture sha256 94453e3cee7f5b7765266bbee7bcf45f02e53ca11e889a6f3f28376b859c132a", - "capture": { - "backend": "Comfy-Org/cloud", - "thread_id": "2ff49d22-e6e9-4fdf-9214-492e06f77dd5", - "message_id": "fde9fd6d-c9a8-4ed4-a524-5f993f7b5cbf", - "exported_at": "2026-09-03T12:40:53Z" - } - }, - "workflow": { - "id": "75ffed90-1d46-4b37-9bbf-abdf3d270811", - "name": "Text to image", - "catalog": { - "types": { - "CheckpointLoaderSimple": { - "widget_order": ["ckpt_name"] - }, - "CLIPTextEncode": { - "widget_order": ["text"] - }, - "KSampler": { - "widget_order": [ - "seed", - "control_after_generate", - "steps", - "cfg", - "sampler_name", - "scheduler", - "denoise" - ] - }, - "VAEDecode": { - "widget_order": [] - }, - "SaveImage": { - "widget_order": ["filename_prefix"] - }, - "PrimitiveStringMultiline": { - "widget_order": ["value"] - }, - "EmptyLatentImage": { - "widget_order": ["width", "height", "batch_size"] - } - } - }, - "seed": { - "nodes": [ - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [0, 0], - "size": [315, 98], - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [3] - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [1] - }, - { - "name": "VAE", - "type": "VAE", - "links": [5] - } - ], - "widgets_values": ["sd_xl_base_1.0.safetensors"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "title": "Positive prompt", - "pos": [400, 0], - "size": [400, 200], - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": null - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [2] - } - ], - "widgets_values": ["a photo of a pier"] - }, - { - "id": 3, - "type": "KSampler", - "pos": [850, 0], - "size": [315, 262], - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 2 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": null - }, - { - "name": "latent_image", - "type": "LATENT", - "link": null - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [4] - } - ], - "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1200, 0], - "size": [210, 46], - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 4 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [6] - } - ], - "widgets_values": [] - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1450, 0], - "size": [315, 270], - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 6 - } - ], - "outputs": [], - "widgets_values": ["ComfyUI"] - } - ], - "links": [ - [1, 4, 1, 6, 0, "CLIP"], - [2, 6, 0, 3, 1, "CONDITIONING"], - [3, 4, 0, 3, 0, "MODEL"], - [4, 3, 0, 8, 0, "LATENT"], - [5, 4, 2, 8, 1, "VAE"], - [6, 8, 0, 9, 0, "IMAGE"] - ] - } - }, - "request": { - "content": "Switch to the tab 'Text to image', then add a PrimitiveStringMultiline node with the text 'a photo of a pier at sunset' and wire its output into the positive prompt's text input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and look at what's there." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "75ffed90-1d46-4b37-9bbf-abdf3d270811" - } - }, - "at_ms": 7 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 9, - "status": "success", - "tool_call_id": "toolu_012df4k3MtbwuCzgUZssmJAe", - "tool_name": "switch_tab" - } - }, - "at_ms": 9 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 623, - "status": "success", - "tool_call_id": "toolu_01Mo1YBLCyoku5dm2D7D6LTH", - "tool_name": "print_workflow" - } - }, - "at_ms": 2586 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 358, - "status": "success", - "tool_call_id": "toolu_01XxtrwYzdvG6VrUXaY2j33W", - "tool_name": "show_node" - } - }, - "at_ms": 7323 - }, - { - "kind": "graph_ops", - "ops": [ - { - "op": "add_node", - "pos": [80, 120], - "node": { - "id": 4111844109938098, - "pos": [80, 120], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] - }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7b4e35b41b8b4d97aebd5b81e90b97b4", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4111844109938098, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 - }, - { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "cf0352bcf66f4a07a81f4ce09133f1e6", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a photo of a pier at sunset", - "widget": "value", - "node_id": 4111844109938098, - "base_version": 2 - }, - { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b3ace060146b432c996e3435be9f05e7", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 883895853831334, - "to_node": 6, - "to_slot": 1, - "from_node": 4111844109938098, - "from_slot": 0, - "link_type": "STRING", - "base_version": 2 - } - ], - "at_ms": 10862 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 216, - "status": "success", - "tool_call_id": "toolu_01VG1kKqa1wQJqK4828x4WnX", - "tool_name": "apply_ops" - } - }, - "at_ms": 10862 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 232, - "status": "success", - "tool_call_id": "toolu_018Eu9J62defBJuvMyb1PbEH", - "tool_name": "print_workflow" - } - }, - "at_ms": 13023 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the text \"a photo of a pier at sunset\" now feeds the positive prompt in your **Text to image** tab.\n\nHeads up: that graph was already missing a negative prompt and an empty latent going into the sampler, so it won't run as-is. Want me to add those?" - } - }, - "at_ms": 17136 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2182, - "cache_read_input_tokens": 231753, - "input_tokens": 12, - "output_tokens": 705, - "total_tokens": 717 - } - } - }, - "at_ms": 17141 - } - ] -} From dd27afbeeafc633dcd9fb267a24daf5b2cafeb11 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 07:58:43 -0700 Subject: [PATCH 42/49] test(agent): record a conversation fixture with one command The two throwaway scripts behind the recorded fixtures become one TypeScript command that drives both turns, applies the assembler gates and writes the fixture through the existing exporter. The gates are a pure function, so the colocated test covers each refusal without opening a socket or touching the filesystem, and the intermediate capture, rows and receipt land in a recordings/ directory beside the fixture rather than in conversations/, which the replay scans for cases. --- browser_tests/fixtures/data/agent/README.md | 65 ++ scripts/agentConversationRecord.test.ts | 566 ++++++++++++++ scripts/agentConversationRecord.ts | 808 ++++++++++++++++++++ 3 files changed, 1439 insertions(+) create mode 100644 scripts/agentConversationRecord.test.ts create mode 100644 scripts/agentConversationRecord.ts diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md index 8ae7ba93eda7..40c4f7eb51bc 100644 --- a/browser_tests/fixtures/data/agent/README.md +++ b/browser_tests/fixtures/data/agent/README.md @@ -11,6 +11,68 @@ Do not write `graph_ops` by hand or relabel a synthesized response. The fixture schema rejects `response_side: recorded` without a cloud thread/message ID and export timestamp. +## Recording a conversation + +One command records a turn against a running agent, applies the gates below, +and writes the fixture through the exporter: + +```bash +AGENT_CLOUD_SHA=$(git -C ../cloud rev-parse HEAD) AGENT_MODEL=claude-opus-5 \ +AGENT_M2M_SECRET_FILE=/path/to/m2m.secret \ +REC_WORKSPACE_ID=... REC_USER_ID=... \ +pnpm exec tsx scripts/agentConversationRecord.ts \ + agent-rec- \ + "Switch to the tab 'Text to image', then ." \ + browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json \ + --out browser_tests/fixtures/data/agent/conversations/agent-rec-.json +``` + +The third positional argument is an existing conversation fixture, used only for +its `workflow` block: the recorder seeds a throwaway turn with that graph and +catalog, then records a second turn against it. The prompt must switch tabs +first, because the replay subscribes to the document only on an +`agent_active_tab` frame and switching to an already-focused tab publishes +nothing. + +Recording runs against the lean stack: frames come from its Redis channel +and rows from Postgres. That is the only path, because the graph operations +a replay asserts exist only where the doc host writes audit rows. + +The recording agent shells out to `comfy-cli` for its read and edit tools inside +a sandbox that overrides `HOME`, so point `COMFY_BIN` at an installation that +works without a home directory. A turn recorded without it produces an answer +reporting tool failures rather than a usable fixture. + +Environment, all optional except the two provenance values: + +| Variable | Purpose | +| ------------------------------------- | ------------------------------------------------------------------ | +| `AGENT_CLOUD_SHA`, `AGENT_MODEL` | recorded into the fixture note; both required | +| `AGENT_FULLSTACK_URL` | agent base URL, default `http://127.0.0.1:8086` | +| `AGENT_M2M_SECRET_FILE` | path to the shared secret, read at runtime and never printed | +| `AGENT_WORKSPACE_ID`, `AGENT_USER_ID` | identity headers; `REC_WORKSPACE_ID` / `REC_USER_ID` also accepted | +| `AGENT_REDIS_EXEC`, `AGENT_PG_EXEC` | commands that reach the stack's Redis and Postgres | +| `AGENT_ATTEMPT` | attempt label in every artifact name; defaults to a UTC stamp | +| `AGENT_TURN_TIMEOUT` | milliseconds to wait for the turn, default 180000 | + +Alongside the fixture the command writes a `recordings/` directory holding the +raw frames, the retrieved rows, the intermediate `agent-backend-capture.v1` +document and a receipt (turn IDs, parent rows, applied ops, dropped-frame +counts, artifact hashes). Those are provenance for the recording, not committed +fixtures. Use `--work ` to put them elsewhere. + +A refused recording names the gate it failed and appends the reason to +`recordings/.refused.jsonl`. Re-record with a new `AGENT_ATTEMPT` +rather than editing a capture by hand. + +Then run the replay, which picks up every recorded fixture in this directory: + +```bash +PLAYWRIGHT_TEST_URL=http://localhost:5173 DISTRIBUTION=cloud \ + pnpm exec playwright test browser_tests/tests/agent/agentConversationReplay.spec.ts \ + --project=cloud -g '' +``` + ## Capture Record the `/ws` frames for one eval turn and remove unrelated frame types. Keep @@ -48,6 +110,9 @@ pnpm exec tsx scripts/agentConversationCapture.ts \ browser_tests/fixtures/data/agent/conversations/.json ``` +The recorder above calls this exporter for you; run it directly when you +already have a capture document. + The exporter strips turn identity from frames (the replay mints its own), inserts each durably accepted op before its terminal tool-call frame, and fails if any frame belongs to another turn, an accepted op is missing from the diff --git a/scripts/agentConversationRecord.test.ts b/scripts/agentConversationRecord.test.ts new file mode 100644 index 000000000000..fa46d427f167 --- /dev/null +++ b/scripts/agentConversationRecord.test.ts @@ -0,0 +1,566 @@ +import { describe, expect, it } from 'vitest' + +import { exportAgentConversation } from './agentConversationCapture' +import type { + AssembleInput, + NormalizedRows, + ParentRow, + RawCapture, + RecordedFrame, + SeedFixture +} from './agentConversationRecord' +import { assembleCapture } from './agentConversationRecord' + +const THREAD = 'thread-1' +const MESSAGE = 'message-1' +const SEED_MESSAGE = 'seed-message-1' +const WORKFLOW = '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01' + +const turnFrame = ( + type: string, + data: Record, + at_ms: number +): RecordedFrame => ({ + type, + data: { thread_id: THREAD, message_id: MESSAGE, ...data }, + at_ms +}) + +const frames = (): RecordedFrame[] => [ + turnFrame('agent_thinking', { delta: 'switching' }, 1_700_000_000_000), + turnFrame( + 'agent_active_tab', + { workflow_id: WORKFLOW, name: 'Text to image' }, + 1_700_000_000_100 + ), + turnFrame( + 'agent_tool_call', + { tool_call_id: 'tool-1', tool_name: 'apply_ops', status: 'success' }, + 1_700_000_000_200 + ), + turnFrame('agent_message_done', {}, 1_700_000_000_300) +] + +const seed = (): SeedFixture => ({ + workflow: { + name: 'Text to image', + catalog: { types: { KSampler: { widget_order: ['steps'] } } }, + seed: { + nodes: [ + { id: 3, type: 'CheckpointLoaderSimple' }, + { id: 4, type: 'KSampler' } + ], + links: [] + } + } +}) + +const addNodeOp = { + op: 'add_node', + op_id: 'op-1', + node_id: 10, + class_type: 'KSampler' +} + +const parent = (overrides: Partial = {}): ParentRow => ({ + id: 'parent-1', + tool_call_id: 'tool-1', + tool_name: 'apply_ops', + status: 'ok', + workflow_id: WORKFLOW, + result: { ok: true, data: { ops: [addNodeOp] } }, + children: [{ op_id: 'op-1', status: 'ok' }], + ...overrides +}) + +const raw = (overrides: Partial = {}): RawCapture => ({ + case_id: 'agent-rec-example', + attempt: 'a1', + prompt: "Switch to the tab 'Text to image', then add a sampler.", + base: 'http://127.0.0.1:8086', + frame_source: 'redis SUBSCRIBE channel:ws::u:', + channel: 'channel:ws:w-secret:u:u-secret', + seed_sha256: 'a'.repeat(64), + seed_name: 'Text to image', + seed_node_ids: [3, 4], + saw_stream: true, + stream_closed: false, + seed_turn: { + status: 202, + body: { thread_id: 'seed-thread', message_id: SEED_MESSAGE } + }, + seed_workflow_id: WORKFLOW, + accepted: { + status: 202, + body: { thread_id: THREAD, message_id: MESSAGE, workflow_id: 'blank-wf' } + }, + saw_done: true, + timed_out: false, + frames: frames(), + rows_artifact: 'rows.json', + retrieval: { kind: 'postgres-json' }, + error: null, + ...overrides +}) + +const rows = (overrides: Partial = {}): NormalizedRows => ({ + parents: [parent()], + draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] }, + retrieval: { kind: 'postgres-json' }, + path: '/tmp/agent-rec-example.a1.rows.json', + sha256: 'b'.repeat(64), + ...overrides +}) + +const input = (overrides: Partial = {}): AssembleInput => ({ + raw: raw(), + rows: rows(), + seed: { json: seed(), path: 'seed.json', sha256: 'a'.repeat(64) }, + provenance: { + cloudSha: '9dc1da7', + model: 'claude-opus-5', + exportedAt: '2026-09-03T01:00:00.000Z' + }, + rawSha256: 'c'.repeat(64), + rawPath: '/tmp/agent-rec-example.a1.raw.json', + ...overrides +}) + +describe('assembleCapture', () => { + it('keeps the turn frames with their receipt times and emits the applied ops', () => { + const { capture, receipt } = assembleCapture(input()) + + expect(capture.workflow.id).toBe(WORKFLOW) + expect(capture.capture).toMatchObject({ + backend: 'Comfy-Org/cloud', + thread_id: THREAD, + message_id: MESSAGE + }) + expect(capture.frames.map((frame) => frame.type)).toEqual([ + 'agent_thinking', + 'agent_active_tab', + 'agent_tool_call', + 'agent_message_done' + ]) + expect(capture.frames.map((frame) => frame.at_ms)).toEqual([ + 1_700_000_000_000, 1_700_000_000_100, 1_700_000_000_200, 1_700_000_000_300 + ]) + expect(capture.tool_calls).toEqual([ + { + tool_call_id: 'tool-1', + result: { ok: true, data: { ops: [addNodeOp] } }, + applied_op_ids: ['op-1'] + } + ]) + expect(receipt).toMatchObject({ + added_node_ids: ['10'], + deleted_node_ids: [], + unexplained_draft_node_ids: [], + mutating_parents: ['parent-1'], + child_statuses: { ok: 1 }, + frames_kept: 4 + }) + }) + + it('produces a capture the exporter turns into a recorded conversation', () => { + const { capture } = assembleCapture(input()) + const conversation = exportAgentConversation(capture) + + expect(conversation.source.response_side).toBe('recorded') + expect( + conversation.response.filter((entry) => entry.kind === 'graph_ops') + ).toHaveLength(1) + }) + + it('records the note without the concrete redis channel', () => { + const { capture, receipt } = assembleCapture(input()) + + expect(capture.source.note).toContain('channel:ws::u:') + expect(capture.source.note).not.toContain('w-secret') + expect(receipt.channel).toContain('w-secret') + }) + + it('parses a result the backend handed back as a JSON string', () => { + const { capture } = assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: JSON.stringify({ ok: true, data: { ops: [addNodeOp] } }) + }) + ] + }) + }) + ) + + expect(capture.tool_calls[0].applied_op_ids).toEqual(['op-1']) + }) + + it('accepts a delete_node whose node id is zero', () => { + const { receipt } = assembleCapture( + input({ + seed: { + json: { + workflow: { + ...seed().workflow, + seed: { nodes: [{ id: 0, type: 'KSampler' }], links: [] } + } + }, + path: 'seed.json', + sha256: 'a'.repeat(64) + }, + raw: raw({ seed_node_ids: [0] }), + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-1', node_id: 0 }] + } + } + }) + ], + draft: { nodes: [], links: [] } + }) + }) + ) + + expect(receipt.deleted_node_ids).toEqual(['0']) + }) + + it('accepts a batch the document rejected as zero applied ops', () => { + const { receipt } = assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + status: 'error', + children: [{ op_id: 'op-1', status: 'error' }] + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }], links: [] } + }) + }) + ) + + expect(receipt.mutating_parents).toEqual([]) + expect(receipt.child_statuses).toEqual({ error: 1 }) + }) + + it('accepts a read tool that echoes ops without writing child rows', () => { + const { receipt } = assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + tool_name: 'plan_path', + children: [], + result: { ok: true, data: { ops: [addNodeOp] } } + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }], links: [] } + }) + }) + ) + + expect(receipt.parent_rows[0].applied_op_ids).toEqual([]) + }) + + it('drops the ambient heartbeat and a non-replay turn frame into buckets', () => { + const { receipt } = assembleCapture( + input({ + raw: raw({ + frames: [ + { + type: 'draft_version', + data: { workflow_id: WORKFLOW, version: 3 } + }, + turnFrame('draft_patch', { seq: 1 }, 1_700_000_000_050), + { + type: 'agent_message_delta', + data: { + thread_id: 'seed-thread', + message_id: SEED_MESSAGE, + delta: 'ack' + } + }, + ...frames() + ] + }) + }) + ) + + expect(receipt.frames_dropped).toEqual({ + 'type:draft_version': 1, + 'type:draft_patch': 1, + seed_turn: 1 + }) + expect(receipt.frames_kept).toBe(4) + }) + + it('refuses a mutating call whose mutation never reached the doc host', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ children: [] })] }) }) + ) + ).toThrow('reports a document mutation but has NO audit child rows') + }) + + it('refuses an applied add_node whose class is outside the seed catalog', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ ...addNodeOp, class_type: 'LatentUpscaleBy' }] + } + } + }) + ] + }) + }) + ) + ).toThrow('are not in the seed catalog') + }) + + it('refuses an applied parent row that names another workflow', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ workflow_id: null })] }) }) + ) + ).toThrow('not the seeded workflow') + }) + + it('refuses an unparseable socket payload', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: [ + { type: '__raw__', data: { payload: 'not json' } }, + ...frames() + ] + }) + }) + ) + ).toThrow('unparseable socket payload') + }) + + it('refuses rows whose tool calls disagree with the recorded frames', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent(), + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + children: [], + result: { ok: true } + }) + ] + }) + }) + ) + ).toThrow('disagree; the rows are not this turn') + }) + + it('refuses an applied op that its parent result never echoed', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ children: [{ op_id: 'op-missing', status: 'ok' }] }) + ] + }) + }) + ) + ).toThrow('are not echoed in its result') + }) + + it('refuses applied ops whose parent recorded no result at all', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ result: null })] }) }) + ) + ).toThrow('has applied ops but a NULL result') + }) + + it('refuses a parent result that is not a JSON object', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [parent({ result: [1, 2], children: [] })] + }) + }) + ) + ).toThrow('not an object; the exporter requires an object') + }) + + it('refuses an echoed op kind outside the exporter frozen set', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [addNodeOp, { op: 'reset_doc', op_id: 'op-2' }] + } + } + }) + ] + }) + }) + ) + ).toThrow('outside the exporter frozen set') + }) + + it('refuses a non-object echoed op entry', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { ok: true, data: { ops: [addNodeOp, 'bogus'] } } + }) + ] + }) + }) + ) + ).toThrow('non-object op entry') + }) + + it('refuses an applied delete_node without a node id', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { ops: [{ op: 'delete_node', op_id: 'op-1' }] } + } + }) + ] + }) + }) + ) + ).toThrow('applied delete_node without node_id') + }) + + it('refuses an attempt label that would collide across attempts', () => { + expect(() => assembleCapture(input({ raw: raw({ attempt: '' }) }))).toThrow( + 'is not [A-Za-z0-9_-]+' + ) + }) + + it('refuses a seed the driver did not use', () => { + expect(() => + assembleCapture(input({ raw: raw({ seed_node_ids: [3, 4, 99] }) })) + ).toThrow('but the seed fixture given here has') + }) + + it('refuses a turn with no active tab frame', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: frames().filter( + (frame) => frame.type !== 'agent_active_tab' + ) + }) + }) + ) + ).toThrow('no agent_active_tab frame in this turn') + }) + + it('refuses a turn whose last frame is not the done frame', () => { + expect(() => + assembleCapture(input({ raw: raw({ frames: frames().slice(0, -1) }) })) + ).toThrow('not agent_message_done') + }) + + it('refuses an agent frame type the replay cannot validate', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: [turnFrame('agent_ask', {}, 1_700_000_000_050), ...frames()] + }) + }) + ) + ).toThrow('outside the replay union') + }) + + it('refuses a draft that lost a seed node nothing deleted', () => { + expect(() => + assembleCapture( + input({ rows: rows({ draft: { nodes: [{ id: 3 }], links: [] } }) }) + ) + ).toThrow('lacks seed node ids') + }) + + it('refuses a draft that still holds a deleted node', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-1', node_id: 3 }] + } + } + }) + ] + }) + }) + ) + ).toThrow('still holds node ids') + }) + + it('refuses a turn the backend never accepted', () => { + expect(() => + assembleCapture( + input({ raw: raw({ accepted: { status: 500, body: null } }) }) + ) + ).toThrow('recorded turn not accepted') + }) + + it('refuses a recording whose frame stream never opened', () => { + expect(() => + assembleCapture(input({ raw: raw({ saw_stream: false }) })) + ).toThrow('frame stream never opened') + }) + + it('refuses a recording with no terminal done frame observed', () => { + expect(() => + assembleCapture(input({ raw: raw({ saw_done: false }) })) + ).toThrow('never arrived') + }) + + it('refuses a seeded workflow the active tab never named', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ seed_workflow_id: '11111111-2222-4333-8444-555555555555' }) + }) + ) + ).toThrow('is not the seeded workflow') + }) +}) diff --git a/scripts/agentConversationRecord.ts b/scripts/agentConversationRecord.ts new file mode 100644 index 000000000000..4ff5bd49df21 --- /dev/null +++ b/scripts/agentConversationRecord.ts @@ -0,0 +1,808 @@ +#!/usr/bin/env tsx + +import { execFileSync, spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { basename, dirname, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' + +import type { AgentBackendCapture } from './agentConversationCapture' +import { exportAgentConversation } from './agentConversationCapture' + +const USAGE = + 'usage: pnpm exec tsx scripts/agentConversationRecord.ts "" --out [--work ]' + +const REPLAYED_FRAMES = + 'agent_thinking agent_tool_call agent_message_delta agent_message_done agent_active_tab'.split( + ' ' + ) + +// boundary.Classify's ActionEdit set minus the tab tools, which move focus +// rather than the document. +const MUTATING_TOOLS = + 'apply_ops add_node connect set_widget delete_node delete_nodes clear_canvas apply_recipe generate_workflow reset_doc open_workflow get_template use_asset_as_input'.split( + ' ' + ) + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +const STACK = + 'NON-standalone local full stack (Postgres + doc host, M2M identity headers)' + +export type Retrieval = Record + +export interface TurnAck { + status: number + body: Record | null +} + +export interface RecordedFrame { + type: string + data: Record + at_ms?: number +} + +export interface RawCapture { + case_id: string + attempt: string + prompt: string + base: string + frame_source: string + channel: string + seed_sha256: string + seed_name: string + seed_node_ids: unknown[] + saw_stream: boolean + stream_closed: boolean + seed_turn: TurnAck | null + seed_workflow_id: string | null + accepted: TurnAck | null + saw_done: boolean + timed_out: boolean + frames: RecordedFrame[] + rows_artifact: string | null + retrieval: Retrieval | null + error: string | null +} + +export interface ParentRow { + id: string + tool_call_id: string | null + tool_name: string | null + status: string | null + workflow_id: string | null + result: unknown + children: Array<{ op_id: string | null; status: string | null }> +} + +export interface NormalizedRows { + parents: ParentRow[] + draft: unknown + retrieval: Retrieval + path: string + sha256: string +} + +export interface SeedFixture { + workflow: { name: string; catalog: WidgetCatalog; seed: WorkflowJSON } +} + +export interface AssembleInput { + raw: RawCapture + rows: NormalizedRows + seed: { json: SeedFixture; path: string; sha256: string } + provenance: { cloudSha: string; model: string; exportedAt: string } + rawSha256: string + rawPath: string +} + +interface TurnIds { + threadId: string + messageId: string + workflowId: string +} + +// A refused recording is an expected outcome: main logs it and exits 1. +export class RecordRefusal extends Error {} + +function refuse(reason: string): never { + throw new RecordRefusal(reason) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +const list = (values: Iterable): string => + [...values].map(String).sort().join(', ') + +const text = (value: unknown): string | null => + value === null || value === undefined ? null : String(value) + +const sameSet = (left: Set, right: Set): boolean => + left.size === right.size && [...left].every((value) => right.has(value)) + +function asJson(value: unknown, what: string): unknown { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) + } catch (error) { + refuse(`${what} is not JSON: ${String(error)}`) + } +} + +// Ops-shaped mutation record: the CRDT-on op echo or the CRDT-off ack summary. +// Bare count keys are excluded because read tools carry them. +function mutationReported(result: Record): boolean { + const data = result.data + if (!isRecord(data)) return false + if ((Array.isArray(data.ops) && data.ops.length > 0) || isRecord(data.op)) + return true + return Boolean(data.ops_by_kind ?? data.nodes_added ?? data.nodes_deleted) +} + +// The exporter's candidate set: every entry of data.ops when it is a list, +// else data.op whenever the key is present. +function echoedOps(result: Record): unknown[] { + const data = result.data + if (!isRecord(data)) return [] + if (Array.isArray(data.ops)) return data.ops + return 'op' in data ? [data.op] : [] +} + +const sha256 = (bytes: Buffer | string): string => + createHash('sha256').update(bytes).digest('hex') + +const sha256OfFile = (path: string): string => sha256(readFileSync(path)) + +const writeJson = (path: string, value: unknown): void => + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`) + +export function assembleCapture(input: AssembleInput) { + const { raw, rows, seed, provenance } = input + const { attempt } = raw + const workflow = seed.json.workflow + + if (!/^[A-Za-z0-9_-]+$/.test(attempt)) + refuse(`attempt label ${JSON.stringify(attempt)} is not [A-Za-z0-9_-]+`) + + const seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) + const driverIds = new Set(raw.seed_node_ids.map(String)) + if (!sameSet(driverIds, seedIds)) + refuse( + `the driver seeded ${list(driverIds)} but the seed fixture given here has ${list(seedIds)}` + ) + + if (raw.accepted?.status !== 202) + refuse(`recorded turn not accepted: ${JSON.stringify(raw.accepted)}`) + const body = raw.accepted.body ?? {} + const threadId = body.thread_id + const messageId = body.message_id + if (typeof threadId !== 'string' || typeof messageId !== 'string') + refuse(`ack without ids: ${JSON.stringify(body)}`) + if (!raw.saw_stream) refuse('frame stream never opened') + if (!raw.saw_done) + refuse( + `agent_message_done never arrived (timed_out=${raw.timed_out}, error=${raw.error})` + ) + + const seedAck = raw.seed_turn?.body ?? {} + const kept: RecordedFrame[] = [] + const dropped: Record = {} + const drop = (bucket: string): void => { + dropped[bucket] = (dropped[bucket] ?? 0) + 1 + } + + for (const frame of raw.frames) { + const { type, data } = frame + // Unparseable payloads belong to no turn; the heartbeat carries no ids. + if (type === '__raw__' || type === 'draft_version') { + drop(type === '__raw__' ? 'unparseable' : 'type:draft_version') + continue + } + if (data.thread_id !== threadId || data.message_id !== messageId) { + const seedTurn = + typeof seedAck.thread_id === 'string' && + data.thread_id === seedAck.thread_id && + data.message_id === seedAck.message_id + drop(seedTurn ? 'seed_turn' : 'foreign') + continue + } + if (!REPLAYED_FRAMES.includes(type)) { + if (type.startsWith('agent_')) + refuse(`frame type ${type} is outside the replay union`) + drop(`type:${type}`) + continue + } + kept.push(frame.at_ms === undefined ? { type, data } : frame) + } + + if (dropped.unparseable) + refuse(`${dropped.unparseable} unparseable socket payload(s) recorded`) + const last = kept.at(-1) + if (last?.type !== 'agent_message_done') + refuse(`last kept frame is ${last?.type}, not agent_message_done`) + + const tabs = new Set( + kept + .filter((frame) => frame.type === 'agent_active_tab') + .map((frame) => frame.data.workflow_id) + ) + if (tabs.size === 0) refuse('no agent_active_tab frame in this turn') + if (tabs.size > 1) + refuse(`agent_active_tab frames disagree on workflow_id: ${list(tabs)}`) + const workflowId = [...tabs][0] + if (typeof workflowId !== 'string' || !UUID.test(workflowId)) + refuse(`active_tab workflow id is not a uuid: ${String(workflowId)}`) + if (workflowId !== raw.seed_workflow_id) + refuse( + `active_tab workflow ${workflowId} is not the seeded workflow ${raw.seed_workflow_id}` + ) + + const toolCalls: AgentBackendCapture['tool_calls'] = [] + const childStatuses: Record = {} + const parentRows = rows.parents.map((parent) => { + const { id, tool_name: tool } = parent + if (!parent.tool_call_id) refuse(`parent row ${id} has NULL tool_call_id`) + for (const child of parent.children) { + const key = String(child.status) + childStatuses[key] = (childStatuses[key] ?? 0) + 1 + } + const applied = parent.children + .filter((child) => child.status === 'ok' && child.op_id) + .map((child) => String(child.op_id)) + + const parsed = asJson(parent.result, `parent row ${id} result`) + if ((parsed ?? null) === null && applied.length > 0) + refuse(`parent row ${id} has applied ops but a NULL result`) + const result = parsed ?? {} + if (!isRecord(result)) + refuse( + `parent row ${id} result is not an object; the exporter requires an object` + ) + if ( + MUTATING_TOOLS.includes(String(tool)) && + mutationReported(result) && + parent.children.length === 0 + ) + refuse( + `parent row ${id} (${tool}) reports a document mutation but has NO audit child rows` + ) + + const echoed = echoedOps(result) + const nonObjects = echoed.filter((op) => !isRecord(op)).length + if (nonObjects > 0) + refuse( + `parent row ${id} echoes ${nonObjects} non-object op entr${nonObjects === 1 ? 'y' : 'ies'}` + ) + const ops = echoed.filter(isRecord) + const offFrozen = new Set( + ops + .filter( + (op) => !(FROZEN_OPS as readonly string[]).includes(String(op.op)) + ) + .map((op) => String(op.op)) + ) + if (offFrozen.size > 0) + refuse( + `parent row ${id} echoes op kinds ${list(offFrozen)} outside the exporter frozen set` + ) + + const byId = new Map(ops.map((op) => [String(op.op_id), op])) + const missing = applied.filter((opId) => !byId.has(opId)) + if (missing.length > 0) + refuse( + `parent row ${id} applied op ids ${missing.join(', ')} are not echoed in its result` + ) + if (applied.length > 0 && parent.workflow_id !== workflowId) + refuse( + `parent row ${id} applied ops on ${String(parent.workflow_id)}, not the seeded workflow ${workflowId}` + ) + const appliedOps = applied.map((opId) => byId.get(opId)!) + // A node id of 0 is a real id, so only a missing one refuses. + if ( + appliedOps.some( + (op) => op.op === 'delete_node' && (op.node_id ?? null) === null + ) + ) + refuse(`parent row ${id} has an applied delete_node without node_id`) + + toolCalls.push({ + tool_call_id: parent.tool_call_id, + result, + applied_op_ids: applied + }) + return { + id, + tool_name: tool, + status: parent.status, + workflow_id: parent.workflow_id, + child_count: parent.children.length, + applied_op_ids: applied, + applied_op_kinds: appliedOps.map((op) => op.op), + applied_ops: appliedOps + } + }) + + const frameCalls = new Set( + kept + .filter( + (frame) => + frame.type === 'agent_tool_call' && frame.data.status !== 'running' + ) + .map((frame) => String(frame.data.tool_call_id)) + ) + const rowCalls = new Set(rows.parents.map((row) => String(row.tool_call_id))) + if (!sameSet(frameCalls, rowCalls)) + refuse( + `frames ${list(frameCalls)} and audit parent rows ${list(rowCalls)} disagree; the rows are not this turn` + ) + + if (rows.draft === null || rows.draft === undefined) + refuse(`no workflow_drafts row for ${workflowId}: the seed did not bind`) + const draft = asJson(rows.draft, `workflow_drafts.content for ${workflowId}`) + const draftIds = new Set( + (isRecord(draft) && Array.isArray(draft.nodes) ? draft.nodes : []) + .filter(isRecord) + .map((node) => String(node.id)) + ) + + const appliedOps = parentRows.flatMap((row) => row.applied_ops) + const opsOfKind = (kind: string): Array> => + appliedOps.filter((op) => op.op === kind) + // An uncatalogued class is stored opaquely by the applier, so a later + // set_widget on it throws. + const catalogTypes = new Set(Object.keys(workflow.catalog.types)) + const offCatalog = new Set( + opsOfKind('add_node') + .filter((op) => !catalogTypes.has(String(op.class_type))) + .map((op) => String(op.class_type)) + ) + if (offCatalog.size > 0) + refuse( + `applied add_node classes ${list(offCatalog)} are not in the seed catalog ${list(catalogTypes)}` + ) + + const deletedIds = new Set( + opsOfKind('delete_node').map((op) => String(op.node_id)) + ) + const addedIds = new Set( + opsOfKind('add_node') + .filter((op) => (op.node_id ?? null) !== null) + .map((op) => String(op.node_id)) + ) + const expectedIds = [...seedIds].filter((id) => !deletedIds.has(id)) + const missingSeed = expectedIds.filter((id) => !draftIds.has(id)) + if (missingSeed.length > 0) + refuse( + `draft for ${workflowId} lacks seed node ids ${list(missingSeed)} that no applied delete_node removed` + ) + const undeleted = [...deletedIds].filter((id) => draftIds.has(id)) + if (undeleted.length > 0) + refuse( + `draft for ${workflowId} still holds node ids ${list(undeleted)} that applied delete_node ops removed` + ) + + const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${threadId}; message ${messageId}; workflow ${workflowId} (seeded by throwaway turn ${String(seedAck.message_id)}; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(parentRows.map((row) => row.id))}; rows ${basename(rows.path)}; raw capture sha256 ${input.rawSha256}` + + const capture: AgentBackendCapture = { + schema_version: 'agent-backend-capture.v1', + source: { + repo: 'Comfy-Org/ComfyUI_frontend', + suite: 'agent', + case_id: raw.case_id, + note + }, + capture: { + backend: 'Comfy-Org/cloud', + thread_id: threadId, + message_id: messageId, + exported_at: provenance.exportedAt + }, + workflow: { + id: workflowId, + name: workflow.name, + catalog: workflow.catalog, + seed: workflow.seed + }, + request: { content: raw.prompt }, + frames: kept, + tool_calls: toolCalls + } + + const receipt = { + attempt, + frame_source: raw.frame_source, + channel: raw.channel, + raw: input.rawPath, + raw_sha256: input.rawSha256, + seed: seed.path, + seed_sha256: seed.sha256, + seed_node_ids: [...seedIds].sort(), + rows: rows.path, + rows_sha256: rows.sha256, + retrieval: rows.retrieval, + thread_id: threadId, + message_id: messageId, + workflow_id: workflowId, + start_workflow_id: body.workflow_id, + seed_turn: raw.seed_turn, + parent_rows: parentRows, + child_statuses: childStatuses, + frames_kept: kept.length, + frames_dropped: dropped, + kept_types: kept.map((frame) => frame.type), + draft_node_ids: [...draftIds].sort(), + deleted_node_ids: [...deletedIds].sort(), + added_node_ids: [...addedIds].sort(), + unexplained_draft_node_ids: [...draftIds] + .filter((id) => !expectedIds.includes(id) && !addedIds.has(id)) + .sort(), + catalog_types: [...catalogTypes].sort(), + mutating_parents: parentRows + .filter((row) => row.applied_op_ids.length > 0) + .map((row) => row.id), + cloud_sha: provenance.cloudSha, + model: provenance.model + } + + return { capture, receipt } +} + +function readRows(exec: string[], path: string, ids: TurnIds): NormalizedRows { + const quote = (value: string): string => `'${value.replace(/'/g, "''")}'` + const sql = `select json_build_object('source', 'postgres', 'parents', coalesce((select json_agg(row_to_json(r) order by r.started_at, r.id) from (select parent.id, parent.tool_call_id, parent.tool_name, parent.status, parent.result, parent.started_at, parent.workflow_id, coalesce(json_agg(json_build_object('op_id', child.op_id, 'status', child.status, 'op_index', child.op_index) order by child.op_index) filter (where child.id is not null), '[]'::json) as children from agent_tool_calls as parent left join agent_tool_calls as child on child.parent_call_id = parent.id where parent.thread_id = ${quote(ids.threadId)} and parent.message_id = ${quote(ids.messageId)} and parent.parent_call_id is null group by parent.id) r), '[]'::json), 'draft', (select content from workflow_drafts where workflow_id = ${quote(ids.workflowId)} limit 1))` + const dump: unknown = JSON.parse( + execFileSync(exec[0], [...exec.slice(1), sql], { encoding: 'utf8' }).trim() + ) + if (!isRecord(dump) || dump.source !== 'postgres') + refuse(`rows dump source is not 'postgres'`) + writeJson(path, dump) + return { + parents: (Array.isArray(dump.parents) ? dump.parents : []) + .filter(isRecord) + .map((row) => ({ + id: String(row.id), + tool_call_id: text(row.tool_call_id), + tool_name: text(row.tool_name), + status: text(row.status), + workflow_id: text(row.workflow_id), + result: row.result, + children: (Array.isArray(row.children) ? row.children : []) + .filter(isRecord) + .map((child) => ({ + op_id: text(child.op_id), + status: text(child.status) + })) + })), + draft: dump.draft ?? null, + retrieval: { kind: 'postgres-json', sql }, + path, + sha256: sha256OfFile(path) + } +} + +function openStream( + raw: RawCapture, + redisExec: string[], + onFrame: (frame: RecordedFrame) => void +): () => void { + const push = (payload: string): void => { + try { + const parsed: unknown = JSON.parse(payload) + if (isRecord(parsed) && typeof parsed.type === 'string') { + onFrame({ + type: parsed.type, + data: isRecord(parsed.data) ? parsed.data : {} + }) + return + } + } catch { + // Falls through to the marker the assembler refuses on. + } + onFrame({ type: '__raw__', data: { payload } }) + } + + const child = spawn( + redisExec[0], + [...redisExec.slice(1), 'SUBSCRIBE', raw.channel], + { stdio: ['ignore', 'pipe', 'inherit'] } + ) + child.stdout.setEncoding('utf8') + child.on('error', (error) => { + raw.error ??= `frame source failed to start: ${error.message}` + raw.stream_closed = true + }) + child.on('exit', () => (raw.stream_closed = true)) + // redis-cli off a tty prints subscribe/message envelopes one line at a time. + let buffered = '' + let pending = 0 + child.stdout.on('data', (chunk: string) => { + buffered += chunk + for (const line of buffered.split('\n').slice(0, -1)) { + if (line === 'subscribe') raw.saw_stream = true + else if (line === 'message') pending = 2 + else if (pending === 2) pending = 1 + else if (pending === 1) { + pending = 0 + push(line) + } + } + buffered = buffered.slice(buffered.lastIndexOf('\n') + 1) + }) + return () => child.kill() +} + +const sleep = (ms: number): Promise => + new Promise((done) => setTimeout(done, ms)) + +async function recordTurns( + raw: RawCapture, + options: { + headers: Record + redisExec: string[] + timeoutMs: number + seed: WorkflowJSON + } +): Promise { + const stop = openStream(raw, options.redisExec, (frame) => + raw.frames.push({ ...frame, at_ms: Date.now() }) + ) + + const postTurn = async (turn: unknown): Promise => { + const response = await fetch(`${raw.base}/agent/threads/new/messages`, { + method: 'POST', + headers: options.headers, + body: JSON.stringify(turn), + signal: AbortSignal.timeout(30_000) + }) + const payload = await response.text() + const ack = ((): Record | null => { + try { + const parsed: unknown = JSON.parse(payload) + return isRecord(parsed) ? parsed : null + } catch { + return null + } + })() + if (response.status === 202 && ack === null) + refuse(`202 with a non-JSON body: ${payload.slice(0, 200)}`) + return { status: response.status, body: ack } + } + + const done = (messageId: unknown): boolean => + raw.frames.some( + (frame) => + frame.type === 'agent_message_done' && + frame.data.message_id === messageId + ) + + const waitDone = async (messageId: unknown, label: string): Promise => { + const started = Date.now() + while ( + !done(messageId) && + !raw.stream_closed && + Date.now() - started < options.timeoutMs + ) + await sleep(250) + await sleep(1_500) + if (done(messageId)) return + raw.timed_out = !raw.stream_closed + refuse( + raw.stream_closed + ? `frame stream closed before the ${label} agent_message_done` + : `no agent_message_done for the ${label} within ${options.timeoutMs}ms` + ) + } + + try { + const opened = Date.now() + while ( + !raw.saw_stream && + !raw.stream_closed && + Date.now() - opened < 10_000 + ) + await sleep(100) + if (!raw.saw_stream) + refuse(`frame source did not open: ${raw.frame_source}`) + + // Turn A mints and seeds the workflow; it is thrown away, not recorded. + const seedTurn = await postTurn({ + content: + 'Hello. Please do nothing to the workflow yet; just acknowledge.', + draft: { content: options.seed } + }) + raw.seed_turn = seedTurn + const seedWorkflowId = seedTurn.body?.workflow_id + if (seedTurn.status !== 202 || typeof seedWorkflowId !== 'string') + refuse(`seed turn not accepted: ${JSON.stringify(seedTurn)}`) + raw.seed_workflow_id = seedWorkflowId + await waitDone(seedTurn.body?.message_id, 'seed turn') + + // Turn B opens on a fresh workflow so switch_tab publishes agent_active_tab. + const recorded = await postTurn({ + content: raw.prompt, + open_tabs: [{ workflow_id: seedWorkflowId, name: raw.seed_name }] + }) + raw.accepted = recorded + if (recorded.status !== 202 || !recorded.body?.message_id) + refuse(`recorded turn not accepted: ${JSON.stringify(recorded)}`) + await waitDone(recorded.body.message_id, 'recorded turn') + raw.saw_done = true + if ( + !raw.frames.some( + (frame) => + frame.type === 'agent_active_tab' && + frame.data.message_id === recorded.body?.message_id && + frame.data.workflow_id === seedWorkflowId + ) + ) + refuse( + `no agent_active_tab for ${seedWorkflowId}: the agent never switched tabs` + ) + } finally { + stop() + } +} + +function readSeedFixture(path: string): SeedFixture { + const parsed: unknown = JSON.parse(readFileSync(path, 'utf8')) + const workflow = isRecord(parsed) ? parsed.workflow : undefined + const seed = isRecord(workflow) ? workflow.seed : undefined + if ( + !isRecord(workflow) || + typeof workflow.name !== 'string' || + !isRecord(workflow.catalog) || + !isRecord(seed) || + !Array.isArray(seed.nodes) || + !Array.isArray(seed.links) + ) + refuse( + `seed fixture ${path} needs workflow.{name,catalog,seed.{nodes,links}}` + ) + return parsed as SeedFixture +} + +async function main(argv: string[]): Promise { + const positional: string[] = [] + const flags: Record = {} + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + if (arg === '--out' || arg === '--work') + flags[arg] = argv[(index += 1)] ?? '' + else positional.push(arg) + } + const [caseId, prompt, seedPath] = positional + const outPath = flags['--out'] + if (!caseId || !prompt || !seedPath || !outPath) throw new Error(USAGE) + + const env = (name: string, fallback = ''): string => + process.env[name] ?? fallback + const cloudSha = env('AGENT_CLOUD_SHA') + const model = env('AGENT_MODEL') + if (!/^[0-9a-f]{7,40}$/.test(cloudSha) || model.trim() === '') + throw new Error('AGENT_CLOUD_SHA (7-40 hex) and AGENT_MODEL are required') + const secretFile = env('AGENT_M2M_SECRET_FILE') + const workspace = env('AGENT_WORKSPACE_ID', env('REC_WORKSPACE_ID')) + const user = env('AGENT_USER_ID', env('REC_USER_ID')) + if (!secretFile || !workspace || !user) + throw new Error( + 'AGENT_M2M_SECRET_FILE, AGENT_WORKSPACE_ID (or REC_WORKSPACE_ID) and AGENT_USER_ID (or REC_USER_ID) are required' + ) + const timeoutMs = Number(env('AGENT_TURN_TIMEOUT', '180000')) + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) + throw new Error(`AGENT_TURN_TIMEOUT must be positive ms, got ${timeoutMs}`) + const attempt = ( + env('AGENT_ATTEMPT').trim() || + new Date().toISOString().replace(/[:.]/g, '-') + ).replace(/[^A-Za-z0-9_-]/g, '-') + + const seed = readSeedFixture(seedPath) + // conversations/ is scanned for replay cases, so sidecars go one level down. + const workDir = flags['--work'] || join(dirname(outPath), 'recordings') + mkdirSync(workDir, { recursive: true }) + mkdirSync(dirname(resolve(outPath)), { recursive: true }) + const sidecar = (suffix: string): string => + join(workDir, `${caseId}.${attempt}.${suffix}`) + const rawPath = sidecar('raw.json') + + const raw: RawCapture = { + case_id: caseId, + attempt, + prompt, + base: env('AGENT_FULLSTACK_URL', 'http://127.0.0.1:8086').replace( + /\/$/, + '' + ), + // The note carries this shape; the concrete channel stays in the receipt. + frame_source: 'redis SUBSCRIBE channel:ws::u:', + channel: `channel:ws:${workspace}:u:${user}`, + seed_sha256: sha256OfFile(seedPath), + seed_name: seed.workflow.name, + seed_node_ids: seed.workflow.seed.nodes.map((node) => node.id), + saw_stream: false, + stream_closed: false, + seed_turn: null, + seed_workflow_id: null, + accepted: null, + saw_done: false, + timed_out: false, + frames: [], + rows_artifact: null, + retrieval: null, + error: null + } + + try { + await recordTurns(raw, { + headers: { + 'content-type': 'application/json', + 'X-Comfy-M2M-Secret': readFileSync(secretFile, 'utf8').trim(), + 'X-Comfy-Workspace': workspace, + 'X-Comfy-User': user + }, + redisExec: env( + 'AGENT_REDIS_EXEC', + 'docker exec -i be11470-redis redis-cli' + ).split(' '), + timeoutMs, + seed: seed.workflow.seed + }) + + const rows = readRows( + env( + 'AGENT_PG_EXEC', + 'docker exec -i be11470-pg psql -U postgres -d postgres -At -c' + ).split(' '), + sidecar('rows.json'), + { + threadId: String(raw.accepted?.body?.thread_id), + messageId: String(raw.accepted?.body?.message_id), + workflowId: String(raw.seed_workflow_id) + } + ) + raw.rows_artifact = rows.path + raw.retrieval = rows.retrieval + writeJson(rawPath, raw) + + const { capture, receipt } = assembleCapture({ + raw, + rows, + seed: { json: seed, path: seedPath, sha256: raw.seed_sha256 }, + provenance: { cloudSha, model, exportedAt: new Date().toISOString() }, + rawSha256: sha256OfFile(rawPath), + rawPath + }) + writeJson(sidecar('capture.json'), capture) + writeJson(outPath, exportAgentConversation(capture)) + writeJson(sidecar('receipt.json'), receipt) + process.stdout.write( + `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} frames=${receipt.frames_kept} dropped=${JSON.stringify(receipt.frames_dropped)} parents=${receipt.parent_rows.length} mutating=${receipt.mutating_parents.length} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` + ) + } catch (error) { + raw.error = error instanceof Error ? error.message : String(error) + writeJson(rawPath, raw) + writeFileSync( + join(workDir, `${caseId}.refused.jsonl`), + `${JSON.stringify({ at: new Date().toISOString(), attempt, refused: raw.error })}\n`, + { flag: 'a' } + ) + throw error + } +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv.slice(2)).catch((error: unknown) => { + const refused = error instanceof RecordRefusal + console.error( + `record: ${refused ? 'REFUSED' : 'FAILED'}: ${error instanceof Error ? error.message : String(error)}` + ) + process.exitCode = 1 + }) +} From f1ebb7bbb49930051519fb507c5d4c13c1239189 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:51:32 -0700 Subject: [PATCH 43/49] refactor(agent): split the recorder's gates behind zod boundaries assembleCapture was one long function that hand-checked the shape of every value it touched. Its three inputs, the seed fixture, the rows dump and the socket frame, now parse through zod once, so each gate that follows is a small named function over typed data and a shape mismatch refuses at the boundary that owns it. The receipt keeps provenance, paths, hashes, ids and counts, instead of copying rows that the capture and the rows file already hold. The replay coverage steps told the next author to capture against the standalone harness and its SQLite store, which has no doc host and so can never produce the audit rows a replay asserts. They now point at the recorder and the local stack. --- browser_tests/README.md | 20 +- scripts/agentConversationRecord.test.ts | 139 ++-- scripts/agentConversationRecord.ts | 835 +++++++++++++----------- 3 files changed, 569 insertions(+), 425 deletions(-) diff --git a/browser_tests/README.md b/browser_tests/README.md index 68dc3e9320f9..ef3cf5512779 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -618,16 +618,16 @@ When a fix changes how the agent's turns affect the app (graph edits, CRDT frames, panel state), add a conversation replay case alongside the fix so the bug stays fixed: -1. **Capture the conversation.** Reproduce the bug's turn against the - local agent harness (`scripts/dev-agent-integration.ts`); the - standalone agent records every turn in its local SQLite store - (`agent_messages` / `agent_tool_calls`). Export the turn's rows and - convert them to a conversation JSON under - `browser_tests/fixtures/data/agent/conversations/` with - `scripts/agentConversationCapture.ts`, marking - `response_side: 'recorded'` (see `fixtures/data/agent/README.md` for - the capture format and backend query). Never write `graph_ops` by - hand and never relabel a synthesized response as recorded. +1. **Record the conversation.** Reproduce the bug's turn with + `scripts/agentConversationRecord.ts` against the non-standalone local + stack (Postgres + doc host), which is the only place the doc host + writes the audit rows a replay asserts. That one command records the + turn and writes the conversation JSON under + `browser_tests/fixtures/data/agent/conversations/` through the + exporter, marked `response_side: 'recorded'`; see + `fixtures/data/agent/README.md` for the stack recipe, the command and + the capture format. Never write `graph_ops` by hand and never relabel + a synthesized response as recorded. 2. **Add the replay case.** Drive the fixture through the conversation replay fixture (`agentConversationFixture`), asserting the canvas-observable outcome the bug corrupted (graph end-state or diff --git a/scripts/agentConversationRecord.test.ts b/scripts/agentConversationRecord.test.ts index fa46d427f167..cd2485c409e5 100644 --- a/scripts/agentConversationRecord.test.ts +++ b/scripts/agentConversationRecord.test.ts @@ -1,15 +1,15 @@ import { describe, expect, it } from 'vitest' +import type { z } from 'zod' import { exportAgentConversation } from './agentConversationCapture' import type { AssembleInput, NormalizedRows, - ParentRow, RawCapture, RecordedFrame, SeedFixture } from './agentConversationRecord' -import { assembleCapture } from './agentConversationRecord' +import { assembleCapture, zRowsDump } from './agentConversationRecord' const THREAD = 'thread-1' const MESSAGE = 'message-1' @@ -43,6 +43,7 @@ const frames = (): RecordedFrame[] => [ const seed = (): SeedFixture => ({ workflow: { + id: WORKFLOW, name: 'Text to image', catalog: { types: { KSampler: { widget_order: ['steps'] } } }, seed: { @@ -62,7 +63,9 @@ const addNodeOp = { class_type: 'KSampler' } -const parent = (overrides: Partial = {}): ParentRow => ({ +const parent = ( + overrides: Partial = {} +): RowsInput['parents'][number] => ({ id: 'parent-1', tool_call_id: 'tool-1', tool_name: 'apply_ops', @@ -103,14 +106,23 @@ const raw = (overrides: Partial = {}): RawCapture => ({ ...overrides }) -const rows = (overrides: Partial = {}): NormalizedRows => ({ - parents: [parent()], - draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] }, - retrieval: { kind: 'postgres-json' }, - path: '/tmp/agent-rec-example.a1.rows.json', - sha256: 'b'.repeat(64), - ...overrides -}) +type RowsInput = z.input + +const rows = (overrides: Partial = {}): NormalizedRows => { + const { parents, draft } = zRowsDump.parse({ + source: 'postgres', + parents: [parent()], + draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] }, + ...overrides + }) + return { + parents, + draft, + retrieval: { kind: 'postgres-json' }, + path: '/tmp/agent-rec-example.a1.rows.json', + sha256: 'b'.repeat(64) + } +} const input = (overrides: Partial = {}): AssembleInput => ({ raw: raw(), @@ -153,10 +165,11 @@ describe('assembleCapture', () => { } ]) expect(receipt).toMatchObject({ - added_node_ids: ['10'], - deleted_node_ids: [], - unexplained_draft_node_ids: [], - mutating_parents: ['parent-1'], + added_nodes: 1, + deleted_nodes: 0, + unexplained_draft_nodes: 0, + parents: 1, + mutating_parents: 1, child_statuses: { ok: 1 }, frames_kept: 4 }) @@ -180,22 +193,6 @@ describe('assembleCapture', () => { expect(receipt.channel).toContain('w-secret') }) - it('parses a result the backend handed back as a JSON string', () => { - const { capture } = assembleCapture( - input({ - rows: rows({ - parents: [ - parent({ - result: JSON.stringify({ ok: true, data: { ops: [addNodeOp] } }) - }) - ] - }) - }) - ) - - expect(capture.tool_calls[0].applied_op_ids).toEqual(['op-1']) - }) - it('accepts a delete_node whose node id is zero', () => { const { receipt } = assembleCapture( input({ @@ -226,7 +223,7 @@ describe('assembleCapture', () => { }) ) - expect(receipt.deleted_node_ids).toEqual(['0']) + expect(receipt.deleted_nodes).toBe(1) }) it('accepts a batch the document rejected as zero applied ops', () => { @@ -244,12 +241,12 @@ describe('assembleCapture', () => { }) ) - expect(receipt.mutating_parents).toEqual([]) + expect(receipt.mutating_parents).toBe(0) expect(receipt.child_statuses).toEqual({ error: 1 }) }) it('accepts a read tool that echoes ops without writing child rows', () => { - const { receipt } = assembleCapture( + const { capture, receipt } = assembleCapture( input({ rows: rows({ parents: [ @@ -264,7 +261,8 @@ describe('assembleCapture', () => { }) ) - expect(receipt.parent_rows[0].applied_op_ids).toEqual([]) + expect(capture.tool_calls[0].applied_op_ids).toEqual([]) + expect(receipt.mutating_parents).toBe(0) }) it('drops the ambient heartbeat and a non-replay turn frame into buckets', () => { @@ -393,18 +391,6 @@ describe('assembleCapture', () => { ).toThrow('has applied ops but a NULL result') }) - it('refuses a parent result that is not a JSON object', () => { - expect(() => - assembleCapture( - input({ - rows: rows({ - parents: [parent({ result: [1, 2], children: [] })] - }) - }) - ) - ).toThrow('not an object; the exporter requires an object') - }) - it('refuses an echoed op kind outside the exporter frozen set', () => { expect(() => assembleCapture( @@ -564,3 +550,62 @@ describe('assembleCapture', () => { ).toThrow('is not the seeded workflow') }) }) + +describe('zRowsDump', () => { + const dump = (result: unknown, overrides: Record = {}) => ({ + source: 'postgres', + parents: [ + { + id: 'parent-1', + tool_call_id: 'tool-1', + tool_name: 'apply_ops', + status: 'ok', + workflow_id: WORKFLOW, + result, + children: [], + ...overrides + } + ], + draft: null + }) + + it('decodes a result psql handed back as a JSON string', () => { + const parsed = zRowsDump.parse( + dump(JSON.stringify({ ok: true, data: { ops: [addNodeOp] } })) + ) + + expect(parsed.parents[0].result).toEqual({ + ok: true, + data: { ops: [addNodeOp] } + }) + }) + + it('decodes the draft column and normalises its node ids to strings', () => { + const parsed = zRowsDump.parse({ + ...dump(null), + draft: JSON.stringify({ nodes: [{ id: 3 }, { id: '4' }], links: [] }) + }) + + expect(parsed.draft?.nodes.map((node) => node.id)).toEqual(['3', '4']) + }) + + it('refuses a result that is not a JSON object', () => { + expect(() => zRowsDump.parse(dump([1, 2]))).toThrow() + }) + + it('refuses a result string that is not JSON', () => { + expect(() => zRowsDump.parse(dump('{'))).toThrow('is not JSON') + }) + + it('refuses a parent row without a tool call id', () => { + expect(() => + zRowsDump.parse(dump({ ok: true }, { tool_call_id: null })) + ).toThrow() + }) + + it('refuses a dump that did not come from postgres', () => { + expect(() => + zRowsDump.parse({ ...dump({ ok: true }), source: 'sqlite' }) + ).toThrow() + }) +}) diff --git a/scripts/agentConversationRecord.ts b/scripts/agentConversationRecord.ts index 4ff5bd49df21..ed481ff584fd 100644 --- a/scripts/agentConversationRecord.ts +++ b/scripts/agentConversationRecord.ts @@ -7,8 +7,11 @@ import { basename, dirname, join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' -import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import type { WorkflowJSON } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' +import type { zRecordedWsEvent } from '../browser_tests/fixtures/data/agent/agentConversation' +import { zAgentConversationWorkflow } from '../browser_tests/fixtures/data/agent/agentConversation' import type { AgentBackendCapture } from './agentConversationCapture' import { exportAgentConversation } from './agentConversationCapture' @@ -32,17 +35,101 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i const STACK = 'NON-standalone local full stack (Postgres + doc host, M2M identity headers)' -export type Retrieval = Record +const zJsonObject = z.record(z.string(), z.unknown()) -export interface TurnAck { - status: number - body: Record | null +// psql hands a json column back as an object, as a string, or as NULL. +const zJsonColumn = z.unknown().transform((value, ctx) => { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) as unknown + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'is not JSON' }) + return z.NEVER + } +}) + +const zEnv = z.object({ + AGENT_CLOUD_SHA: z.string().regex(/^[0-9a-f]{7,40}$/), + AGENT_MODEL: z.string().trim().min(1), + AGENT_M2M_SECRET_FILE: z.string().min(1), + AGENT_WORKSPACE_ID: z.string().min(1), + AGENT_USER_ID: z.string().min(1), + AGENT_FULLSTACK_URL: z.string().default('http://127.0.0.1:8086'), + AGENT_REDIS_EXEC: z + .string() + .default('docker exec -i be11470-redis redis-cli'), + AGENT_PG_EXEC: z + .string() + .default('docker exec -i be11470-pg psql -U postgres -d postgres -At -c'), + AGENT_TURN_TIMEOUT: z.coerce.number().positive().default(180_000), + AGENT_ATTEMPT: z.string().default('') +}) + +const zAck = z.object({ + thread_id: z.string().min(1), + message_id: z.string().min(1) +}) + +const zSeedAck = zAck.extend({ workflow_id: z.string().min(1) }) + +export const zSeedFixture = z.object({ + workflow: zAgentConversationWorkflow +}) + +const zParentRow = z.object({ + id: z.coerce.string(), + tool_call_id: z.string().min(1), + tool_name: z.string().nullable(), + status: z.string().nullable(), + workflow_id: z.string().nullable(), + result: zJsonColumn.pipe(zJsonObject.nullable()), + children: z.array( + z.object({ + op_id: z.string().nullable(), + status: z.string().nullable() + }) + ) +}) + +export const zRowsDump = z.object({ + source: z.literal('postgres'), + parents: z.array(zParentRow), + draft: zJsonColumn.pipe( + z + .object({ nodes: z.array(z.object({ id: z.coerce.string() })) }) + .passthrough() + .nullable() + ) +}) + +// The socket carries frames without a data object; the replay union does not. +const zSocketFrame = z + .object({ + type: z.string(), + data: zJsonObject.optional().catch(undefined) + }) + .transform((frame) => ({ type: frame.type, data: frame.data ?? {} })) + +// The exporter's candidate set: data.ops when it is a list, else data.op. +const zOpsCarrier = z.object({ + data: z + .object({ ops: z.array(z.unknown()).optional().catch(undefined) }) + .passthrough() +}) + +export type RecordedFrame = z.infer +export type SeedFixture = z.infer +export type ParentRow = z.infer + +export type NormalizedRows = Omit, 'source'> & { + retrieval: Record + path: string + sha256: string } -export interface RecordedFrame { - type: string - data: Record - at_ms?: number +export interface TurnAck { + status: number + body: unknown } export interface RawCapture { @@ -54,7 +141,7 @@ export interface RawCapture { channel: string seed_sha256: string seed_name: string - seed_node_ids: unknown[] + seed_node_ids: Array saw_stream: boolean stream_closed: boolean seed_turn: TurnAck | null @@ -64,32 +151,10 @@ export interface RawCapture { timed_out: boolean frames: RecordedFrame[] rows_artifact: string | null - retrieval: Retrieval | null + retrieval: Record | null error: string | null } -export interface ParentRow { - id: string - tool_call_id: string | null - tool_name: string | null - status: string | null - workflow_id: string | null - result: unknown - children: Array<{ op_id: string | null; status: string | null }> -} - -export interface NormalizedRows { - parents: ParentRow[] - draft: unknown - retrieval: Retrieval - path: string - sha256: string -} - -export interface SeedFixture { - workflow: { name: string; catalog: WidgetCatalog; seed: WorkflowJSON } -} - export interface AssembleInput { raw: RawCapture rows: NormalizedRows @@ -99,10 +164,16 @@ export interface AssembleInput { rawPath: string } +interface DraftCounts { + draft_nodes: number + added_nodes: number + deleted_nodes: number + unexplained_draft_nodes: number +} + interface TurnIds { threadId: string messageId: string - workflowId: string } // A refused recording is an expected outcome: main logs it and exits 1. @@ -112,46 +183,35 @@ function refuse(reason: string): never { throw new RecordRefusal(reason) } -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) +// A shape mismatch at a boundary is a refusal, named by its json path. +function parseOrRefuse( + schema: S, + value: unknown, + what: string +): z.output { + const parsed = schema.safeParse(value) + if (!parsed.success) + refuse( + `${what}: ${parsed.error.issues + .map((issue) => `${issue.path.join('.') || '(root)'} ${issue.message}`) + .join('; ')}` + ) + return parsed.data } -const list = (values: Iterable): string => - [...values].map(String).sort().join(', ') - -const text = (value: unknown): string | null => - value === null || value === undefined ? null : String(value) - -const sameSet = (left: Set, right: Set): boolean => - left.size === right.size && [...left].every((value) => right.has(value)) - -function asJson(value: unknown, what: string): unknown { - if (typeof value !== 'string') return value +const safeJson = (text: string): unknown => { try { - return JSON.parse(value) - } catch (error) { - refuse(`${what} is not JSON: ${String(error)}`) + return JSON.parse(text) + } catch { + return undefined } } -// Ops-shaped mutation record: the CRDT-on op echo or the CRDT-off ack summary. -// Bare count keys are excluded because read tools carry them. -function mutationReported(result: Record): boolean { - const data = result.data - if (!isRecord(data)) return false - if ((Array.isArray(data.ops) && data.ops.length > 0) || isRecord(data.op)) - return true - return Boolean(data.ops_by_kind ?? data.nodes_added ?? data.nodes_deleted) -} +const list = (values: Iterable): string => + [...values].map(String).sort().join(', ') -// The exporter's candidate set: every entry of data.ops when it is a list, -// else data.op whenever the key is present. -function echoedOps(result: Record): unknown[] { - const data = result.data - if (!isRecord(data)) return [] - if (Array.isArray(data.ops)) return data.ops - return 'op' in data ? [data.op] : [] -} +const sameSet = (left: Set, right: Set): boolean => + left.size === right.size && [...left].every((value) => right.has(value)) const sha256 = (bytes: Buffer | string): string => createHash('sha256').update(bytes).digest('hex') @@ -161,63 +221,59 @@ const sha256OfFile = (path: string): string => sha256(readFileSync(path)) const writeJson = (path: string, value: unknown): void => writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`) -export function assembleCapture(input: AssembleInput) { - const { raw, rows, seed, provenance } = input - const { attempt } = raw - const workflow = seed.json.workflow - - if (!/^[A-Za-z0-9_-]+$/.test(attempt)) - refuse(`attempt label ${JSON.stringify(attempt)} is not [A-Za-z0-9_-]+`) - - const seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) +// Everything the recorded turn must clear before its frames mean anything. +function turnIds(raw: RawCapture, seedIds: Set): TurnIds { + if (!/^[A-Za-z0-9_-]+$/.test(raw.attempt)) + refuse(`attempt label ${JSON.stringify(raw.attempt)} is not [A-Za-z0-9_-]+`) const driverIds = new Set(raw.seed_node_ids.map(String)) if (!sameSet(driverIds, seedIds)) refuse( `the driver seeded ${list(driverIds)} but the seed fixture given here has ${list(seedIds)}` ) - if (raw.accepted?.status !== 202) refuse(`recorded turn not accepted: ${JSON.stringify(raw.accepted)}`) - const body = raw.accepted.body ?? {} - const threadId = body.thread_id - const messageId = body.message_id - if (typeof threadId !== 'string' || typeof messageId !== 'string') - refuse(`ack without ids: ${JSON.stringify(body)}`) + const ack = zAck.safeParse(raw.accepted.body) + if (!ack.success) + refuse(`ack without ids: ${JSON.stringify(raw.accepted.body ?? {})}`) if (!raw.saw_stream) refuse('frame stream never opened') if (!raw.saw_done) refuse( `agent_message_done never arrived (timed_out=${raw.timed_out}, error=${raw.error})` ) + return { threadId: ack.data.thread_id, messageId: ack.data.message_id } +} - const seedAck = raw.seed_turn?.body ?? {} +function keepTurnFrames( + frames: RecordedFrame[], + ids: TurnIds, + seedTurn: TurnIds | null +): { kept: RecordedFrame[]; dropped: Record } { const kept: RecordedFrame[] = [] const dropped: Record = {} const drop = (bucket: string): void => { dropped[bucket] = (dropped[bucket] ?? 0) + 1 } - for (const frame of raw.frames) { + for (const frame of frames) { const { type, data } = frame // Unparseable payloads belong to no turn; the heartbeat carries no ids. - if (type === '__raw__' || type === 'draft_version') { + if (type === '__raw__' || type === 'draft_version') drop(type === '__raw__' ? 'unparseable' : 'type:draft_version') - continue - } - if (data.thread_id !== threadId || data.message_id !== messageId) { - const seedTurn = - typeof seedAck.thread_id === 'string' && - data.thread_id === seedAck.thread_id && - data.message_id === seedAck.message_id - drop(seedTurn ? 'seed_turn' : 'foreign') - continue - } - if (!REPLAYED_FRAMES.includes(type)) { - if (type.startsWith('agent_')) - refuse(`frame type ${type} is outside the replay union`) - drop(`type:${type}`) - continue - } - kept.push(frame.at_ms === undefined ? { type, data } : frame) + else if ( + data.thread_id !== ids.threadId || + data.message_id !== ids.messageId + ) + drop( + seedTurn !== null && + data.thread_id === seedTurn.threadId && + data.message_id === seedTurn.messageId + ? 'seed_turn' + : 'foreign' + ) + else if (REPLAYED_FRAMES.includes(type)) kept.push(frame) + else if (type.startsWith('agent_')) + refuse(`frame type ${type} is outside the replay union`) + else drop(`type:${type}`) } if (dropped.unparseable) @@ -225,7 +281,13 @@ export function assembleCapture(input: AssembleInput) { const last = kept.at(-1) if (last?.type !== 'agent_message_done') refuse(`last kept frame is ${last?.type}, not agent_message_done`) + return { kept, dropped } +} +function activeWorkflowId( + kept: RecordedFrame[], + seededId: string | null +): string { const tabs = new Set( kept .filter((frame) => frame.type === 'agent_active_tab') @@ -234,161 +296,204 @@ export function assembleCapture(input: AssembleInput) { if (tabs.size === 0) refuse('no agent_active_tab frame in this turn') if (tabs.size > 1) refuse(`agent_active_tab frames disagree on workflow_id: ${list(tabs)}`) - const workflowId = [...tabs][0] - if (typeof workflowId !== 'string' || !UUID.test(workflowId)) - refuse(`active_tab workflow id is not a uuid: ${String(workflowId)}`) - if (workflowId !== raw.seed_workflow_id) + const [tab] = [...tabs] + const workflowId = z.string().regex(UUID).safeParse(tab) + if (!workflowId.success) + refuse(`active_tab workflow id is not a uuid: ${String(tab)}`) + if (workflowId.data !== seededId) refuse( - `active_tab workflow ${workflowId} is not the seeded workflow ${raw.seed_workflow_id}` + `active_tab workflow ${workflowId.data} is not the seeded workflow ${seededId}` ) + return workflowId.data +} - const toolCalls: AgentBackendCapture['tool_calls'] = [] - const childStatuses: Record = {} - const parentRows = rows.parents.map((parent) => { - const { id, tool_name: tool } = parent - if (!parent.tool_call_id) refuse(`parent row ${id} has NULL tool_call_id`) - for (const child of parent.children) { - const key = String(child.status) - childStatuses[key] = (childStatuses[key] ?? 0) + 1 - } - const applied = parent.children - .filter((child) => child.status === 'ok' && child.op_id) - .map((child) => String(child.op_id)) - - const parsed = asJson(parent.result, `parent row ${id} result`) - if ((parsed ?? null) === null && applied.length > 0) - refuse(`parent row ${id} has applied ops but a NULL result`) - const result = parsed ?? {} - if (!isRecord(result)) - refuse( - `parent row ${id} result is not an object; the exporter requires an object` - ) - if ( - MUTATING_TOOLS.includes(String(tool)) && - mutationReported(result) && - parent.children.length === 0 +// Every op the exporter will read out of this result, shape-checked once. +function echoedOps(row: ParentRow): Array> { + const carrier = zOpsCarrier.safeParse(row.result ?? {}) + if (!carrier.success) return [] + const { data } = carrier.data + const ops = z + .array(zJsonObject) + .safeParse(data.ops ?? ('op' in data ? [data.op] : [])) + if (!ops.success) refuse(`parent row ${row.id} echoes a non-object op entry`) + const kinds = ops.data.map((op) => String(op.op)) + const offFrozen = new Set( + kinds.filter((kind) => !(FROZEN_OPS as readonly string[]).includes(kind)) + ) + if (offFrozen.size > 0) + refuse( + `parent row ${row.id} echoes op kinds ${list(offFrozen)} outside the exporter frozen set` ) - refuse( - `parent row ${id} (${tool}) reports a document mutation but has NO audit child rows` - ) + return ops.data +} - const echoed = echoedOps(result) - const nonObjects = echoed.filter((op) => !isRecord(op)).length - if (nonObjects > 0) - refuse( - `parent row ${id} echoes ${nonObjects} non-object op entr${nonObjects === 1 ? 'y' : 'ies'}` - ) - const ops = echoed.filter(isRecord) - const offFrozen = new Set( - ops - .filter( - (op) => !(FROZEN_OPS as readonly string[]).includes(String(op.op)) - ) - .map((op) => String(op.op)) +// Ops-shaped mutation record: the CRDT-on op echo or the CRDT-off ack summary. +// Bare count keys are excluded because read tools carry them. +function mutationReported(row: ParentRow): boolean { + const carrier = zOpsCarrier.safeParse(row.result ?? {}) + if (!carrier.success) return false + const { data } = carrier.data + return ( + (data.ops?.length ?? 0) > 0 || + zJsonObject.safeParse(data.op).success || + Boolean(data.ops_by_kind ?? data.nodes_added ?? data.nodes_deleted) + ) +} + +function appliedOps( + row: ParentRow, + applied: string[] +): Array> { + const byId = new Map( + echoedOps(row).flatMap((op) => { + const opId = z.string().safeParse(op.op_id) + return opId.success ? [[opId.data, op] as const] : [] + }) + ) + const missing = applied.filter((opId) => !byId.has(opId)) + if (missing.length > 0) + refuse( + `parent row ${row.id} applied op ids ${missing.join(', ')} are not echoed in its result` ) - if (offFrozen.size > 0) - refuse( - `parent row ${id} echoes op kinds ${list(offFrozen)} outside the exporter frozen set` - ) + const ops = applied.map((opId) => byId.get(opId)!) + // A node id of 0 is a real id, so only a missing one refuses. + if ( + ops.some((op) => op.op === 'delete_node' && (op.node_id ?? null) === null) + ) + refuse(`parent row ${row.id} has an applied delete_node without node_id`) + return ops +} - const byId = new Map(ops.map((op) => [String(op.op_id), op])) - const missing = applied.filter((opId) => !byId.has(opId)) - if (missing.length > 0) - refuse( - `parent row ${id} applied op ids ${missing.join(', ')} are not echoed in its result` - ) - if (applied.length > 0 && parent.workflow_id !== workflowId) - refuse( - `parent row ${id} applied ops on ${String(parent.workflow_id)}, not the seeded workflow ${workflowId}` - ) - const appliedOps = applied.map((opId) => byId.get(opId)!) - // A node id of 0 is a real id, so only a missing one refuses. - if ( - appliedOps.some( - (op) => op.op === 'delete_node' && (op.node_id ?? null) === null - ) +function parentToolCall( + row: ParentRow, + workflowId: string +): { + toolCall: AgentBackendCapture['tool_calls'][number] + appliedOps: Array> +} { + const applied = row.children.flatMap((child) => + child.status === 'ok' && child.op_id ? [child.op_id] : [] + ) + if (row.result === null && applied.length > 0) + refuse(`parent row ${row.id} has applied ops but a NULL result`) + if ( + row.tool_name !== null && + MUTATING_TOOLS.includes(row.tool_name) && + mutationReported(row) && + row.children.length === 0 + ) + refuse( + `parent row ${row.id} (${row.tool_name}) reports a document mutation but has NO audit child rows` + ) + + if (applied.length > 0 && row.workflow_id !== workflowId) + refuse( + `parent row ${row.id} applied ops on ${row.workflow_id}, not the seeded workflow ${workflowId}` ) - refuse(`parent row ${id} has an applied delete_node without node_id`) - toolCalls.push({ - tool_call_id: parent.tool_call_id, - result, + return { + appliedOps: appliedOps(row, applied), + toolCall: { + tool_call_id: row.tool_call_id, + result: row.result ?? {}, applied_op_ids: applied - }) - return { - id, - tool_name: tool, - status: parent.status, - workflow_id: parent.workflow_id, - child_count: parent.children.length, - applied_op_ids: applied, - applied_op_kinds: appliedOps.map((op) => op.op), - applied_ops: appliedOps } - }) + } +} +// The frames and the audit rows must describe the same turn's tool calls. +function checkTurnAgreement(kept: RecordedFrame[], rows: ParentRow[]): void { const frameCalls = new Set( kept .filter( (frame) => frame.type === 'agent_tool_call' && frame.data.status !== 'running' ) - .map((frame) => String(frame.data.tool_call_id)) + .flatMap( + (frame) => z.string().safeParse(frame.data.tool_call_id).data ?? [] + ) ) - const rowCalls = new Set(rows.parents.map((row) => String(row.tool_call_id))) + const rowCalls = new Set(rows.map((row) => row.tool_call_id)) if (!sameSet(frameCalls, rowCalls)) refuse( `frames ${list(frameCalls)} and audit parent rows ${list(rowCalls)} disagree; the rows are not this turn` ) +} - if (rows.draft === null || rows.draft === undefined) +// The draft is the only witness that the applied ops reached the document. +function checkDraft( + draft: NormalizedRows['draft'], + seedIds: Set, + appliedOps: Array>, + workflowId: string +): DraftCounts { + if (draft === null) refuse(`no workflow_drafts row for ${workflowId}: the seed did not bind`) - const draft = asJson(rows.draft, `workflow_drafts.content for ${workflowId}`) - const draftIds = new Set( - (isRecord(draft) && Array.isArray(draft.nodes) ? draft.nodes : []) - .filter(isRecord) - .map((node) => String(node.id)) - ) - - const appliedOps = parentRows.flatMap((row) => row.applied_ops) - const opsOfKind = (kind: string): Array> => - appliedOps.filter((op) => op.op === kind) - // An uncatalogued class is stored opaquely by the applier, so a later - // set_widget on it throws. - const catalogTypes = new Set(Object.keys(workflow.catalog.types)) - const offCatalog = new Set( - opsOfKind('add_node') - .filter((op) => !catalogTypes.has(String(op.class_type))) - .map((op) => String(op.class_type)) - ) - if (offCatalog.size > 0) - refuse( - `applied add_node classes ${list(offCatalog)} are not in the seed catalog ${list(catalogTypes)}` + const draftIds = new Set(draft.nodes.map((node) => node.id)) + const nodeIds = (kind: string): Set => + new Set( + appliedOps + .filter((op) => op.op === kind && (op.node_id ?? null) !== null) + .map((op) => String(op.node_id)) ) - const deletedIds = new Set( - opsOfKind('delete_node').map((op) => String(op.node_id)) - ) - const addedIds = new Set( - opsOfKind('add_node') - .filter((op) => (op.node_id ?? null) !== null) - .map((op) => String(op.node_id)) - ) - const expectedIds = [...seedIds].filter((id) => !deletedIds.has(id)) - const missingSeed = expectedIds.filter((id) => !draftIds.has(id)) - if (missingSeed.length > 0) + const deleted = nodeIds('delete_node') + const added = nodeIds('add_node') + const expected = [...seedIds].filter((id) => !deleted.has(id)) + const missing = expected.filter((id) => !draftIds.has(id)) + if (missing.length > 0) refuse( - `draft for ${workflowId} lacks seed node ids ${list(missingSeed)} that no applied delete_node removed` + `draft for ${workflowId} lacks seed node ids ${list(missing)} that no applied delete_node removed` ) - const undeleted = [...deletedIds].filter((id) => draftIds.has(id)) + const undeleted = [...deleted].filter((id) => draftIds.has(id)) if (undeleted.length > 0) refuse( `draft for ${workflowId} still holds node ids ${list(undeleted)} that applied delete_node ops removed` ) - const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${threadId}; message ${messageId}; workflow ${workflowId} (seeded by throwaway turn ${String(seedAck.message_id)}; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(parentRows.map((row) => row.id))}; rows ${basename(rows.path)}; raw capture sha256 ${input.rawSha256}` + return { + draft_nodes: draftIds.size, + added_nodes: added.size, + deleted_nodes: deleted.size, + unexplained_draft_nodes: [...draftIds].filter( + (id) => !expected.includes(id) && !added.has(id) + ).length + } +} + +// An uncatalogued class is stored opaquely by the applier, so a later +// set_widget on it throws. +function checkAddedClasses( + appliedOps: Array>, + catalog: SeedFixture['workflow']['catalog'] +): void { + const types = new Set(Object.keys(catalog.types)) + const offCatalog = new Set( + appliedOps + .filter((op) => op.op === 'add_node') + .map((op) => String(op.class_type)) + .filter((type) => !types.has(type)) + ) + if (offCatalog.size > 0) + refuse( + `applied add_node classes ${list(offCatalog)} are not in the seed catalog ${list(types)}` + ) +} + +function buildCapture(options: { + input: AssembleInput + ids: TurnIds + workflowId: string + seedMessageId: string | null + kept: RecordedFrame[] + toolCalls: AgentBackendCapture['tool_calls'] +}): AgentBackendCapture { + const { input, ids, workflowId, kept, toolCalls } = options + const { raw, rows, provenance } = input + const { workflow } = input.seed.json + const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${ids.threadId}; message ${ids.messageId}; workflow ${workflowId} (seeded by throwaway turn ${options.seedMessageId}; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(rows.parents.map((row) => row.id))}; rows ${basename(rows.path)}; raw capture sha256 ${input.rawSha256}` - const capture: AgentBackendCapture = { + return { schema_version: 'agent-backend-capture.v1', source: { repo: 'Comfy-Org/ComfyUI_frontend', @@ -398,8 +503,8 @@ export function assembleCapture(input: AssembleInput) { }, capture: { backend: 'Comfy-Org/cloud', - thread_id: threadId, - message_id: messageId, + thread_id: ids.threadId, + message_id: ids.messageId, exported_at: provenance.exportedAt }, workflow: { @@ -412,73 +517,108 @@ export function assembleCapture(input: AssembleInput) { frames: kept, tool_calls: toolCalls } +} - const receipt = { - attempt, +function buildReceipt(options: { + input: AssembleInput + ids: TurnIds + workflowId: string + frames: { kept: number; dropped: Record } + parents: { count: number; mutating: number; children: Record } + draft: DraftCounts +}) { + const { input, ids, workflowId, frames, parents } = options + const { raw, rows, seed, provenance } = input + return { + attempt: raw.attempt, frame_source: raw.frame_source, channel: raw.channel, raw: input.rawPath, raw_sha256: input.rawSha256, seed: seed.path, seed_sha256: seed.sha256, - seed_node_ids: [...seedIds].sort(), rows: rows.path, rows_sha256: rows.sha256, retrieval: rows.retrieval, - thread_id: threadId, - message_id: messageId, + thread_id: ids.threadId, + message_id: ids.messageId, workflow_id: workflowId, - start_workflow_id: body.workflow_id, - seed_turn: raw.seed_turn, - parent_rows: parentRows, - child_statuses: childStatuses, - frames_kept: kept.length, - frames_dropped: dropped, - kept_types: kept.map((frame) => frame.type), - draft_node_ids: [...draftIds].sort(), - deleted_node_ids: [...deletedIds].sort(), - added_node_ids: [...addedIds].sort(), - unexplained_draft_node_ids: [...draftIds] - .filter((id) => !expectedIds.includes(id) && !addedIds.has(id)) - .sort(), - catalog_types: [...catalogTypes].sort(), - mutating_parents: parentRows - .filter((row) => row.applied_op_ids.length > 0) - .map((row) => row.id), + frames_kept: frames.kept, + frames_dropped: frames.dropped, + parents: parents.count, + mutating_parents: parents.mutating, + child_statuses: parents.children, + ...options.draft, cloud_sha: provenance.cloudSha, model: provenance.model } +} - return { capture, receipt } +function childStatuses(parents: ParentRow[]): Record { + const tally: Record = {} + for (const child of parents.flatMap((row) => row.children)) + tally[child.status ?? 'null'] = (tally[child.status ?? 'null'] ?? 0) + 1 + return tally } -function readRows(exec: string[], path: string, ids: TurnIds): NormalizedRows { +export function assembleCapture(input: AssembleInput) { + const { raw, rows } = input + const { workflow } = input.seed.json + const seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) + + const ids = turnIds(raw, seedIds) + const seedAck = zAck.safeParse(raw.seed_turn?.body) + const seedTurn = seedAck.success + ? { threadId: seedAck.data.thread_id, messageId: seedAck.data.message_id } + : null + const { kept, dropped } = keepTurnFrames(raw.frames, ids, seedTurn) + const workflowId = activeWorkflowId(kept, raw.seed_workflow_id) + + const calls = rows.parents.map((row) => parentToolCall(row, workflowId)) + checkTurnAgreement(kept, rows.parents) + const appliedOps = calls.flatMap((call) => call.appliedOps) + const draft = checkDraft(rows.draft, seedIds, appliedOps, workflowId) + checkAddedClasses(appliedOps, workflow.catalog) + + return { + capture: buildCapture({ + input, + ids, + workflowId, + seedMessageId: seedTurn?.messageId ?? null, + kept, + toolCalls: calls.map((call) => call.toolCall) + }), + receipt: buildReceipt({ + input, + ids, + workflowId, + frames: { kept: kept.length, dropped }, + parents: { + count: rows.parents.length, + mutating: calls.filter((call) => call.appliedOps.length > 0).length, + children: childStatuses(rows.parents) + }, + draft + }) + } +} + +function readRows( + exec: string[], + path: string, + ids: TurnIds & { workflowId: string } +): NormalizedRows { const quote = (value: string): string => `'${value.replace(/'/g, "''")}'` const sql = `select json_build_object('source', 'postgres', 'parents', coalesce((select json_agg(row_to_json(r) order by r.started_at, r.id) from (select parent.id, parent.tool_call_id, parent.tool_name, parent.status, parent.result, parent.started_at, parent.workflow_id, coalesce(json_agg(json_build_object('op_id', child.op_id, 'status', child.status, 'op_index', child.op_index) order by child.op_index) filter (where child.id is not null), '[]'::json) as children from agent_tool_calls as parent left join agent_tool_calls as child on child.parent_call_id = parent.id where parent.thread_id = ${quote(ids.threadId)} and parent.message_id = ${quote(ids.messageId)} and parent.parent_call_id is null group by parent.id) r), '[]'::json), 'draft', (select content from workflow_drafts where workflow_id = ${quote(ids.workflowId)} limit 1))` const dump: unknown = JSON.parse( execFileSync(exec[0], [...exec.slice(1), sql], { encoding: 'utf8' }).trim() ) - if (!isRecord(dump) || dump.source !== 'postgres') - refuse(`rows dump source is not 'postgres'`) writeJson(path, dump) + const { parents, draft } = parseOrRefuse(zRowsDump, dump, 'rows dump') return { - parents: (Array.isArray(dump.parents) ? dump.parents : []) - .filter(isRecord) - .map((row) => ({ - id: String(row.id), - tool_call_id: text(row.tool_call_id), - tool_name: text(row.tool_name), - status: text(row.status), - workflow_id: text(row.workflow_id), - result: row.result, - children: (Array.isArray(row.children) ? row.children : []) - .filter(isRecord) - .map((child) => ({ - op_id: text(child.op_id), - status: text(child.status) - })) - })), - draft: dump.draft ?? null, + parents, + draft, retrieval: { kind: 'postgres-json', sql }, path, sha256: sha256OfFile(path) @@ -491,19 +631,8 @@ function openStream( onFrame: (frame: RecordedFrame) => void ): () => void { const push = (payload: string): void => { - try { - const parsed: unknown = JSON.parse(payload) - if (isRecord(parsed) && typeof parsed.type === 'string') { - onFrame({ - type: parsed.type, - data: isRecord(parsed.data) ? parsed.data : {} - }) - return - } - } catch { - // Falls through to the marker the assembler refuses on. - } - onFrame({ type: '__raw__', data: { payload } }) + const frame = zSocketFrame.safeParse(safeJson(payload)) + onFrame(frame.success ? frame.data : { type: '__raw__', data: { payload } }) } const child = spawn( @@ -560,27 +689,20 @@ async function recordTurns( signal: AbortSignal.timeout(30_000) }) const payload = await response.text() - const ack = ((): Record | null => { - try { - const parsed: unknown = JSON.parse(payload) - return isRecord(parsed) ? parsed : null - } catch { - return null - } - })() - if (response.status === 202 && ack === null) + const body = zJsonObject.safeParse(safeJson(payload)) + if (response.status === 202 && !body.success) refuse(`202 with a non-JSON body: ${payload.slice(0, 200)}`) - return { status: response.status, body: ack } + return { status: response.status, body: body.data ?? null } } - const done = (messageId: unknown): boolean => + const done = (messageId: string): boolean => raw.frames.some( (frame) => frame.type === 'agent_message_done' && frame.data.message_id === messageId ) - const waitDone = async (messageId: unknown, label: string): Promise => { + const waitDone = async (messageId: string, label: string): Promise => { const started = Date.now() while ( !done(messageId) && @@ -616,56 +738,41 @@ async function recordTurns( draft: { content: options.seed } }) raw.seed_turn = seedTurn - const seedWorkflowId = seedTurn.body?.workflow_id - if (seedTurn.status !== 202 || typeof seedWorkflowId !== 'string') + const seedAck = zSeedAck.safeParse(seedTurn.body) + if (seedTurn.status !== 202 || !seedAck.success) refuse(`seed turn not accepted: ${JSON.stringify(seedTurn)}`) - raw.seed_workflow_id = seedWorkflowId - await waitDone(seedTurn.body?.message_id, 'seed turn') + raw.seed_workflow_id = seedAck.data.workflow_id + await waitDone(seedAck.data.message_id, 'seed turn') // Turn B opens on a fresh workflow so switch_tab publishes agent_active_tab. const recorded = await postTurn({ content: raw.prompt, - open_tabs: [{ workflow_id: seedWorkflowId, name: raw.seed_name }] + open_tabs: [ + { workflow_id: seedAck.data.workflow_id, name: raw.seed_name } + ] }) raw.accepted = recorded - if (recorded.status !== 202 || !recorded.body?.message_id) + const ack = zAck.safeParse(recorded.body) + if (recorded.status !== 202 || !ack.success) refuse(`recorded turn not accepted: ${JSON.stringify(recorded)}`) - await waitDone(recorded.body.message_id, 'recorded turn') + await waitDone(ack.data.message_id, 'recorded turn') raw.saw_done = true if ( !raw.frames.some( (frame) => frame.type === 'agent_active_tab' && - frame.data.message_id === recorded.body?.message_id && - frame.data.workflow_id === seedWorkflowId + frame.data.message_id === ack.data.message_id && + frame.data.workflow_id === seedAck.data.workflow_id ) ) refuse( - `no agent_active_tab for ${seedWorkflowId}: the agent never switched tabs` + `no agent_active_tab for ${seedAck.data.workflow_id}: the agent never switched tabs` ) } finally { stop() } } -function readSeedFixture(path: string): SeedFixture { - const parsed: unknown = JSON.parse(readFileSync(path, 'utf8')) - const workflow = isRecord(parsed) ? parsed.workflow : undefined - const seed = isRecord(workflow) ? workflow.seed : undefined - if ( - !isRecord(workflow) || - typeof workflow.name !== 'string' || - !isRecord(workflow.catalog) || - !isRecord(seed) || - !Array.isArray(seed.nodes) || - !Array.isArray(seed.links) - ) - refuse( - `seed fixture ${path} needs workflow.{name,catalog,seed.{nodes,links}}` - ) - return parsed as SeedFixture -} - async function main(argv: string[]): Promise { const positional: string[] = [] const flags: Record = {} @@ -679,28 +786,25 @@ async function main(argv: string[]): Promise { const outPath = flags['--out'] if (!caseId || !prompt || !seedPath || !outPath) throw new Error(USAGE) - const env = (name: string, fallback = ''): string => - process.env[name] ?? fallback - const cloudSha = env('AGENT_CLOUD_SHA') - const model = env('AGENT_MODEL') - if (!/^[0-9a-f]{7,40}$/.test(cloudSha) || model.trim() === '') - throw new Error('AGENT_CLOUD_SHA (7-40 hex) and AGENT_MODEL are required') - const secretFile = env('AGENT_M2M_SECRET_FILE') - const workspace = env('AGENT_WORKSPACE_ID', env('REC_WORKSPACE_ID')) - const user = env('AGENT_USER_ID', env('REC_USER_ID')) - if (!secretFile || !workspace || !user) - throw new Error( - 'AGENT_M2M_SECRET_FILE, AGENT_WORKSPACE_ID (or REC_WORKSPACE_ID) and AGENT_USER_ID (or REC_USER_ID) are required' - ) - const timeoutMs = Number(env('AGENT_TURN_TIMEOUT', '180000')) - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) - throw new Error(`AGENT_TURN_TIMEOUT must be positive ms, got ${timeoutMs}`) + const env = parseOrRefuse( + zEnv, + { + ...process.env, + AGENT_WORKSPACE_ID: + process.env.AGENT_WORKSPACE_ID ?? process.env.REC_WORKSPACE_ID, + AGENT_USER_ID: process.env.AGENT_USER_ID ?? process.env.REC_USER_ID + }, + 'environment' + ) const attempt = ( - env('AGENT_ATTEMPT').trim() || - new Date().toISOString().replace(/[:.]/g, '-') + env.AGENT_ATTEMPT.trim() || new Date().toISOString().replace(/[:.]/g, '-') ).replace(/[^A-Za-z0-9_-]/g, '-') - const seed = readSeedFixture(seedPath) + const seed = parseOrRefuse( + zSeedFixture, + JSON.parse(readFileSync(seedPath, 'utf8')), + `seed fixture ${seedPath}` + ) // conversations/ is scanned for replay cases, so sidecars go one level down. const workDir = flags['--work'] || join(dirname(outPath), 'recordings') mkdirSync(workDir, { recursive: true }) @@ -713,13 +817,10 @@ async function main(argv: string[]): Promise { case_id: caseId, attempt, prompt, - base: env('AGENT_FULLSTACK_URL', 'http://127.0.0.1:8086').replace( - /\/$/, - '' - ), + base: env.AGENT_FULLSTACK_URL.replace(/\/$/, ''), // The note carries this shape; the concrete channel stays in the receipt. frame_source: 'redis SUBSCRIBE channel:ws::u:', - channel: `channel:ws:${workspace}:u:${user}`, + channel: `channel:ws:${env.AGENT_WORKSPACE_ID}:u:${env.AGENT_USER_ID}`, seed_sha256: sha256OfFile(seedPath), seed_name: seed.workflow.name, seed_node_ids: seed.workflow.seed.nodes.map((node) => node.id), @@ -740,30 +841,24 @@ async function main(argv: string[]): Promise { await recordTurns(raw, { headers: { 'content-type': 'application/json', - 'X-Comfy-M2M-Secret': readFileSync(secretFile, 'utf8').trim(), - 'X-Comfy-Workspace': workspace, - 'X-Comfy-User': user + 'X-Comfy-M2M-Secret': readFileSync( + env.AGENT_M2M_SECRET_FILE, + 'utf8' + ).trim(), + 'X-Comfy-Workspace': env.AGENT_WORKSPACE_ID, + 'X-Comfy-User': env.AGENT_USER_ID }, - redisExec: env( - 'AGENT_REDIS_EXEC', - 'docker exec -i be11470-redis redis-cli' - ).split(' '), - timeoutMs, + redisExec: env.AGENT_REDIS_EXEC.split(' '), + timeoutMs: env.AGENT_TURN_TIMEOUT, seed: seed.workflow.seed }) - const rows = readRows( - env( - 'AGENT_PG_EXEC', - 'docker exec -i be11470-pg psql -U postgres -d postgres -At -c' - ).split(' '), - sidecar('rows.json'), - { - threadId: String(raw.accepted?.body?.thread_id), - messageId: String(raw.accepted?.body?.message_id), - workflowId: String(raw.seed_workflow_id) - } - ) + const ids = zAck.parse(raw.accepted?.body) + const rows = readRows(env.AGENT_PG_EXEC.split(' '), sidecar('rows.json'), { + threadId: ids.thread_id, + messageId: ids.message_id, + workflowId: String(raw.seed_workflow_id) + }) raw.rows_artifact = rows.path raw.retrieval = rows.retrieval writeJson(rawPath, raw) @@ -772,7 +867,11 @@ async function main(argv: string[]): Promise { raw, rows, seed: { json: seed, path: seedPath, sha256: raw.seed_sha256 }, - provenance: { cloudSha, model, exportedAt: new Date().toISOString() }, + provenance: { + cloudSha: env.AGENT_CLOUD_SHA, + model: env.AGENT_MODEL, + exportedAt: new Date().toISOString() + }, rawSha256: sha256OfFile(rawPath), rawPath }) @@ -780,7 +879,7 @@ async function main(argv: string[]): Promise { writeJson(outPath, exportAgentConversation(capture)) writeJson(sidecar('receipt.json'), receipt) process.stdout.write( - `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} frames=${receipt.frames_kept} dropped=${JSON.stringify(receipt.frames_dropped)} parents=${receipt.parent_rows.length} mutating=${receipt.mutating_parents.length} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` + `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} frames=${receipt.frames_kept} dropped=${JSON.stringify(receipt.frames_dropped)} parents=${receipt.parents} mutating=${receipt.mutating_parents} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` ) } catch (error) { raw.error = error instanceof Error ? error.message : String(error) @@ -800,8 +899,8 @@ if ( ) { main(process.argv.slice(2)).catch((error: unknown) => { const refused = error instanceof RecordRefusal - console.error( - `record: ${refused ? 'REFUSED' : 'FAILED'}: ${error instanceof Error ? error.message : String(error)}` + process.stderr.write( + `record: ${refused ? 'REFUSED' : 'FAILED'}: ${error instanceof Error ? error.message : String(error)}\n` ) process.exitCode = 1 }) From 76f3092c5cf6c22a32185bfaee73b1faf281cc4b Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:56:24 -0700 Subject: [PATCH 44/49] docs(browser-tests): recordings need the agent's non-standalone mode The standalone agent never writes the per-op audit rows the exporter reads, so the capture step names the mode that does. --- browser_tests/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/browser_tests/README.md b/browser_tests/README.md index 68dc3e9320f9..85c6b4524d0b 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -619,9 +619,12 @@ CRDT frames, panel state), add a conversation replay case alongside the fix so the bug stays fixed: 1. **Capture the conversation.** Reproduce the bug's turn against the - local agent harness (`scripts/dev-agent-integration.ts`); the - standalone agent records every turn in its local SQLite store - (`agent_messages` / `agent_tool_calls`). Export the turn's rows and + cloud agent run locally in its non-standalone mode, with Postgres, + Redis and the doc host beside it: that is the only mode that writes + the per-op audit rows (`agent_tool_calls` parent and child rows) the + exporter reads. The same agent in standalone mode + (SQLite, no doc host) never writes them, so it can only + yield text-only or tool-error turns. Export the turn's rows and convert them to a conversation JSON under `browser_tests/fixtures/data/agent/conversations/` with `scripts/agentConversationCapture.ts`, marking From 3a42b7579051351371de224b089bc8a99cd71882 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:57:01 -0700 Subject: [PATCH 45/49] docs(agent): name the agent mode a recording needs and why --- browser_tests/fixtures/data/agent/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md index 40c4f7eb51bc..d2640bc8ab66 100644 --- a/browser_tests/fixtures/data/agent/README.md +++ b/browser_tests/fixtures/data/agent/README.md @@ -34,9 +34,12 @@ first, because the replay subscribes to the document only on an `agent_active_tab` frame and switching to an already-focused tab publishes nothing. -Recording runs against the lean stack: frames come from its Redis channel -and rows from Postgres. That is the only path, because the graph operations -a replay asserts exist only where the doc host writes audit rows. +Recording runs against the cloud agent in its non-standalone mode, with +Postgres, Redis and the doc host beside it (the lean stack): frames come from +its Redis channel and rows from Postgres. That is the only path. The same agent +in standalone mode (SQLite, no doc host) never writes the audit rows that carry +the graph operations a replay asserts, so it can only produce text-only or +tool-error turns. The recording agent shells out to `comfy-cli` for its read and edit tools inside a sandbox that overrides `HOME`, so point `COMFY_BIN` at an installation that From a1ac2caafe8e00951e87240cd28dbeea9df3ed20 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 08:46:56 -0700 Subject: [PATCH 46/49] test(agent): agent-conversation.v2 replays every turn of a thread A conversation is now a list of turns on one thread, each with its own message id, request and response entries. The exporter emits one turn per recorded message, the harness answers each prompt with that turn's ack and stamps its frames, and every expectation folds over all turns. The nine recordings are rewritten to the new shape. --- .../fixtures/agentConversationFixture.ts | 108 +-- .../fixtures/data/agent/agentConversation.ts | 28 +- .../agent-l4-zimage-string-node-prompt.json | 664 +++++++++--------- .../conversations/agent-rec-batched-ops.json | 346 ++++----- .../agent-rec-refiner-between.json | 464 ++++++------ .../agent-rec-replace-prompt-encoder.json | 400 +++++------ .../agent-rec-set-widget-existing.json | 282 ++++---- .../agent-rec-text-only-answer.json | 154 ++-- .../agent-rec-three-sequential-adds.json | 414 +++++------ .../conversations/agent-rec-tool-error.json | 156 ++-- .../agent-workflow-editing-05.json | 438 ++++++------ .../agentConversationConcurrentEdits.spec.ts | 4 +- .../agent/agentConversationReplay.spec.ts | 4 +- .../agent/agentConversationTiming.spec.ts | 2 +- scripts/agentConversationCapture.test.ts | 171 +++-- scripts/agentConversationCapture.ts | 42 +- 16 files changed, 1893 insertions(+), 1784 deletions(-) diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts index ccab82bd13f0..a70c38261614 100644 --- a/browser_tests/fixtures/agentConversationFixture.ts +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -15,7 +15,6 @@ import { mintWireOps } from '@/workbench/extensions/agent/crdt/opEnvelope' import type { AgentCancelAccepted, AgentMessages, - AgentTurnAccepted, AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' @@ -23,6 +22,7 @@ import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApi import { agentTest, bootAgentApp } from '@e2e/fixtures/agentPanelFixture' import type { AgentConversation, + AgentConversationTurn, RecordedWsEvent } from '@e2e/fixtures/data/agent/agentConversation' import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' @@ -32,7 +32,9 @@ import { knownTool } from '@/workbench/extensions/agent/services/agent/agentTool import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' -const TURN_ID = '0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5f60' +// One synthetic message id per turn; the recorded ids never reach the page. +const turnId = (turn: number): string => + `0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5${turn.toString(16).padStart(3, '0')}` // useAgentSession.ts keeps THREAD_STORAGE_KEY module-private. const THREAD_STORAGE_KEY = 'Comfy.Agent.ThreadId' const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' @@ -190,7 +192,6 @@ class AgentConversationHarness { // Human-op minting is observable only on the client side of the socket. readonly clientFrames: { type?: unknown; data?: unknown }[] = [] readonly panel: Locator - readonly ack: AgentTurnAccepted private readonly host: HostDoc private socket: WebSocketRoute | null = null @@ -206,11 +207,6 @@ class AgentConversationHarness { ) { const { workflow } = conversation this.host = new HostDoc(workflow.id, workflow.seed, workflow.catalog) - this.ack = { - thread_id: THREAD_ID, - message_id: TURN_ID, - workflow_id: workflow.id - } this.panel = page.locator('#agent-panel-root') } @@ -239,15 +235,15 @@ class AgentConversationHarness { await expect(this.panel).toBeVisible({ timeout: PANEL_MOUNT_TIMEOUT }) } - async sendPrompt(): Promise { - const { content } = this.conversation.request + async sendPrompt(turn = 0): Promise { + const { content } = this.conversation.turns[turn].request const composer = this.panel.getByRole('textbox', { name: /^Describe ideas/ }) await composer.fill(content) await this.panel.getByRole('button', { name: SEND_LABEL }).click() - await expect.poll(() => this.postedMessages.length).toBeGreaterThan(0) - expect(this.postedMessages[0]).toContain(content) + await expect.poll(() => this.postedMessages.length).toBeGreaterThan(turn) + expect(this.postedMessages[turn]).toContain(content) // Replay frames are dropped until the page has applied the ack's thread id. await expect .poll(() => @@ -260,16 +256,23 @@ class AgentConversationHarness { } async replayResponse( + turn = 0, afterEntry?: (index: number) => Promise ): Promise { const startedAt = Date.now() - for (const [index, entry] of this.conversation.response.entries()) { - if (this.replayTiming === 'recorded' && entry.at_ms !== undefined) { - const wait = entry.at_ms - (Date.now() - startedAt) - if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait)) - } + const entries = this.conversation.turns[turn].response.entries() + for (const [index, entry] of entries) { + // A timer can fire a millisecond early, so wait until the offset has really passed. + while ( + this.replayTiming === 'recorded' && + entry.at_ms !== undefined && + Date.now() - startedAt < entry.at_ms + ) + await new Promise((resolve) => + setTimeout(resolve, entry.at_ms! - (Date.now() - startedAt)) + ) if (entry.kind === 'event') { - this.send(this.stampTurn(entry.event)) + this.send(this.stampTurn(entry.event, turn)) continue } await this.waitForSubscribe() @@ -281,6 +284,18 @@ class AgentConversationHarness { replayElapsedMs = 0 + async runTurns(): Promise { + for (const turn of this.conversation.turns.keys()) { + await this.sendPrompt(turn) + await this.replayResponse(turn) + await this.waitForTurnComplete() + } + } + + private entries(): AgentConversationTurn['response'] { + return this.conversation.turns.flatMap((turn) => turn.response) + } + async waitForTurnComplete(): Promise { await expect( this.panel.getByRole('button', { name: SEND_LABEL }) @@ -293,7 +308,7 @@ class AgentConversationHarness { // Doc-id filter: a stray template node must not pin the template here. private nodeBodies(): NodeBody[] { const seed = this.conversation.workflow.seed.nodes as NodeBody[] - const added = this.conversation.response.flatMap((entry) => + const added = this.entries().flatMap((entry) => entry.kind === 'graph_ops' ? entry.ops.flatMap((op) => op.op === 'add_node' ? [op.node as NodeBody] : [] @@ -370,27 +385,29 @@ class AgentConversationHarness { toolCallGroups(): RecordedToolCall[][] { const groups: RecordedToolCall[][] = [] - let current: RecordedToolCall[] = [] - for (const entry of this.conversation.response) { - if (entry.kind !== 'event') continue - const { type, data } = entry.event - if (type === 'agent_tool_call') { - if (data.status !== 'running') - current.push({ - name: String(data.tool_name), - ok: data.status === 'success' - }) - continue - } - if ( - (type === 'agent_thinking' || type === 'agent_message_delta') && - current.length > 0 - ) { - groups.push(current) - current = [] + for (const turn of this.conversation.turns) { + let current: RecordedToolCall[] = [] + for (const entry of turn.response) { + if (entry.kind !== 'event') continue + const { type, data } = entry.event + if (type === 'agent_tool_call') { + if (data.status !== 'running') + current.push({ + name: String(data.tool_name), + ok: data.status === 'success' + }) + continue + } + if ( + (type === 'agent_thinking' || type === 'agent_message_delta') && + current.length > 0 + ) { + groups.push(current) + current = [] + } } + if (current.length > 0) groups.push(current) } - if (current.length > 0) groups.push(current) return groups } @@ -398,7 +415,7 @@ class AgentConversationHarness { recordedWidgetValues(): RecordedWidgetValue[] { const graph = this.host.graph() const latest = new Map() - for (const entry of this.conversation.response) { + for (const entry of this.entries()) { if (entry.kind !== 'graph_ops') continue for (const op of entry.ops) { if (op.op !== 'set_widget') continue @@ -428,8 +445,7 @@ class AgentConversationHarness { // One update per ops entry plus the subscribe catch-up. expectedUpdateCount(): number { return ( - this.conversation.response.filter((entry) => entry.kind === 'graph_ops') - .length + 1 + this.entries().filter((entry) => entry.kind === 'graph_ops').length + 1 ) } @@ -475,7 +491,11 @@ class AgentConversationHarness { return route.fulfill({ status: 202, contentType: 'application/json', - body: JSON.stringify(this.ack) + body: JSON.stringify({ + thread_id: THREAD_ID, + message_id: turnId(this.postedMessages.length - 1), + workflow_id: this.conversation.workflow.id + }) }) } const history: AgentMessages = [] @@ -495,10 +515,10 @@ class AgentConversationHarness { ) } - private stampTurn(event: RecordedWsEvent): AgentWsEvent { + private stampTurn(event: RecordedWsEvent, turn: number): AgentWsEvent { const stamped = { type: event.type, - data: { ...event.data, message_id: TURN_ID, thread_id: THREAD_ID } + data: { ...event.data, message_id: turnId(turn), thread_id: THREAD_ID } } const parsed = parseAgentWsEvent(stamped) if (!parsed.success) diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts index 31136126997f..b81b9ce11e06 100644 --- a/browser_tests/fixtures/data/agent/agentConversation.ts +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -58,9 +58,16 @@ const zResponseEntry = z.discriminatedUnion('kind', [ }) ]) +const zTurn = z.object({ + message_id: z.string().min(1).optional(), + request: zAgentConversationRequest, + response: z.array(zResponseEntry).min(1) +}) +export type AgentConversationTurn = z.infer + export const zAgentConversation = z .object({ - schema_version: z.literal('agent-conversation.v1'), + schema_version: z.literal('agent-conversation.v2'), source: z.object({ repo: z.string(), suite: z.string(), @@ -71,26 +78,29 @@ export const zAgentConversation = z .object({ backend: z.literal('Comfy-Org/cloud'), thread_id: z.string().min(1), - message_id: z.string().min(1), exported_at: z.string().datetime() }) .optional() }), workflow: zAgentConversationWorkflow, - request: zAgentConversationRequest, - response: z.array(zResponseEntry).min(1) + // One thread; each turn lands on the graph the previous turn left. + turns: z.array(zTurn).min(1) }) .superRefine((conversation, ctx) => { - if ( - conversation.source.response_side === 'recorded' && - conversation.source.capture === undefined - ) { + if (conversation.source.response_side !== 'recorded') return + if (conversation.source.capture === undefined) ctx.addIssue({ code: 'custom', path: ['source', 'capture'], message: 'recorded responses require backend capture provenance' }) - } + for (const [index, turn] of conversation.turns.entries()) + if (turn.message_id === undefined) + ctx.addIssue({ + code: 'custom', + path: ['turns', index, 'message_id'], + message: 'recorded turns carry the message id they were captured from' + }) }) export type AgentConversation = z.infer diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json index b759d1ac505f..55e16cb9a37f 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "91a905f3-b619-44e3-a6e5-5a047d8d0066", - "message_id": "4293701d-cd6f-41cf-b03d-ea73df6af846", "exported_at": "2026-09-03T12:40:57Z" } }, @@ -194,357 +193,362 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch over and take a look at that workflow." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "891d1304-09ae-4d51-9bea-aaa4756822d4" - } - }, - "at_ms": 6 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 6, - "status": "success", - "tool_call_id": "toolu_01KZvQdg1MdhKgbWQsvScSpN", - "tool_name": "switch_tab" - } - }, - "at_ms": 9 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 774, - "status": "success", - "tool_call_id": "toolu_015MGXQKWWD64ZWnv7jWQNLw", - "tool_name": "print_workflow" - } - }, - "at_ms": 2809 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 308, - "status": "success", - "tool_call_id": "toolu_015ngjjFSSFFFRrsQvJwbnut", - "tool_name": "find_nodes" - } - }, - "at_ms": 7587 - }, - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "The prompt node is easy — but this graph is also missing a couple of connections it needs to run, so I'll complete those too." - } + "message_id": "4293701d-cd6f-41cf-b03d-ea73df6af846", + "request": { + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." }, - "at_ms": 13907 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [395, 280], - "node": { - "id": 3876406316923056, - "pos": [395, 280], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look at that workflow." + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b41f1bc5b13f44efb10bb51847859813", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 3876406316923056, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 + "at_ms": 0 }, { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "90d12d67f1a54617af277c333f9880be", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a red bicycle on a pier at dusk", - "widget": "value", - "node_id": 3876406316923056, - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "891d1304-09ae-4d51-9bea-aaa4756822d4" + } + }, + "at_ms": 6 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c0b8ec72e8d64de7bcf9b45125f6c207", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 1887935066439213, - "to_node": 6, - "to_slot": 1, - "from_node": 3876406316923056, - "from_slot": 0, - "link_type": "STRING", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01KZvQdg1MdhKgbWQsvScSpN", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 }, { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 3880387162517873, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 774, + "status": "success", + "tool_call_id": "toolu_015MGXQKWWD64ZWnv7jWQNLw", + "tool_name": "print_workflow" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "a5da691f2d07486aa354c5ee7c7fb6f1", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 3880387162517873, - "class_type": "CLIPTextEncode", - "base_version": 2 + "at_ms": 2809 }, { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "d02e085ef78d41109f09ed846715fdfa", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "", - "widget": "text", - "node_id": 3880387162517873, - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 308, + "status": "success", + "tool_call_id": "toolu_015ngjjFSSFFFRrsQvJwbnut", + "tool_name": "find_nodes" + } + }, + "at_ms": 7587 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "54508721c33e478b94db088a93568f4c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2636276550287764, - "to_node": 3880387162517873, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "The prompt node is easy — but this graph is also missing a couple of connections it needs to run, so I'll complete those too." + } + }, + "at_ms": 13907 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "98f104d28a664643b45551a76f4b6733", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2833535472085255, - "to_node": 3, - "to_slot": 2, - "from_node": 3880387162517873, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [395, 280], + "node": { + "id": 3876406316923056, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b41f1bc5b13f44efb10bb51847859813", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876406316923056, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "90d12d67f1a54617af277c333f9880be", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 3876406316923056, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c0b8ec72e8d64de7bcf9b45125f6c207", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1887935066439213, + "to_node": 6, + "to_slot": 1, + "from_node": 3876406316923056, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3880387162517873, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5da691f2d07486aa354c5ee7c7fb6f1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3880387162517873, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "d02e085ef78d41109f09ed846715fdfa", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "", + "widget": "text", + "node_id": 3880387162517873, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "54508721c33e478b94db088a93568f4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2636276550287764, + "to_node": 3880387162517873, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "98f104d28a664643b45551a76f4b6733", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2833535472085255, + "to_node": 3, + "to_slot": 2, + "from_node": 3880387162517873, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 3876084544359242, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "9e977850c86447acb7359b7a9b484a83", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876084544359242, + "class_type": "EmptyLatentImage", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c9e0686dff2245fa90d9e720c0133928", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2211554611670965, + "to_node": 3, + "to_slot": 3, + "from_node": 3876084544359242, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + } + ], + "at_ms": 14347 }, { - "op": "add_node", - "pos": [395, 406], - "node": { - "id": 3876084544359242, - "pos": [395, 406], - "mode": 0, - "size": [240, 134], - "type": "EmptyLatentImage", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [512, 512, 1] + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 447, + "status": "success", + "tool_call_id": "toolu_0126og8icdp1EUnSmtrcSZv6", + "tool_name": "apply_ops" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "9e977850c86447acb7359b7a9b484a83", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 3876084544359242, - "class_type": "EmptyLatentImage", - "base_version": 2 + "at_ms": 14347 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c9e0686dff2245fa90d9e720c0133928", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2211554611670965, - "to_node": 3, - "to_slot": 3, - "from_node": 3876084544359242, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 - } - ], - "at_ms": 14347 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 447, - "status": "success", - "tool_call_id": "toolu_0126og8icdp1EUnSmtrcSZv6", - "tool_name": "apply_ops" - } - }, - "at_ms": 14347 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 249, - "status": "error", - "tool_call_id": "toolu_01RQTp9JXwFc1AdgmAKBiMYm", - "tool_name": "validate" - } - }, - "at_ms": 16172 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 232, - "status": "success", - "tool_call_id": "toolu_01Lq8eZseF7AhmTd9UuxvcNb", - "tool_name": "show_node" - } - }, - "at_ms": 19599 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 248, - "error_code": "knowledge_unavailable", - "status": "error", - "tool_call_id": "toolu_01VcWs7EVFuk18ct7nX9nEoi", - "tool_name": "list_model_picks" - } - }, - "at_ms": 24061 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 887, - "status": "success", - "tool_call_id": "toolu_01AB6XXLQrdF3EFbDRV9GtSx", - "tool_name": "list_generate_models" - } - }, - "at_ms": 28706 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 324, - "status": "success", - "tool_call_id": "toolu_01U2bR9WL8Pc5H35X2JERBwr", - "tool_name": "print_workflow" - } - }, - "at_ms": 38928 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all.\n\nI couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up.\n\nTwo ways forward, your call:\n- Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct.\n- Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first." - } - }, - "at_ms": 44823 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 4467, - "cache_read_input_tokens": 395126, - "input_tokens": 20, - "output_tokens": 2467, - "total_tokens": 2487 - } + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 249, + "status": "error", + "tool_call_id": "toolu_01RQTp9JXwFc1AdgmAKBiMYm", + "tool_name": "validate" + } + }, + "at_ms": 16172 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 232, + "status": "success", + "tool_call_id": "toolu_01Lq8eZseF7AhmTd9UuxvcNb", + "tool_name": "show_node" + } + }, + "at_ms": 19599 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 248, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01VcWs7EVFuk18ct7nX9nEoi", + "tool_name": "list_model_picks" + } + }, + "at_ms": 24061 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 887, + "status": "success", + "tool_call_id": "toolu_01AB6XXLQrdF3EFbDRV9GtSx", + "tool_name": "list_generate_models" + } + }, + "at_ms": 28706 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 324, + "status": "success", + "tool_call_id": "toolu_01U2bR9WL8Pc5H35X2JERBwr", + "tool_name": "print_workflow" + } + }, + "at_ms": 38928 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all.\n\nI couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up.\n\nTwo ways forward, your call:\n- Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct.\n- Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first." + } + }, + "at_ms": 44823 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 4467, + "cache_read_input_tokens": 395126, + "input_tokens": 20, + "output_tokens": 2467, + "total_tokens": 2487 + } + } + }, + "at_ms": 44827 } - }, - "at_ms": 44827 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json index 0a9624b06b65..89acc9502c74 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "a6fd73be-e957-45b7-a5e0-5edd091670fb", - "message_id": "9f88da92-5074-4091-850b-7cbc65005cd1", "exported_at": "2026-09-03T12:40:55Z" } }, @@ -194,187 +193,192 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "e539bb63-bc5d-4f97-b918-d85862c0f591" - } - }, - "at_ms": 0 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 7, - "status": "success", - "tool_call_id": "toolu_01UJ5hWx9Rxkvwon9F7nMGaK", - "tool_name": "switch_tab" - } - }, - "at_ms": 2 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 368, - "status": "success", - "tool_call_id": "toolu_011QfP8raZRXJbK5WZYpKTkv", - "tool_name": "print_workflow" - } + "message_id": "9f88da92-5074-4091-850b-7cbc65005cd1", + "request": { + "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." }, - "at_ms": 2139 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 2559261621971657, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "e539bb63-bc5d-4f97-b918-d85862c0f591" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7a130c22fe934bc98e2ea2fed6bd10ad", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2559261621971657, - "class_type": "CLIPTextEncode", - "base_version": 2 + "at_ms": 0 }, { - "op": "add_node", - "pos": [715, 406], - "node": { - "id": 2214070360209981, - "pos": [715, 406], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 7, + "status": "success", + "tool_call_id": "toolu_01UJ5hWx9Rxkvwon9F7nMGaK", + "tool_name": "switch_tab" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "00b500c587744366b85d5c71ddbc9fdf", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2214070360209981, - "class_type": "CLIPTextEncode", - "base_version": 2 + "at_ms": 2 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "e9e5c529eb574496a516dd6014ffd5e6", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 879068618000012, - "to_node": 2559261621971657, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_011QfP8raZRXJbK5WZYpKTkv", + "tool_name": "print_workflow" + } + }, + "at_ms": 2139 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "c7f256bd0cd34e92830062aeea907b4c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3832075311322831, - "to_node": 2214070360209981, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 - } - ], - "at_ms": 5208 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "success", - "tool_call_id": "toolu_01PhJ8AyewzRU6PHk7aFGfbC", - "tool_name": "apply_ops" - } - }, - "at_ms": 5208 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt." - } - }, - "at_ms": 7784 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 1437, - "cache_read_input_tokens": 153211, - "input_tokens": 8, - "output_tokens": 574, - "total_tokens": 582 - } + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2559261621971657, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7a130c22fe934bc98e2ea2fed6bd10ad", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2559261621971657, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 406], + "node": { + "id": 2214070360209981, + "pos": [715, 406], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "00b500c587744366b85d5c71ddbc9fdf", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2214070360209981, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e9e5c529eb574496a516dd6014ffd5e6", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 879068618000012, + "to_node": 2559261621971657, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c7f256bd0cd34e92830062aeea907b4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3832075311322831, + "to_node": 2214070360209981, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + } + ], + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01PhJ8AyewzRU6PHk7aFGfbC", + "tool_name": "apply_ops" + } + }, + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt." + } + }, + "at_ms": 7784 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1437, + "cache_read_input_tokens": 153211, + "input_tokens": 8, + "output_tokens": 574, + "total_tokens": 582 + } + } + }, + "at_ms": 7786 } - }, - "at_ms": 7786 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json index c69b9c4fbc17..b5ca120cf8bc 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a", - "message_id": "b363781f-6c9a-4087-9190-d478781bbcd2", "exported_at": "2026-09-03T12:40:55Z" } }, @@ -194,251 +193,256 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and read what's there." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "b7d1e09f-682a-48f5-a388-d74a33ad5184" - } - }, - "at_ms": 5 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 5, - "status": "success", - "tool_call_id": "toolu_01Bq7cRyMamrEnmo4yHuMphE", - "tool_name": "switch_tab" - } - }, - "at_ms": 6 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 747, - "status": "success", - "tool_call_id": "toolu_01FEPdyTv3FtKtL8S6zTKj33", - "tool_name": "print_workflow" - } + "message_id": "b363781f-6c9a-4087-9190-d478781bbcd2", + "request": { + "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." }, - "at_ms": 3053 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [1565, 280], - "node": { - "id": 2092962753103866, - "pos": [1565, 280], - "mode": 0, - "size": [240, 290], - "type": "KSampler", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "model", - "type": "MODEL" - }, - { - "link": null, - "name": "positive", - "type": "CONDITIONING" - }, - { - "link": null, - "name": "negative", - "type": "CONDITIONING" - }, - { - "link": null, - "name": "latent_image", - "type": "LATENT" - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read what's there." + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "a3b5b79791364f75b1d89391a7a87564", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2092962753103866, - "class_type": "KSampler", - "base_version": 2 + "at_ms": 0 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "a5d9733c860c40798848bf92d649988f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 4414751122447181, - "to_node": 2092962753103866, - "to_slot": 3, - "from_node": 3, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b7d1e09f-682a-48f5-a388-d74a33ad5184" + } + }, + "at_ms": 5 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7ced254ebc4a417589c32b5a1797452d", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 863234800411381, - "to_node": 2092962753103866, - "to_slot": 0, - "from_node": 4, - "from_slot": 0, - "link_type": "MODEL", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 5, + "status": "success", + "tool_call_id": "toolu_01Bq7cRyMamrEnmo4yHuMphE", + "tool_name": "switch_tab" + } + }, + "at_ms": 6 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "57585fd740934c15835cef60c216c84c", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3220756471207314, - "to_node": 2092962753103866, - "to_slot": 1, - "from_node": 6, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 747, + "status": "success", + "tool_call_id": "toolu_01FEPdyTv3FtKtL8S6zTKj33", + "tool_name": "print_workflow" + } + }, + "at_ms": 3053 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "bb113cee68314103861361151c93ad36", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 619699595909791, - "to_node": 8, - "to_slot": 0, - "from_node": 2092962753103866, - "from_slot": 0, - "link_type": "LATENT", - "base_version": 2 + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1565, 280], + "node": { + "id": 2092962753103866, + "pos": [1565, 280], + "mode": 0, + "size": [240, 290], + "type": "KSampler", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "model", + "type": "MODEL" + }, + { + "link": null, + "name": "positive", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "negative", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "latent_image", + "type": "LATENT" + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a3b5b79791364f75b1d89391a7a87564", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2092962753103866, + "class_type": "KSampler", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5d9733c860c40798848bf92d649988f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4414751122447181, + "to_node": 2092962753103866, + "to_slot": 3, + "from_node": 3, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ced254ebc4a417589c32b5a1797452d", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 863234800411381, + "to_node": 2092962753103866, + "to_slot": 0, + "from_node": 4, + "from_slot": 0, + "link_type": "MODEL", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "57585fd740934c15835cef60c216c84c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3220756471207314, + "to_node": 2092962753103866, + "to_slot": 1, + "from_node": 6, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "bb113cee68314103861361151c93ad36", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 619699595909791, + "to_node": 8, + "to_slot": 0, + "from_node": 2092962753103866, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "set_widget", + "old": 1, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8ba837eb610e4e0db4561b789bb44953", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 0.5, + "widget": "denoise", + "node_id": 2092962753103866, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e83bac70162244fa96c7d4633dd5fccc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 20, + "widget": "steps", + "node_id": 2092962753103866, + "base_version": 2 + } + ], + "at_ms": 11793 }, { - "op": "set_widget", - "old": 1, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "8ba837eb610e4e0db4561b789bb44953", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 0.5, - "widget": "denoise", - "node_id": 2092962753103866, - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 198, + "status": "success", + "tool_call_id": "toolu_01LqYJ1bi41MTaeDXdzC9A9j", + "tool_name": "apply_ops" + } + }, + "at_ms": 11793 }, { - "op": "set_widget", - "old": 20, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "e83bac70162244fa96c7d4633dd5fccc", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 20, - "widget": "steps", - "node_id": 2092962753103866, - "base_version": 2 - } - ], - "at_ms": 11793 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 198, - "status": "success", - "tool_call_id": "toolu_01LqYJ1bi41MTaeDXdzC9A9j", - "tool_name": "apply_ops" - } - }, - "at_ms": 11793 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 379, - "status": "success", - "tool_call_id": "toolu_01DAfeiUo97WETr7TRCrv4Cp", - "tool_name": "print_workflow" - } - }, - "at_ms": 14468 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "error", - "tool_call_id": "toolu_01G8xCCA9NuoEur1n8RySCWK", - "tool_name": "validate" - } - }, - "at_ms": 19339 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node.\n\nTwo things were already unfinished in this graph before I touched it, and they'll block a run:\n\n- Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler.\n- The checkpoint it points at isn't installed here, so no model file is actually available.\n\nI set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?" - } - }, - "at_ms": 29476 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2906, - "cache_read_input_tokens": 232638, - "input_tokens": 12, - "output_tokens": 1422, - "total_tokens": 1434 - } + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 379, + "status": "success", + "tool_call_id": "toolu_01DAfeiUo97WETr7TRCrv4Cp", + "tool_name": "print_workflow" + } + }, + "at_ms": 14468 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_01G8xCCA9NuoEur1n8RySCWK", + "tool_name": "validate" + } + }, + "at_ms": 19339 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node.\n\nTwo things were already unfinished in this graph before I touched it, and they'll block a run:\n\n- Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler.\n- The checkpoint it points at isn't installed here, so no model file is actually available.\n\nI set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?" + } + }, + "at_ms": 29476 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2906, + "cache_read_input_tokens": 232638, + "input_tokens": 12, + "output_tokens": 1422, + "total_tokens": 1434 + } + } + }, + "at_ms": 29482 } - }, - "at_ms": 29482 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json index f25d041a11d0..964ce2952c2f 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "2ce25f4b-d581-4348-ab9c-3f4170222f6f", - "message_id": "37504547-e90d-4c14-91c7-b8f8fb65a0eb", "exported_at": "2026-09-03T12:40:55Z" } }, @@ -194,214 +193,219 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first." - } - }, - "at_ms": 0 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90" - } - }, - "at_ms": 7 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 10, - "status": "success", - "tool_call_id": "toolu_01PbXSHPoAUnHhX2dqzJXiLB", - "tool_name": "switch_tab" - } - }, - "at_ms": 11 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 295, - "status": "success", - "tool_call_id": "toolu_01C2g2RACFUykeibS1je1Tbh", - "tool_name": "print_workflow" - } + "message_id": "37504547-e90d-4c14-91c7-b8f8fb65a0eb", + "request": { + "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." }, - "at_ms": 3673 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [715, 280], - "node": { - "id": 4181654812796082, - "pos": [715, 280], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "4c4d739392f14272939cf46a335652c8", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 4181654812796082, - "class_type": "CLIPTextEncode", - "base_version": 2 + "at_ms": 0 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "f1cd3aef4b984b04a9611a53f69763ba", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3320191292493608, - "to_node": 4181654812796082, - "to_slot": 0, - "from_node": 4, - "from_slot": 1, - "link_type": "CLIP", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90" + } + }, + "at_ms": 7 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "4f4de6968f124e23a3114f8c88a1a766", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2686273519871711, - "to_node": 3, - "to_slot": 1, - "from_node": 4181654812796082, - "from_slot": 0, - "link_type": "CONDITIONING", - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 10, + "status": "success", + "tool_call_id": "toolu_01PbXSHPoAUnHhX2dqzJXiLB", + "tool_name": "switch_tab" + } + }, + "at_ms": 11 }, { - "op": "set_widget", - "old": null, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "279c55a82f09416f9fa68e47ee11fafc", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": "a photo of a pier", - "widget": "text", - "node_id": 4181654812796082, - "base_version": 2 - } - ], - "at_ms": 10451 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 223, - "status": "success", - "tool_call_id": "toolu_018Ao8cDcxmDBaz9qV7PWpGx", - "tool_name": "apply_ops" - } - }, - "at_ms": 10451 - }, - { - "kind": "graph_ops", - "ops": [ + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 295, + "status": "success", + "tool_call_id": "toolu_01C2g2RACFUykeibS1je1Tbh", + "tool_name": "print_workflow" + } + }, + "at_ms": 3673 + }, { - "op": "delete_node", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "b141d8503ee54cde94d3a4e3f6b5204f", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 6, - "base_version": 3, - "removed_links": [1] - } - ], - "at_ms": 15515 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 202, - "status": "success", - "tool_call_id": "toolu_01QkN5N4e8fkpBrT9aWbb4aK", - "tool_name": "delete_node" - } - }, - "at_ms": 15515 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "success", - "tool_call_id": "toolu_01QUTSXXBfU7oShpWRkxtLzD", - "tool_name": "print_workflow" - } - }, - "at_ms": 18022 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — I'm on the \"Text to image\" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler's positive input. I carried over the same prompt text (\"a photo of a pier\").\n\nOne thing unrelated to this swap: the sampler's negative prompt and empty latent inputs were already unconnected before I started, so the graph won't run until those are filled in. Want me to add them?" - } - }, - "at_ms": 22947 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 2216, - "cache_read_input_tokens": 232121, - "input_tokens": 12, - "output_tokens": 885, - "total_tokens": 897 - } + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 4181654812796082, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4c4d739392f14272939cf46a335652c8", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4181654812796082, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "f1cd3aef4b984b04a9611a53f69763ba", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3320191292493608, + "to_node": 4181654812796082, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4f4de6968f124e23a3114f8c88a1a766", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2686273519871711, + "to_node": 3, + "to_slot": 1, + "from_node": 4181654812796082, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "279c55a82f09416f9fa68e47ee11fafc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier", + "widget": "text", + "node_id": 4181654812796082, + "base_version": 2 + } + ], + "at_ms": 10451 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 223, + "status": "success", + "tool_call_id": "toolu_018Ao8cDcxmDBaz9qV7PWpGx", + "tool_name": "apply_ops" + } + }, + "at_ms": 10451 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b141d8503ee54cde94d3a4e3f6b5204f", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 6, + "base_version": 3, + "removed_links": [1] + } + ], + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 202, + "status": "success", + "tool_call_id": "toolu_01QkN5N4e8fkpBrT9aWbb4aK", + "tool_name": "delete_node" + } + }, + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01QUTSXXBfU7oShpWRkxtLzD", + "tool_name": "print_workflow" + } + }, + "at_ms": 18022 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm on the \"Text to image\" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler's positive input. I carried over the same prompt text (\"a photo of a pier\").\n\nOne thing unrelated to this swap: the sampler's negative prompt and empty latent inputs were already unconnected before I started, so the graph won't run until those are filled in. Want me to add them?" + } + }, + "at_ms": 22947 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2216, + "cache_read_input_tokens": 232121, + "input_tokens": 12, + "output_tokens": 885, + "total_tokens": 897 + } + } + }, + "at_ms": 22951 } - }, - "at_ms": 22951 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json index b73aebeaaa99..695b4c38a9e8 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "b5d11187-d40e-4be1-aeab-39869516773b", - "message_id": "19230873-0f07-4f85-a0ae-e9f1d740cb3a", "exported_at": "2026-09-03T12:38:52Z" } }, @@ -194,149 +193,154 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "5733b17e-39f4-40f5-aa1c-468a4f355b55" - } - }, - "at_ms": 6 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 12, - "status": "success", - "tool_call_id": "toolu_0127MABi8i6cpznwzfMSk2b6", - "tool_name": "switch_tab" - } - }, - "at_ms": 12 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 498, - "status": "success", - "tool_call_id": "toolu_01DTMkDXHKWELHfKUTXUynfK", - "tool_name": "ls_nodes" - } - }, - "at_ms": 3903 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 425, - "status": "success", - "tool_call_id": "toolu_018Ck46iqTkvLjT12cDdcyH6", - "tool_name": "list_slots" - } + "message_id": "19230873-0f07-4f85-a0ae-e9f1d740cb3a", + "request": { + "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." }, - "at_ms": 9834 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "set_widget", - "old": 20, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "5ee9143999c148439b6fc848bc8d309a", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 30, - "widget": "steps", - "node_id": 3, - "base_version": 2 + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 }, { - "op": "set_widget", - "old": 7, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7ad6e06be4f84086bec999cfc5177816", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "value": 5, - "widget": "cfg", - "node_id": 3, - "base_version": 2 - } - ], - "at_ms": 16908 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 253, - "status": "success", - "tool_call_id": "toolu_01BaTP7o2iAhJbBjQ9tChoEG", - "tool_name": "apply_ops" - } - }, - "at_ms": 16908 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 199, - "status": "success", - "tool_call_id": "toolu_01MLp2wvSBojXfwPYKqT7dvB", - "tool_name": "list_slots" - } - }, - "at_ms": 21478 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "You're now on the \"Text to image\" tab, with steps at 30 and cfg at 5. Nothing else was touched." - } - }, - "at_ms": 23927 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 1549, - "cache_read_input_tokens": 230832, - "input_tokens": 12, - "output_tokens": 509, - "total_tokens": 521 - } + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5733b17e-39f4-40f5-aa1c-468a4f355b55" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 12, + "status": "success", + "tool_call_id": "toolu_0127MABi8i6cpznwzfMSk2b6", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 498, + "status": "success", + "tool_call_id": "toolu_01DTMkDXHKWELHfKUTXUynfK", + "tool_name": "ls_nodes" + } + }, + "at_ms": 3903 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 425, + "status": "success", + "tool_call_id": "toolu_018Ck46iqTkvLjT12cDdcyH6", + "tool_name": "list_slots" + } + }, + "at_ms": 9834 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "5ee9143999c148439b6fc848bc8d309a", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 30, + "widget": "steps", + "node_id": 3, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 7, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ad6e06be4f84086bec999cfc5177816", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 5, + "widget": "cfg", + "node_id": 3, + "base_version": 2 + } + ], + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 253, + "status": "success", + "tool_call_id": "toolu_01BaTP7o2iAhJbBjQ9tChoEG", + "tool_name": "apply_ops" + } + }, + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01MLp2wvSBojXfwPYKqT7dvB", + "tool_name": "list_slots" + } + }, + "at_ms": 21478 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "You're now on the \"Text to image\" tab, with steps at 30 and cfg at 5. Nothing else was touched." + } + }, + "at_ms": 23927 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1549, + "cache_read_input_tokens": 230832, + "input_tokens": 12, + "output_tokens": 509, + "total_tokens": 521 + } + } + }, + "at_ms": 23931 } - }, - "at_ms": 23931 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json index ed3c3c5b48ed..66fc1ca04a23 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "f36ef047-82b4-4f2b-b480-6df821df46da", - "message_id": "7de6039c-c1b4-437c-871c-2f1fc67e1750", "exported_at": "2026-09-03T12:40:56Z" } }, @@ -194,82 +193,87 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then tell me what this workflow does." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch over and take a look." - } - }, - "at_ms": 0 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "d762837d-c64d-4829-9aba-05e71b64adc3" - } - }, - "at_ms": 6 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 8, - "status": "success", - "tool_call_id": "toolu_013WLWbbrNwTqV9UguY6rEu6", - "tool_name": "switch_tab" - } - }, - "at_ms": 8 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 368, - "status": "success", - "tool_call_id": "toolu_01PjsdjzNXWZ8oDYHQAbjo5o", - "tool_name": "print_workflow" - } - }, - "at_ms": 2364 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I'm on the \"Text to image\" tab now.\n\nIt's a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently \"a photo of a pier\" — into guidance, generates an image over 20 steps, and saves the result.\n\nOne thing to flag: it isn't runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?" - } + "message_id": "7de6039c-c1b4-437c-871c-2f1fc67e1750", + "request": { + "content": "Switch to the tab 'Text to image', then tell me what this workflow does." }, - "at_ms": 6244 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 960, - "cache_read_input_tokens": 114249, - "input_tokens": 6, - "output_tokens": 349, - "total_tokens": 355 - } + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d762837d-c64d-4829-9aba-05e71b64adc3" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_013WLWbbrNwTqV9UguY6rEu6", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_01PjsdjzNXWZ8oDYHQAbjo5o", + "tool_name": "print_workflow" + } + }, + "at_ms": 2364 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now.\n\nIt's a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently \"a photo of a pier\" — into guidance, generates an image over 20 steps, and saves the result.\n\nOne thing to flag: it isn't runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?" + } + }, + "at_ms": 6244 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 960, + "cache_read_input_tokens": 114249, + "input_tokens": 6, + "output_tokens": 349, + "total_tokens": 355 + } + } + }, + "at_ms": 6246 } - }, - "at_ms": 6246 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json index 04a3b7fdaa63..9b3462f25909 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "3710f44b-9941-4bc4-a806-5c15d00cd807", - "message_id": "d0d6b5dc-b189-491a-804e-f52fdb9495cb", "exported_at": "2026-09-03T14:05:21Z" } }, @@ -194,219 +193,224 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then add three nodes one at a time, each with its own separate tool call and nothing else in that call: first a PrimitiveStringMultiline, then an EmptyLatentImage, then a CLIPTextEncode. Do not connect or configure them." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch focus to that tab first." - } - }, - "at_ms": 0 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "7afd1f94-6cb5-4923-9388-18851b0fba5d" - } - }, - "at_ms": 7 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 8, - "status": "success", - "tool_call_id": "toolu_0199bXc7oQVbgJA4aY56VcfY", - "tool_name": "switch_tab" - } + "message_id": "d0d6b5dc-b189-491a-804e-f52fdb9495cb", + "request": { + "content": "Switch to the tab 'Text to image', then add three nodes one at a time, each with its own separate tool call and nothing else in that call: first a PrimitiveStringMultiline, then an EmptyLatentImage, then a CLIPTextEncode. Do not connect or configure them." }, - "at_ms": 9 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [1845, 0], - "node": { - "id": 2478057798252548, - "pos": [1845, 0], - "mode": 0, - "size": [240, 86], - "type": "PrimitiveStringMultiline", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [] - } - ], - "properties": {}, - "widgets_values": [null] + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch focus to that tab first." + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "cf6a6fdb84ee4f7dbec557f999d2fa8f", - "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2478057798252548, - "class_type": "PrimitiveStringMultiline", - "base_version": 2 - } - ], - "at_ms": 2686 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 964, - "status": "success", - "tool_call_id": "toolu_014VjtpNeXB9q2rBXxK9bkm9", - "tool_name": "add_node" - } - }, - "at_ms": 2686 - }, - { - "kind": "graph_ops", - "ops": [ + "at_ms": 0 + }, { - "op": "add_node", - "pos": [2165, 0], - "node": { - "id": 2772376668635982, - "pos": [2165, 0], - "mode": 0, - "size": [240, 134], - "type": "EmptyLatentImage", - "flags": {}, - "order": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [] - } - ], - "properties": {}, - "widgets_values": [512, 512, 1] + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "7afd1f94-6cb5-4923-9388-18851b0fba5d" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "48a4c2f2ea8647d1be5b0817985bda87", - "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2772376668635982, - "class_type": "EmptyLatentImage", - "base_version": 3 - } - ], - "at_ms": 4441 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 220, - "status": "success", - "tool_call_id": "toolu_01M6Jpuof94JLfgBij4etkaN", - "tool_name": "add_node" - } - }, - "at_ms": 4441 - }, - { - "kind": "graph_ops", - "ops": [ + "at_ms": 7 + }, { - "op": "add_node", - "pos": [2485, 0], - "node": { - "id": 2514973844700532, - "pos": [2485, 0], - "mode": 0, - "size": [240, 86], - "type": "CLIPTextEncode", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "clip", - "type": "CLIP" - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [] + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_0199bXc7oQVbgJA4aY56VcfY", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1845, 0], + "node": { + "id": 2478057798252548, + "pos": [1845, 0], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "cf6a6fdb84ee4f7dbec557f999d2fa8f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2478057798252548, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + } + ], + "at_ms": 2686 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 964, + "status": "success", + "tool_call_id": "toolu_014VjtpNeXB9q2rBXxK9bkm9", + "tool_name": "add_node" + } + }, + "at_ms": 2686 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2165, 0], + "node": { + "id": 2772376668635982, + "pos": [2165, 0], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "48a4c2f2ea8647d1be5b0817985bda87", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2772376668635982, + "class_type": "EmptyLatentImage", + "base_version": 3 + } + ], + "at_ms": 4441 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 220, + "status": "success", + "tool_call_id": "toolu_01M6Jpuof94JLfgBij4etkaN", + "tool_name": "add_node" + } + }, + "at_ms": 4441 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2485, 0], + "node": { + "id": 2514973844700532, + "pos": [2485, 0], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "da3e136d6b224cc8838a0d9aa569ede3", + "stamp": [4, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2514973844700532, + "class_type": "CLIPTextEncode", + "base_version": 4 + } + ], + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 192, + "status": "success", + "tool_call_id": "toolu_01FzkN53SHp6ePAFbmX2H8oX", + "tool_name": "add_node" + } + }, + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "All three nodes are on the \"Text to image\" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked." + } + }, + "at_ms": 9873 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1944, + "cache_read_input_tokens": 192480, + "input_tokens": 10, + "output_tokens": 322, + "total_tokens": 332 } - ], - "properties": {}, - "widgets_values": [null] + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "da3e136d6b224cc8838a0d9aa569ede3", - "stamp": [4, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 2514973844700532, - "class_type": "CLIPTextEncode", - "base_version": 4 - } - ], - "at_ms": 7816 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 192, - "status": "success", - "tool_call_id": "toolu_01FzkN53SHp6ePAFbmX2H8oX", - "tool_name": "add_node" - } - }, - "at_ms": 7816 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "All three nodes are on the \"Text to image\" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked." + "at_ms": 9875 } - }, - "at_ms": 9873 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 1944, - "cache_read_input_tokens": 192480, - "input_tokens": 10, - "output_tokens": 322, - "total_tokens": 332 - } - } - }, - "at_ms": 9875 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json index 4b2c416321e5..0508a4b8c07b 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386", - "message_id": "a5ec5ead-1909-4322-9d62-edbb03a3ded9", "exported_at": "2026-09-03T12:40:56Z" } }, @@ -194,83 +193,88 @@ ] } }, - "request": { - "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." - }, - "response": [ - { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab first." - } - }, - "at_ms": 0 - }, + "turns": [ { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Text to image", - "workflow_id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5" - } - }, - "at_ms": 9 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 8, - "status": "success", - "tool_call_id": "toolu_01YWm16Ejys8FJBofYwL7k9E", - "tool_name": "switch_tab" - } - }, - "at_ms": 12 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 346, - "error_code": "node_not_found", - "status": "error", - "tool_call_id": "toolu_01XqvtmCXKboog1UTZ3MGNJR", - "tool_name": "add_node" - } - }, - "at_ms": 2654 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "I'm now on the \"Text to image\" tab, but there's no node called \"DoesNotExist\" — nothing was added, and the canvas is unchanged.\n\nIf you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I'll find the real one and wire it in." - } + "message_id": "a5ec5ead-1909-4322-9d62-edbb03a3ded9", + "request": { + "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." }, - "at_ms": 5701 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 752, - "cache_read_input_tokens": 114291, - "input_tokens": 6, - "output_tokens": 295, - "total_tokens": 301 - } + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01YWm16Ejys8FJBofYwL7k9E", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 346, + "error_code": "node_not_found", + "status": "error", + "tool_call_id": "toolu_01XqvtmCXKboog1UTZ3MGNJR", + "tool_name": "add_node" + } + }, + "at_ms": 2654 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm now on the \"Text to image\" tab, but there's no node called \"DoesNotExist\" — nothing was added, and the canvas is unchanged.\n\nIf you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I'll find the real one and wire it in." + } + }, + "at_ms": 5701 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 752, + "cache_read_input_tokens": 114291, + "input_tokens": 6, + "output_tokens": 295, + "total_tokens": 301 + } + } + }, + "at_ms": 5703 } - }, - "at_ms": 5703 + ] } ] } diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json index e9722356e35d..7547ef59dcbc 100644 --- a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -1,5 +1,5 @@ { - "schema_version": "agent-conversation.v1", + "schema_version": "agent-conversation.v2", "source": { "repo": "Comfy-Org/ComfyUI_frontend", "suite": "agent", @@ -9,7 +9,6 @@ "capture": { "backend": "Comfy-Org/cloud", "thread_id": "8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3", - "message_id": "b199d9ab-1800-49b2-8a43-458cda7f104d", "exported_at": "2026-09-03T12:40:56Z" } }, @@ -188,233 +187,238 @@ ] } }, - "request": { - "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" - }, - "response": [ + "turns": [ { - "kind": "event", - "event": { - "type": "agent_thinking", - "data": { - "delta": "I'll switch to that tab and take a look at what's there." - } - }, - "at_ms": 0 - }, - { - "kind": "event", - "event": { - "type": "agent_active_tab", - "data": { - "name": "Image edit", - "workflow_id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22" - } - }, - "at_ms": 7 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 9, - "status": "success", - "tool_call_id": "toolu_01RMDkomGUUpDL8wixWq9Fhe", - "tool_name": "switch_tab" - } - }, - "at_ms": 14 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 366, - "status": "success", - "tool_call_id": "toolu_01JECLyu71YJ1niLhepMB63u", - "tool_name": "print_workflow" - } - }, - "at_ms": 2905 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 194, - "status": "success", - "tool_call_id": "toolu_01Sttwp3yTjiF6LdrL5GdmVy", - "tool_name": "find_nodes" - } + "message_id": "b199d9ab-1800-49b2-8a43-458cda7f104d", + "request": { + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" }, - "at_ms": 7034 - }, - { - "kind": "graph_ops", - "ops": [ + "response": [ { - "op": "add_node", - "pos": [1710, 280], - "node": { - "id": 1177948620476144, - "pos": [1710, 280], - "mode": 0, - "size": [240, 178], - "type": "ImageStitch", - "flags": {}, - "order": 0, - "inputs": [ - { - "link": null, - "name": "image1", - "type": "IMAGE" + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Image edit", + "workflow_id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_01RMDkomGUUpDL8wixWq9Fhe", + "tool_name": "switch_tab" + } + }, + "at_ms": 14 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 366, + "status": "success", + "tool_call_id": "toolu_01JECLyu71YJ1niLhepMB63u", + "tool_name": "print_workflow" + } + }, + "at_ms": 2905 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 194, + "status": "success", + "tool_call_id": "toolu_01Sttwp3yTjiF6LdrL5GdmVy", + "tool_name": "find_nodes" + } + }, + "at_ms": 7034 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1710, 280], + "node": { + "id": 1177948620476144, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], + "type": "ImageStitch", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, + "widgets_values": ["right", true, 0, "white"] }, - { - "link": null, - "name": "image2", - "type": "IMAGE" - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [] - } - ], - "properties": {}, - "widgets_values": ["right", true, 0, "white"] + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8e44f8dcf31e46f6bf6d487c1bec2676", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1177948620476144, + "class_type": "ImageStitch", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "792213bf2f824cfd8866f157117795a4", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3298141227647238, + "to_node": 1177948620476144, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7c91fdf62a4d437dac4ada8d3ae13661", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2572689854678654, + "to_node": 1177948620476144, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "3eaf0c9e52cb4f0994962ff4b1e341ff", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3588278739910384, + "to_node": 9, + "to_slot": 0, + "from_node": 1177948620476144, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + } + ], + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 208, + "status": "success", + "tool_call_id": "toolu_01DNrpeJQDjW2v32xme5eyDq", + "tool_name": "apply_ops" + } }, - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "8e44f8dcf31e46f6bf6d487c1bec2676", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "node_id": 1177948620476144, - "class_type": "ImageStitch", - "base_version": 1 + "at_ms": 13620 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "792213bf2f824cfd8866f157117795a4", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3298141227647238, - "to_node": 1177948620476144, - "to_slot": 0, - "from_node": 1, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 210, + "status": "success", + "tool_call_id": "toolu_01VFMK15TdDdT4zFK3w8bRqE", + "tool_name": "list_slots" + } + }, + "at_ms": 20740 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "7c91fdf62a4d437dac4ada8d3ae13661", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 2572689854678654, - "to_node": 1177948620476144, - "to_slot": 1, - "from_node": 8, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 206, + "error_code": "conflict", + "status": "error", + "tool_call_id": "toolu_01GZUZLG9RtoeGym25Hy5p5P", + "tool_name": "set_widget" + } + }, + "at_ms": 25170 }, { - "op": "connect", - "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", - "op_id": "3eaf0c9e52cb4f0994962ff4b1e341ff", - "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], - "link_id": 3588278739910384, - "to_node": 9, - "to_slot": 0, - "from_node": 1177948620476144, - "from_slot": 0, - "link_type": "IMAGE", - "base_version": 1 - } - ], - "at_ms": 13620 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 208, - "status": "success", - "tool_call_id": "toolu_01DNrpeJQDjW2v32xme5eyDq", - "tool_name": "apply_ops" - } - }, - "at_ms": 13620 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 210, - "status": "success", - "tool_call_id": "toolu_01VFMK15TdDdT4zFK3w8bRqE", - "tool_name": "list_slots" - } - }, - "at_ms": 20740 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 206, - "error_code": "conflict", - "status": "error", - "tool_call_id": "toolu_01GZUZLG9RtoeGym25Hy5p5P", - "tool_name": "set_widget" - } - }, - "at_ms": 25170 - }, - { - "kind": "event", - "event": { - "type": "agent_tool_call", - "data": { - "duration_ms": 216, - "status": "error", - "tool_call_id": "toolu_01FCgU1tuBtusNgYiPMRaUQL", - "tool_name": "validate" - } - }, - "at_ms": 29613 - }, - { - "kind": "event", - "event": { - "type": "agent_message_delta", - "data": { - "delta": "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved.\n\nTwo things worth knowing:\n\n- The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider.\n- The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?" - } - }, - "at_ms": 38675 - }, - { - "kind": "event", - "event": { - "type": "agent_message_done", - "data": { - "usage": { - "cache_creation_input_tokens": 3425, - "cache_read_input_tokens": 313028, - "input_tokens": 16, - "output_tokens": 1658, - "total_tokens": 1674 - } + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 216, + "status": "error", + "tool_call_id": "toolu_01FCgU1tuBtusNgYiPMRaUQL", + "tool_name": "validate" + } + }, + "at_ms": 29613 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved.\n\nTwo things worth knowing:\n\n- The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider.\n- The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?" + } + }, + "at_ms": 38675 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 3425, + "cache_read_input_tokens": 313028, + "input_tokens": 16, + "output_tokens": 1658, + "total_tokens": 1674 + } + } + }, + "at_ms": 38677 } - }, - "at_ms": 38677 + ] } ] } diff --git a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts index be045eae74ac..060ac4db5647 100644 --- a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts +++ b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts @@ -19,7 +19,7 @@ if (hasAgentConversation('agent-rec-three-sequential-adds')) }) => { test.setTimeout(60_000) const { conversation } = agentConversation - const opsEntries = conversation.response + const opsEntries = conversation.turns[0].response .map((entry, index) => (entry.kind === 'graph_ops' ? index : -1)) .filter((index) => index >= 0) expect(opsEntries.length).toBeGreaterThanOrEqual(3) @@ -28,7 +28,7 @@ if (hasAgentConversation('agent-rec-three-sequential-adds')) .getByLabel('text', { exact: true }) await agentConversation.sendPrompt() - await agentConversation.replayResponse(async (index) => { + await agentConversation.replayResponse(0, async (index) => { if (index !== opsEntries[0]) return await promptText.fill(HUMAN_TEXT) await expect diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts index dbbd588a9dcc..562291c38d1b 100644 --- a/browser_tests/tests/agent/agentConversationReplay.spec.ts +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -15,9 +15,7 @@ test.describe('Agent conversation replay', { tag: '@cloud' }, () => { test.setTimeout(60_000) const { panel } = agentConversation - await agentConversation.sendPrompt() - await agentConversation.replayResponse() - await agentConversation.waitForTurnComplete() + await agentConversation.runTurns() const groups = agentConversation.toolCallGroups() const groupButtons = panel.getByRole('button', { diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts index fc6e180f6c0f..8a6494b84588 100644 --- a/browser_tests/tests/agent/agentConversationTiming.spec.ts +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -15,7 +15,7 @@ if (hasAgentConversation('agent-rec-set-widget-existing')) agentConversation }) => { test.setTimeout(120_000) - const offsets = agentConversation.conversation.response.flatMap( + const offsets = agentConversation.conversation.turns[0].response.flatMap( (entry) => (entry.at_ms === undefined ? [] : [entry.at_ms]) ) const span = offsets.length ? Math.max(...offsets) : undefined diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts index cd917e65a830..53bd09f73eb4 100644 --- a/scripts/agentConversationCapture.test.ts +++ b/scripts/agentConversationCapture.test.ts @@ -3,25 +3,8 @@ import { describe, expect, it } from 'vitest' import { zAgentConversation } from '../browser_tests/fixtures/data/agent/agentConversation' import { exportAgentConversation } from './agentConversationCapture' -const capture = { - schema_version: 'agent-backend-capture.v1', - source: { - repo: 'Comfy-Org/evals', - suite: 'agent', - case_id: 'recorded-case' - }, - capture: { - backend: 'Comfy-Org/cloud', - thread_id: 'thread-1', - message_id: 'message-1', - exported_at: '2026-09-02T16:00:00.000Z' - }, - workflow: { - id: '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01', - name: 'Captured workflow', - catalog: { types: {} }, - seed: { nodes: [], links: [] } - }, +const turn = { + message_id: 'message-1', request: { content: 'Add a node' }, frames: [ { @@ -61,6 +44,32 @@ const capture = { ] } as const +const capture = { + schema_version: 'agent-backend-capture.v2', + source: { + repo: 'Comfy-Org/evals', + suite: 'agent', + case_id: 'recorded-case' + }, + capture: { + backend: 'Comfy-Org/cloud', + thread_id: 'thread-1', + exported_at: '2026-09-02T16:00:00.000Z' + }, + workflow: { + id: '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01', + name: 'Captured workflow', + catalog: { types: {} }, + seed: { nodes: [], links: [] } + }, + turns: [turn] +} as const + +const withTurn = (patch: Partial) => ({ + ...capture, + turns: [{ ...turn, ...patch }] +}) + describe('exportAgentConversation', () => { it('interleaves only durably applied backend ops before the terminal frame', () => { const conversation = exportAgentConversation(capture) @@ -69,7 +78,7 @@ describe('exportAgentConversation', () => { response_side: 'recorded', capture: capture.capture }) - expect(conversation.response).toEqual([ + expect(conversation.turns[0].response).toEqual([ { kind: 'event', event: { @@ -100,85 +109,113 @@ describe('exportAgentConversation', () => { }) it('emits each entry at its offset from the first recorded frame', () => { - const [running, success] = capture.frames - const exported = exportAgentConversation({ - ...capture, - frames: [ - { ...running, at_ms: 1_700_000_001_000 }, - { ...success, at_ms: 1_700_000_001_750 } - ] - }) - expect(exported.response.map((entry) => entry.at_ms)).toEqual([0, 750, 750]) + const [running, success] = turn.frames + const exported = exportAgentConversation( + withTurn({ + frames: [ + { ...running, at_ms: 1_700_000_001_000 }, + { ...success, at_ms: 1_700_000_001_750 } + ] + }) + ) + const [{ response }] = exported.turns + expect(response.map((entry) => entry.at_ms)).toEqual([0, 750, 750]) expect( - exported.response.every( + response.every( (entry) => entry.kind !== 'event' || !('at_ms' in entry.event) ) ).toBe(true) }) it('leaves the offset out when the frames carry no receipt time', () => { - const exported = exportAgentConversation(capture) - expect(exported.response.every((entry) => entry.at_ms === undefined)).toBe( - true - ) + const [{ response }] = exportAgentConversation(capture).turns + expect(response.every((entry) => entry.at_ms === undefined)).toBe(true) }) it('refuses a tool-call frame with a missing status', () => { - const [running, success] = capture.frames + const [running, success] = turn.frames const { status: _status, ...withoutStatus } = success.data expect(() => - exportAgentConversation({ - ...capture, - frames: [running, { ...success, data: withoutStatus }] - }) + exportAgentConversation( + withTurn({ frames: [running, { ...success, data: withoutStatus }] }) + ) ).toThrow(/status undefined/) }) it('refuses a tool-call frame with an unknown status', () => { - const [running, success] = capture.frames + const [running, success] = turn.frames expect(() => - exportAgentConversation({ - ...capture, - frames: [ - running, - { ...success, data: { ...success.data, status: 'done' } } - ] - }) + exportAgentConversation( + withTurn({ + frames: [ + running, + { ...success, data: { ...success.data, status: 'done' } } + ] + }) + ) ).toThrow(/status "done"/) }) it('refuses an accepted op missing from the recorded result', () => { expect(() => - exportAgentConversation({ - ...capture, - tool_calls: [ - { - ...capture.tool_calls[0], - applied_op_ids: ['op-not-in-result'] - } - ] - }) + exportAgentConversation( + withTurn({ + tool_calls: [ + { ...turn.tool_calls[0], applied_op_ids: ['op-not-in-result'] } + ] + }) + ) ).toThrow('accepted op op-not-in-result') }) it('refuses to silently omit a mutating backend call', () => { expect(() => - exportAgentConversation({ ...capture, frames: [capture.frames[0]] }) + exportAgentConversation(withTurn({ frames: [turn.frames[0]] })) ).toThrow('no terminal websocket frame for tool call(s): tool-1') }) it('refuses a websocket frame from another turn', () => { expect(() => - exportAgentConversation({ - ...capture, - frames: [ - { - ...capture.frames[0], - data: { ...capture.frames[0].data, message_id: 'another-message' } - } - ] - }) - ).toThrow('does not belong to capture thread-1/message-1') + exportAgentConversation( + withTurn({ + frames: [ + { + ...turn.frames[0], + data: { ...turn.frames[0].data, message_id: 'another-message' } + } + ] + }) + ) + ).toThrow('does not belong to turn thread-1/message-1') + }) + + it('exports every turn of the thread in order with its own frames', () => { + const second = { + ...turn, + message_id: 'message-2', + request: { content: 'Connect it' }, + frames: turn.frames.map((frame) => ({ + ...frame, + data: { ...frame.data, message_id: 'message-2' } + })) + } + const exported = exportAgentConversation({ + ...capture, + turns: [turn, second] + }) + expect( + exported.turns.map((exportedTurn) => exportedTurn.message_id) + ).toEqual(['message-1', 'message-2']) + expect(exported.turns[1].request).toEqual({ content: 'Connect it' }) + expect(exported.turns[1].response).toHaveLength(3) + }) + + it('refuses a recorded turn without the message id it came from', () => { + const conversation = exportAgentConversation(capture) + const { message_id: _messageId, ...anonymous } = conversation.turns[0] + expect(() => + zAgentConversation.parse({ ...conversation, turns: [anonymous] }) + ).toThrow('recorded turns carry the message id') }) it('refuses a recorded label without backend provenance', () => { diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts index 567e36f8c543..5ccf7d86e904 100644 --- a/scripts/agentConversationCapture.ts +++ b/scripts/agentConversationCapture.ts @@ -28,8 +28,15 @@ const zToolCall = z.object({ applied_op_ids: z.array(z.string().min(1)) }) +const zBackendTurn = z.object({ + message_id: z.string().min(1), + request: zAgentConversationRequest, + frames: z.array(zRecordedWsEvent).min(1), + tool_calls: z.array(zToolCall) +}) + const zBackendCapture = z.object({ - schema_version: z.literal('agent-backend-capture.v1'), + schema_version: z.literal('agent-backend-capture.v2'), source: z.object({ repo: z.string(), suite: z.string(), @@ -39,13 +46,10 @@ const zBackendCapture = z.object({ capture: z.object({ backend: z.literal('Comfy-Org/cloud'), thread_id: z.string().min(1), - message_id: z.string().min(1), exported_at: z.string().datetime() }), workflow: zAgentConversationWorkflow, - request: zAgentConversationRequest, - frames: z.array(zRecordedWsEvent).min(1), - tool_calls: z.array(zToolCall) + turns: z.array(zBackendTurn).min(1) }) export type AgentBackendCapture = z.input @@ -81,29 +85,28 @@ function operationsFromResult(toolCall: z.infer) { } // Ops are inserted before their terminal tool-call frame and must exist in the accepted-op rows; nothing is authored here. -export function exportAgentConversation(input: unknown) { - const capture = zBackendCapture.parse(input) +function exportTurn(turn: z.infer, threadId: string) { const toolCalls = new Map( - capture.tool_calls.map((toolCall) => [toolCall.tool_call_id, toolCall]) + turn.tool_calls.map((toolCall) => [toolCall.tool_call_id, toolCall]) ) const emittedToolCalls = new Set() const response: Array< | { kind: 'event'; event: z.infer; at_ms?: number } | { kind: 'graph_ops'; ops: Array>; at_ms?: number } > = [] - const firstAt = capture.frames[0].at_ms + const firstAt = turn.frames[0].at_ms const relativeAt = (frame: { at_ms?: number }) => firstAt === undefined || frame.at_ms === undefined ? undefined : frame.at_ms - firstAt - for (const frame of capture.frames) { + for (const frame of turn.frames) { if ( - frame.data.thread_id !== capture.capture.thread_id || - frame.data.message_id !== capture.capture.message_id + frame.data.thread_id !== threadId || + frame.data.message_id !== turn.message_id ) { throw new Error( - `recorded ${frame.type} frame does not belong to capture ${capture.capture.thread_id}/${capture.capture.message_id}` + `recorded ${frame.type} frame does not belong to turn ${threadId}/${turn.message_id}` ) } const at_ms = relativeAt(frame) @@ -137,7 +140,7 @@ export function exportAgentConversation(input: unknown) { response.push({ kind: 'event', event, at_ms }) } - const omitted = capture.tool_calls.filter( + const omitted = turn.tool_calls.filter( (toolCall) => toolCall.applied_op_ids.length > 0 && !emittedToolCalls.has(toolCall.tool_call_id) @@ -147,17 +150,22 @@ export function exportAgentConversation(input: unknown) { `recorded response has no terminal websocket frame for tool call(s): ${omitted.map((call) => call.tool_call_id).join(', ')}` ) } + return { message_id: turn.message_id, request: turn.request, response } +} +export function exportAgentConversation(input: unknown) { + const capture = zBackendCapture.parse(input) return zAgentConversation.parse({ - schema_version: 'agent-conversation.v1', + schema_version: 'agent-conversation.v2', source: { ...capture.source, response_side: 'recorded', capture: capture.capture }, workflow: capture.workflow, - request: capture.request, - response + turns: capture.turns.map((turn) => + exportTurn(turn, capture.capture.thread_id) + ) }) } From 928221c87861e5afc006d9ad8b7e9f3817206236 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 09:05:31 -0700 Subject: [PATCH 47/49] test(agent): record every turn of a thread in one command The recorder took a single prompt and emitted a one-turn capture. It now takes --prompt once per turn, opens the thread on the first and posts the rest to it the way the panel does, buckets frames by the message id of the turn that owns them, reads one audit row set per turn, and emits the turns array agent-backend-capture.v2 expects. The gates that belong to a turn run per turn and name it; the draft check runs once at the end over every applied op of the thread, because only the final draft can witness them, and the active tab frame is required on the opening turn alone since the replay subscribes once. agent-rec-two-turn-dependent-edit is the first case whose second turn depends on its first: turn 1 adds a CLIPTextEncode fed from the checkpoint loader, turn 2 wires that same node into the sampler's negative input. --- browser_tests/fixtures/data/agent/README.md | 52 +- .../agent-rec-two-turn-dependent-edit.json | 449 ++++++++++++++++++ scripts/agentConversationRecord.test.ts | 302 ++++++++++-- scripts/agentConversationRecord.ts | 327 ++++++++----- 4 files changed, 962 insertions(+), 168 deletions(-) create mode 100644 browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md index d2640bc8ab66..1cbf8c9c3088 100644 --- a/browser_tests/fixtures/data/agent/README.md +++ b/browser_tests/fixtures/data/agent/README.md @@ -1,20 +1,22 @@ # Agent conversation capture -Conversation replays must use recorded cloud-agent responses. The exporter -combines two records from the same turn: +Conversation replays must use recorded cloud-agent responses. A fixture holds +one thread, and each of its turns lands on the graph the previous turn left. The +exporter combines two records per turn: 1. Websocket `agent_*` frames, in received order. 2. The cloud backend's durable parent `agent_tool_calls.result` plus the accepted operation IDs from its child audit rows. Do not write `graph_ops` by hand or relabel a synthesized response. The fixture -schema rejects `response_side: recorded` without a cloud thread/message ID and -export timestamp. +schema rejects `response_side: recorded` without a cloud thread ID, a per-turn +message ID and an export timestamp. ## Recording a conversation -One command records a turn against a running agent, applies the gates below, -and writes the fixture through the exporter: +One command records a whole thread against a running agent, applies the gates +below, and writes the fixture through the exporter. Repeat `--prompt` once per +turn; they run in order on one thread: ```bash AGENT_CLOUD_SHA=$(git -C ../cloud rev-parse HEAD) AGENT_MODEL=claude-opus-5 \ @@ -22,17 +24,20 @@ AGENT_M2M_SECRET_FILE=/path/to/m2m.secret \ REC_WORKSPACE_ID=... REC_USER_ID=... \ pnpm exec tsx scripts/agentConversationRecord.ts \ agent-rec- \ - "Switch to the tab 'Text to image', then ." \ browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json \ + --prompt "Switch to the tab 'Text to image', then ." \ + --prompt "" \ --out browser_tests/fixtures/data/agent/conversations/agent-rec-.json ``` -The third positional argument is an existing conversation fixture, used only for -its `workflow` block: the recorder seeds a throwaway turn with that graph and -catalog, then records a second turn against it. The prompt must switch tabs -first, because the replay subscribes to the document only on an +The second positional argument is an existing conversation fixture, used only +for its `workflow` block: the recorder seeds a throwaway turn with that graph +and catalog, then records the prompted turns against it. The first prompt must +switch tabs, because the replay subscribes to the document only on an `agent_active_tab` frame and switching to an already-focused tab publishes -nothing. +nothing; later turns inherit that subscription, so only the first one needs it. +Turn 1 opens the thread, and every later turn posts to it the way the panel +does. Recording runs against the cloud agent in its non-standalone mode, with Postgres, Redis and the doc host beside it (the lean stack): frames come from @@ -59,7 +64,8 @@ Environment, all optional except the two provenance values: | `AGENT_TURN_TIMEOUT` | milliseconds to wait for the turn, default 180000 | Alongside the fixture the command writes a `recordings/` directory holding the -raw frames, the retrieved rows, the intermediate `agent-backend-capture.v1` +raw frames, one retrieved row set per turn, the intermediate +`agent-backend-capture.v2` document and a receipt (turn IDs, parent rows, applied ops, dropped-frame counts, artifact hashes). Those are provenance for the recording, not committed fixtures. Use `--work ` to put them elsewhere. @@ -101,9 +107,12 @@ GROUP BY parent.id ORDER BY parent.started_at, parent.id; ``` -Put those values in an `agent-backend-capture.v1` JSON document together with -the eval source, seed workflow, widget catalog, request, and captured frames. -The thread ID and message ID in `capture` must be the IDs used for both exports. +Put those values in an `agent-backend-capture.v2` JSON document: the eval +source, the seed workflow and widget catalog, a `capture` block naming the +backend, the thread ID and the export timestamp, and a `turns` array carrying, +per turn, its `message_id`, its `request`, its captured `frames` and its +`tool_calls`. Run the query once per turn, binding `$2` to that turn's message +ID. A frame belongs to the turn whose message ID it carries. ## Export @@ -116,8 +125,9 @@ pnpm exec tsx scripts/agentConversationCapture.ts \ The recorder above calls this exporter for you; run it directly when you already have a capture document. -The exporter strips turn identity from frames (the replay mints its own), -inserts each durably accepted op before its terminal tool-call frame, and fails -if any frame belongs to another turn, an accepted op is missing from the -recorded parent result, or a mutating call has no terminal frame. Then run the -conversation replay test for the case. +The exporter emits one conversation turn per captured turn. It strips turn +identity from frames (the replay mints its own), inserts each durably accepted +op before its terminal tool-call frame, and fails if any frame belongs to +another turn, an accepted op is missing from the recorded parent result, or a +mutating call has no terminal frame. Then run the conversation replay test for +the case. diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json new file mode 100644 index 000000000000..7cd2dbd9d7c1 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json @@ -0,0 +1,449 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-two-turn-dependent-edit", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6cafa19d-11f0-45e9-bcf1-e52529eb9ebb; messages 445f7340-ea9c-4456-8ffa-5d37ea6c7b98, 07fc770a-df43-42d8-8f2c-07bf9c2db9f8; workflow d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447 (seeded by throwaway turn af193614-a407-4987-99a5-97a04d52eb35; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 42fbcd91-26cb-415a-b471-8daa44a8bd6c, 55c090e2-11f7-48b7-93dc-5fa9fd73e1ac, abbd53d0-2352-4d5d-bfd0-0bd2601a23ce, b5bde96e-939a-4b5b-992d-5fc9ece40a96, da2d0551-1fcb-435d-9dd1-745a451d75f9, f7084475-8dd0-4789-a14f-0a2cd47379dc; rows agent-rec-two-turn-dependent-edit.m1.rows.1.json, agent-rec-two-turn-dependent-edit.m1.rows.2.json; raw capture sha256 49a64b8ac694b4778b78b3df03a372eaed59e60baca70299ee5cee867b23be43", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "6cafa19d-11f0-45e9-bcf1-e52529eb9ebb", + "exported_at": "2026-09-03T16:03:42.552Z" + } + }, + "workflow": { + "id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "445f7340-ea9c-4456-8ffa-5d37ea6c7b98", + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and give it the text 'blurry, low quality'." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at the graph first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01WdrFCAxyg6dbYLfwVHfPTD", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 752, + "status": "success", + "tool_call_id": "toolu_01E4tYHkXrhtb6ktKRki4z4r", + "tool_name": "print_workflow" + } + }, + "at_ms": 2435 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3337990138003756, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c00397c5ac4a411bafa8c29389fb1515", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3337990138003756, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "720d36f953794c1a8d05c3558c8c7a0f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2557345504960691, + "to_node": 3337990138003756, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "959f6c6e6cbc4dd3a39f784437208f1c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "blurry, low quality", + "widget": "text", + "node_id": 3337990138003756, + "base_version": 2 + } + ], + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 195, + "status": "success", + "tool_call_id": "toolu_013FPeYq4Ya1M9VJKsMPJE4j", + "tool_name": "apply_ops" + } + }, + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 196, + "status": "success", + "tool_call_id": "toolu_01RL1yma6QfBbL3cZQs3qsf6", + "tool_name": "print_workflow" + } + }, + "at_ms": 11731 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now, and the second text box is added with \"blurry, low quality\" and fed from the checkpoint loader.\n\nOne thing worth flagging: it isn't connected to anything downstream yet, so it has no effect on the image. The sampler's negative input is currently empty (and its latent input too) — say the word and I'll wire this new text into the negative slot." + } + }, + "at_ms": 15233 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2013, + "cache_read_input_tokens": 192537, + "input_tokens": 10, + "output_tokens": 822, + "total_tokens": 832 + } + } + }, + "at_ms": 15235 + } + ] + }, + { + "message_id": "07fc770a-df43-42d8-8f2c-07bf9c2db9f8", + "request": { + "content": "Now connect that new CLIPTextEncode node to the KSampler's negative input." + }, + "response": [ + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "0a73a31fd4c54bf8a2bf0746d41710cd", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1795569972720099, + "to_node": 3, + "to_slot": 2, + "from_node": 3337990138003756, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 3 + } + ], + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 298, + "status": "success", + "tool_call_id": "toolu_016EPZQoQXsKsbm5qAzKPHGD", + "tool_name": "connect" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01RZH4jfk5SdpGS1xF2dMfvw", + "tool_name": "print_workflow" + } + }, + "at_ms": 4272 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Connected — the \"blurry, low quality\" text now feeds the negative input.\n\nThe graph still won't run as-is: nothing is feeding the sampler's latent input, so it needs an empty latent (which is also where the image size gets set). Want me to add one?" + } + }, + "at_ms": 8606 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1542, + "cache_read_input_tokens": 115114, + "input_tokens": 6, + "output_tokens": 376, + "total_tokens": 382 + } + } + }, + "at_ms": 8610 + } + ] + } + ] +} diff --git a/scripts/agentConversationRecord.test.ts b/scripts/agentConversationRecord.test.ts index cd2485c409e5..9ffcd6527f26 100644 --- a/scripts/agentConversationRecord.test.ts +++ b/scripts/agentConversationRecord.test.ts @@ -7,6 +7,7 @@ import type { NormalizedRows, RawCapture, RecordedFrame, + RecordedTurn, SeedFixture } from './agentConversationRecord' import { assembleCapture, zRowsDump } from './agentConversationRecord' @@ -79,7 +80,6 @@ const parent = ( const raw = (overrides: Partial = {}): RawCapture => ({ case_id: 'agent-rec-example', attempt: 'a1', - prompt: "Switch to the tab 'Text to image', then add a sampler.", base: 'http://127.0.0.1:8086', frame_source: 'redis SUBSCRIBE channel:ws::u:', channel: 'channel:ws:w-secret:u:u-secret', @@ -93,14 +93,23 @@ const raw = (overrides: Partial = {}): RawCapture => ({ body: { thread_id: 'seed-thread', message_id: SEED_MESSAGE } }, seed_workflow_id: WORKFLOW, - accepted: { - status: 202, - body: { thread_id: THREAD, message_id: MESSAGE, workflow_id: 'blank-wf' } - }, - saw_done: true, + turns: [ + { + prompt: "Switch to the tab 'Text to image', then add a sampler.", + accepted: { + status: 202, + body: { + thread_id: THREAD, + message_id: MESSAGE, + workflow_id: 'blank-wf' + } + }, + saw_done: true + } + ], timed_out: false, frames: frames(), - rows_artifact: 'rows.json', + rows_artifacts: ['rows.json'], retrieval: { kind: 'postgres-json' }, error: null, ...overrides @@ -124,40 +133,49 @@ const rows = (overrides: Partial = {}): NormalizedRows => { } } -const input = (overrides: Partial = {}): AssembleInput => ({ - raw: raw(), - rows: rows(), - seed: { json: seed(), path: 'seed.json', sha256: 'a'.repeat(64) }, - provenance: { - cloudSha: '9dc1da7', - model: 'claude-opus-5', - exportedAt: '2026-09-03T01:00:00.000Z' - }, - rawSha256: 'c'.repeat(64), - rawPath: '/tmp/agent-rec-example.a1.raw.json', - ...overrides -}) +const input = ( + overrides: Partial> & { + rows?: NormalizedRows | NormalizedRows[] + } = {} +): AssembleInput => { + const { rows: given, ...rest } = overrides + return { + raw: raw(), + seed: { json: seed(), path: 'seed.json', sha256: 'a'.repeat(64) }, + provenance: { + cloudSha: '9dc1da7', + model: 'claude-opus-5', + exportedAt: '2026-09-03T01:00:00.000Z' + }, + rawSha256: 'c'.repeat(64), + rawPath: '/tmp/agent-rec-example.a1.raw.json', + ...rest, + rows: given === undefined ? [rows()] : [given].flat() + } +} describe('assembleCapture', () => { it('keeps the turn frames with their receipt times and emits the applied ops', () => { const { capture, receipt } = assembleCapture(input()) expect(capture.workflow.id).toBe(WORKFLOW) + expect(capture.schema_version).toBe('agent-backend-capture.v2') expect(capture.capture).toMatchObject({ backend: 'Comfy-Org/cloud', - thread_id: THREAD, - message_id: MESSAGE + thread_id: THREAD }) - expect(capture.frames.map((frame) => frame.type)).toEqual([ + expect(capture.turns).toHaveLength(1) + expect(capture.turns[0].message_id).toBe(MESSAGE) + expect(capture.turns[0].frames.map((frame) => frame.type)).toEqual([ 'agent_thinking', 'agent_active_tab', 'agent_tool_call', 'agent_message_done' ]) - expect(capture.frames.map((frame) => frame.at_ms)).toEqual([ + expect(capture.turns[0].frames.map((frame) => frame.at_ms)).toEqual([ 1_700_000_000_000, 1_700_000_000_100, 1_700_000_000_200, 1_700_000_000_300 ]) - expect(capture.tool_calls).toEqual([ + expect(capture.turns[0].tool_calls).toEqual([ { tool_call_id: 'tool-1', result: { ok: true, data: { ops: [addNodeOp] } }, @@ -167,12 +185,19 @@ describe('assembleCapture', () => { expect(receipt).toMatchObject({ added_nodes: 1, deleted_nodes: 0, - unexplained_draft_nodes: 0, - parents: 1, - mutating_parents: 1, - child_statuses: { ok: 1 }, - frames_kept: 4 + unexplained_draft_nodes: 0 }) + expect(receipt.turns).toEqual([ + { + message_id: MESSAGE, + frames_kept: 4, + parents: 1, + mutating_parents: 1, + child_statuses: { ok: 1 }, + rows: '/tmp/agent-rec-example.a1.rows.json', + rows_sha256: 'b'.repeat(64) + } + ]) }) it('produces a capture the exporter turns into a recorded conversation', () => { @@ -180,8 +205,12 @@ describe('assembleCapture', () => { const conversation = exportAgentConversation(capture) expect(conversation.source.response_side).toBe('recorded') + expect(conversation.turns).toHaveLength(1) + expect(conversation.turns[0].message_id).toBe(MESSAGE) expect( - conversation.response.filter((entry) => entry.kind === 'graph_ops') + conversation.turns[0].response.filter( + (entry) => entry.kind === 'graph_ops' + ) ).toHaveLength(1) }) @@ -241,8 +270,8 @@ describe('assembleCapture', () => { }) ) - expect(receipt.mutating_parents).toBe(0) - expect(receipt.child_statuses).toEqual({ error: 1 }) + expect(receipt.turns[0].mutating_parents).toBe(0) + expect(receipt.turns[0].child_statuses).toEqual({ error: 1 }) }) it('accepts a read tool that echoes ops without writing child rows', () => { @@ -261,8 +290,8 @@ describe('assembleCapture', () => { }) ) - expect(capture.tool_calls[0].applied_op_ids).toEqual([]) - expect(receipt.mutating_parents).toBe(0) + expect(capture.turns[0].tool_calls[0].applied_op_ids).toEqual([]) + expect(receipt.turns[0].mutating_parents).toBe(0) }) it('drops the ambient heartbeat and a non-replay turn frame into buckets', () => { @@ -294,7 +323,7 @@ describe('assembleCapture', () => { 'type:draft_patch': 1, seed_turn: 1 }) - expect(receipt.frames_kept).toBe(4) + expect(receipt.turns[0].frames_kept).toBe(4) }) it('refuses a mutating call whose mutation never reached the doc host', () => { @@ -523,9 +552,15 @@ describe('assembleCapture', () => { it('refuses a turn the backend never accepted', () => { expect(() => assembleCapture( - input({ raw: raw({ accepted: { status: 500, body: null } }) }) + input({ + raw: raw({ + turns: [ + { ...raw().turns[0], accepted: { status: 500, body: null } } + ] + }) + }) ) - ).toThrow('recorded turn not accepted') + ).toThrow('turn 1 not accepted') }) it('refuses a recording whose frame stream never opened', () => { @@ -536,7 +571,9 @@ describe('assembleCapture', () => { it('refuses a recording with no terminal done frame observed', () => { expect(() => - assembleCapture(input({ raw: raw({ saw_done: false }) })) + assembleCapture( + input({ raw: raw({ turns: [{ ...raw().turns[0], saw_done: false }] }) }) + ) ).toThrow('never arrived') }) @@ -551,6 +588,193 @@ describe('assembleCapture', () => { }) }) +const MESSAGE_2 = 'message-2' + +const connectOp = { op: 'connect', op_id: 'op-2', from: 10, to: 3 } + +const secondTurnFrames = (): RecordedFrame[] => [ + { + type: 'agent_thinking', + data: { thread_id: THREAD, message_id: MESSAGE_2, delta: 'wiring' }, + at_ms: 1_700_000_001_000 + }, + { + type: 'agent_tool_call', + data: { + thread_id: THREAD, + message_id: MESSAGE_2, + tool_call_id: 'tool-2', + tool_name: 'connect', + status: 'success' + }, + at_ms: 1_700_000_001_100 + }, + { + type: 'agent_message_done', + data: { thread_id: THREAD, message_id: MESSAGE_2 }, + at_ms: 1_700_000_001_200 + } +] + +const secondTurn = (overrides: Partial = {}): RecordedTurn => ({ + prompt: 'Now connect it to the sampler.', + accepted: { + status: 202, + body: { thread_id: THREAD, message_id: MESSAGE_2 } + }, + saw_done: true, + ...overrides +}) + +const secondRows = (): NormalizedRows => + rows({ + parents: [ + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + tool_name: 'connect', + result: { ok: true, data: { ops: [connectOp] } }, + children: [{ op_id: 'op-2', status: 'ok' }] + }) + ] + }) + +const twoTurns = (overrides: Partial = {}): AssembleInput => + input({ + raw: raw({ + turns: [raw().turns[0], secondTurn()], + frames: [...frames(), ...secondTurnFrames()] + }), + rows: [rows(), secondRows()], + ...overrides + }) + +describe('assembleCapture across turns', () => { + it('buckets frames by the message id of the turn that owns them', () => { + const { capture, receipt } = assembleCapture(twoTurns()) + + expect(capture.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect( + capture.turns[0].frames.every( + (frame) => frame.data.message_id === MESSAGE + ) + ).toBe(true) + expect(capture.turns[1].frames.map((frame) => frame.type)).toEqual([ + 'agent_thinking', + 'agent_tool_call', + 'agent_message_done' + ]) + expect(capture.turns[1].request.content).toBe( + 'Now connect it to the sampler.' + ) + expect(receipt.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect(receipt.turns.map((turn) => turn.frames_kept)).toEqual([4, 3]) + }) + + it('exports one recorded conversation turn per recorded turn', () => { + const conversation = exportAgentConversation( + assembleCapture(twoTurns()).capture + ) + + expect(conversation.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect( + conversation.turns.map( + (turn) => + turn.response.filter((entry) => entry.kind === 'graph_ops').length + ) + ).toEqual([1, 1]) + }) + + it('requires the active tab frame only on the opening turn', () => { + const { capture } = assembleCapture(twoTurns()) + const tabFrames = (index: number) => + capture.turns[index].frames.filter( + (frame) => frame.type === 'agent_active_tab' + ) + + expect(tabFrames(0)).toHaveLength(1) + expect(tabFrames(1)).toHaveLength(0) + }) + + it('checks the draft once, against every applied op of the thread', () => { + expect(() => + assembleCapture( + twoTurns({ + rows: [ + rows(), + rows({ + parents: [ + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + tool_name: 'delete_node', + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-2', node_id: 4 }] + } + }, + children: [{ op_id: 'op-2', status: 'ok' }] + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] } + }) + ] + }) + ) + ).toThrow('still holds node ids 4') + }) + + it('refuses a turn that landed on another thread', () => { + expect(() => + assembleCapture( + twoTurns({ + raw: raw({ + turns: [ + raw().turns[0], + secondTurn({ + accepted: { + status: 202, + body: { thread_id: 'thread-2', message_id: MESSAGE_2 } + } + }) + ], + frames: [...frames(), ...secondTurnFrames()] + }) + }) + ) + ).toThrow('turn 2 landed on thread thread-2') + }) + + it('refuses a turn whose audit rows were never read', () => { + expect(() => assembleCapture(twoTurns({ rows: [rows()] }))).toThrow( + 'recorded 2 turn(s) but read 1 audit row set(s)' + ) + }) + + it('names the turn whose last kept frame is not the done frame', () => { + expect(() => + assembleCapture( + twoTurns({ + raw: raw({ + turns: [raw().turns[0], secondTurn()], + frames: [...frames(), ...secondTurnFrames().slice(0, 2)] + }) + }) + ) + ).toThrow('last kept frame of turn 2 is agent_tool_call') + }) +}) + describe('zRowsDump', () => { const dump = (result: unknown, overrides: Record = {}) => ({ source: 'postgres', diff --git a/scripts/agentConversationRecord.ts b/scripts/agentConversationRecord.ts index ed481ff584fd..bfa2a0a6716a 100644 --- a/scripts/agentConversationRecord.ts +++ b/scripts/agentConversationRecord.ts @@ -16,7 +16,7 @@ import type { AgentBackendCapture } from './agentConversationCapture' import { exportAgentConversation } from './agentConversationCapture' const USAGE = - 'usage: pnpm exec tsx scripts/agentConversationRecord.ts "" --out [--work ]' + 'usage: pnpm exec tsx scripts/agentConversationRecord.ts --prompt "" [--prompt "" ...] --out [--work ]' const REPLAYED_FRAMES = 'agent_thinking agent_tool_call agent_message_delta agent_message_done agent_active_tab'.split( @@ -132,10 +132,15 @@ export interface TurnAck { body: unknown } +export interface RecordedTurn { + prompt: string + accepted: TurnAck | null + saw_done: boolean +} + export interface RawCapture { case_id: string attempt: string - prompt: string base: string frame_source: string channel: string @@ -146,18 +151,17 @@ export interface RawCapture { stream_closed: boolean seed_turn: TurnAck | null seed_workflow_id: string | null - accepted: TurnAck | null - saw_done: boolean + turns: RecordedTurn[] timed_out: boolean frames: RecordedFrame[] - rows_artifact: string | null + rows_artifacts: string[] retrieval: Record | null error: string | null } export interface AssembleInput { raw: RawCapture - rows: NormalizedRows + rows: NormalizedRows[] seed: { json: SeedFixture; path: string; sha256: string } provenance: { cloudSha: string; model: string; exportedAt: string } rawSha256: string @@ -221,8 +225,14 @@ const sha256OfFile = (path: string): string => sha256(readFileSync(path)) const writeJson = (path: string, value: unknown): void => writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`) -// Everything the recorded turn must clear before its frames mean anything. -function turnIds(raw: RawCapture, seedIds: Set): TurnIds { +const turnLabel = (index: number): string => `turn ${index + 1}` + +// Everything the recording as a whole must clear before a turn means anything. +function checkRecording( + raw: RawCapture, + seedIds: Set, + rowSets: number +): void { if (!/^[A-Za-z0-9_-]+$/.test(raw.attempt)) refuse(`attempt label ${JSON.stringify(raw.attempt)} is not [A-Za-z0-9_-]+`) const driverIds = new Set(raw.seed_node_ids.map(String)) @@ -230,25 +240,35 @@ function turnIds(raw: RawCapture, seedIds: Set): TurnIds { refuse( `the driver seeded ${list(driverIds)} but the seed fixture given here has ${list(seedIds)}` ) - if (raw.accepted?.status !== 202) - refuse(`recorded turn not accepted: ${JSON.stringify(raw.accepted)}`) - const ack = zAck.safeParse(raw.accepted.body) - if (!ack.success) - refuse(`ack without ids: ${JSON.stringify(raw.accepted.body ?? {})}`) if (!raw.saw_stream) refuse('frame stream never opened') - if (!raw.saw_done) + if (raw.turns.length === 0) refuse('no turns recorded') + if (raw.turns.length !== rowSets) refuse( - `agent_message_done never arrived (timed_out=${raw.timed_out}, error=${raw.error})` + `recorded ${raw.turns.length} turn(s) but read ${rowSets} audit row set(s)` + ) +} + +function turnIds(turn: RecordedTurn, label: string, raw: RawCapture): TurnIds { + if (turn.accepted?.status !== 202) + refuse(`${label} not accepted: ${JSON.stringify(turn.accepted)}`) + const ack = zAck.safeParse(turn.accepted.body) + if (!ack.success) + refuse( + `${label} ack without ids: ${JSON.stringify(turn.accepted.body ?? {})}` + ) + if (!turn.saw_done) + refuse( + `${label}: agent_message_done never arrived (timed_out=${raw.timed_out}, error=${raw.error})` ) return { threadId: ack.data.thread_id, messageId: ack.data.message_id } } function keepTurnFrames( frames: RecordedFrame[], - ids: TurnIds, + ids: TurnIds[], seedTurn: TurnIds | null -): { kept: RecordedFrame[]; dropped: Record } { - const kept: RecordedFrame[] = [] +): { kept: RecordedFrame[][]; dropped: Record } { + const kept: RecordedFrame[][] = ids.map(() => []) const dropped: Record = {} const drop = (bucket: string): void => { dropped[bucket] = (dropped[bucket] ?? 0) + 1 @@ -256,13 +276,13 @@ function keepTurnFrames( for (const frame of frames) { const { type, data } = frame + const turn = ids.findIndex( + (id) => data.thread_id === id.threadId && data.message_id === id.messageId + ) // Unparseable payloads belong to no turn; the heartbeat carries no ids. if (type === '__raw__' || type === 'draft_version') drop(type === '__raw__' ? 'unparseable' : 'type:draft_version') - else if ( - data.thread_id !== ids.threadId || - data.message_id !== ids.messageId - ) + else if (turn === -1) drop( seedTurn !== null && data.thread_id === seedTurn.threadId && @@ -270,7 +290,7 @@ function keepTurnFrames( ? 'seed_turn' : 'foreign' ) - else if (REPLAYED_FRAMES.includes(type)) kept.push(frame) + else if (REPLAYED_FRAMES.includes(type)) kept[turn].push(frame) else if (type.startsWith('agent_')) refuse(`frame type ${type} is outside the replay union`) else drop(`type:${type}`) @@ -278,9 +298,13 @@ function keepTurnFrames( if (dropped.unparseable) refuse(`${dropped.unparseable} unparseable socket payload(s) recorded`) - const last = kept.at(-1) - if (last?.type !== 'agent_message_done') - refuse(`last kept frame is ${last?.type}, not agent_message_done`) + for (const [index, turn] of kept.entries()) { + const last = turn.at(-1) + if (last?.type !== 'agent_message_done') + refuse( + `last kept frame of ${turnLabel(index)} is ${last?.type}, not agent_message_done` + ) + } return { kept, dropped } } @@ -368,7 +392,7 @@ function parentToolCall( row: ParentRow, workflowId: string ): { - toolCall: AgentBackendCapture['tool_calls'][number] + toolCall: AgentBackendCapture['turns'][number]['tool_calls'][number] appliedOps: Array> } { const applied = row.children.flatMap((child) => @@ -402,7 +426,11 @@ function parentToolCall( } // The frames and the audit rows must describe the same turn's tool calls. -function checkTurnAgreement(kept: RecordedFrame[], rows: ParentRow[]): void { +function checkTurnAgreement( + kept: RecordedFrame[], + rows: ParentRow[], + label: string +): void { const frameCalls = new Set( kept .filter( @@ -416,7 +444,7 @@ function checkTurnAgreement(kept: RecordedFrame[], rows: ParentRow[]): void { const rowCalls = new Set(rows.map((row) => row.tool_call_id)) if (!sameSet(frameCalls, rowCalls)) refuse( - `frames ${list(frameCalls)} and audit parent rows ${list(rowCalls)} disagree; the rows are not this turn` + `${label}: frames ${list(frameCalls)} and audit parent rows ${list(rowCalls)} disagree; the rows are not this turn` ) } @@ -482,19 +510,18 @@ function checkAddedClasses( function buildCapture(options: { input: AssembleInput - ids: TurnIds + threadId: string workflowId: string seedMessageId: string | null - kept: RecordedFrame[] - toolCalls: AgentBackendCapture['tool_calls'] + turns: AgentBackendCapture['turns'] }): AgentBackendCapture { - const { input, ids, workflowId, kept, toolCalls } = options + const { input, threadId, workflowId, turns } = options const { raw, rows, provenance } = input const { workflow } = input.seed.json - const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${ids.threadId}; message ${ids.messageId}; workflow ${workflowId} (seeded by throwaway turn ${options.seedMessageId}; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(rows.parents.map((row) => row.id))}; rows ${basename(rows.path)}; raw capture sha256 ${input.rawSha256}` + const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${threadId}; messages ${turns.map((turn) => turn.message_id).join(', ')}; workflow ${workflowId} (seeded by throwaway turn ${options.seedMessageId}; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(rows.flatMap((set) => set.parents.map((row) => row.id)))}; rows ${rows.map((set) => basename(set.path)).join(', ')}; raw capture sha256 ${input.rawSha256}` return { - schema_version: 'agent-backend-capture.v1', + schema_version: 'agent-backend-capture.v2', source: { repo: 'Comfy-Org/ComfyUI_frontend', suite: 'agent', @@ -503,8 +530,7 @@ function buildCapture(options: { }, capture: { backend: 'Comfy-Org/cloud', - thread_id: ids.threadId, - message_id: ids.messageId, + thread_id: threadId, exported_at: provenance.exportedAt }, workflow: { @@ -513,21 +539,29 @@ function buildCapture(options: { catalog: workflow.catalog, seed: workflow.seed }, - request: { content: raw.prompt }, - frames: kept, - tool_calls: toolCalls + turns } } +interface TurnReceipt { + message_id: string + frames_kept: number + parents: number + mutating_parents: number + child_statuses: Record + rows: string + rows_sha256: string +} + function buildReceipt(options: { input: AssembleInput - ids: TurnIds + threadId: string workflowId: string - frames: { kept: number; dropped: Record } - parents: { count: number; mutating: number; children: Record } + turns: TurnReceipt[] + framesDropped: Record draft: DraftCounts }) { - const { input, ids, workflowId, frames, parents } = options + const { input, threadId, workflowId, turns } = options const { raw, rows, seed, provenance } = input return { attempt: raw.attempt, @@ -537,17 +571,11 @@ function buildReceipt(options: { raw_sha256: input.rawSha256, seed: seed.path, seed_sha256: seed.sha256, - rows: rows.path, - rows_sha256: rows.sha256, - retrieval: rows.retrieval, - thread_id: ids.threadId, - message_id: ids.messageId, + retrieval: rows[0].retrieval, + thread_id: threadId, workflow_id: workflowId, - frames_kept: frames.kept, - frames_dropped: frames.dropped, - parents: parents.count, - mutating_parents: parents.mutating, - child_statuses: parents.children, + turns, + frames_dropped: options.framesDropped, ...options.draft, cloud_sha: provenance.cloudSha, model: provenance.model @@ -561,44 +589,94 @@ function childStatuses(parents: ParentRow[]): Record { return tally } +// One recorded turn: its own frames, its own rows, and the gates binding them. +function assembleTurn( + turn: RecordedTurn, + ids: TurnIds, + frames: RecordedFrame[], + rows: NormalizedRows, + workflowId: string, + label: string +): { + capture: AgentBackendCapture['turns'][number] + receipt: TurnReceipt + appliedOps: Array> +} { + const calls = rows.parents.map((row) => parentToolCall(row, workflowId)) + checkTurnAgreement(frames, rows.parents, label) + return { + capture: { + message_id: ids.messageId, + request: { content: turn.prompt }, + frames, + tool_calls: calls.map((call) => call.toolCall) + }, + receipt: { + message_id: ids.messageId, + frames_kept: frames.length, + parents: rows.parents.length, + mutating_parents: calls.filter((call) => call.appliedOps.length > 0) + .length, + child_statuses: childStatuses(rows.parents), + rows: rows.path, + rows_sha256: rows.sha256 + }, + appliedOps: calls.flatMap((call) => call.appliedOps) + } +} + export function assembleCapture(input: AssembleInput) { const { raw, rows } = input const { workflow } = input.seed.json const seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) + checkRecording(raw, seedIds, rows.length) - const ids = turnIds(raw, seedIds) const seedAck = zAck.safeParse(raw.seed_turn?.body) const seedTurn = seedAck.success ? { threadId: seedAck.data.thread_id, messageId: seedAck.data.message_id } : null + const ids = raw.turns.map((turn, index) => + turnIds(turn, turnLabel(index), raw) + ) + const { threadId } = ids[0] + const strayed = ids.findIndex((id) => id.threadId !== threadId) + if (strayed > 0) + refuse( + `${turnLabel(strayed)} landed on thread ${ids[strayed].threadId}, not ${threadId}` + ) + const { kept, dropped } = keepTurnFrames(raw.frames, ids, seedTurn) - const workflowId = activeWorkflowId(kept, raw.seed_workflow_id) + // The replay subscribes once, so only the opening turn must switch tabs. + const workflowId = activeWorkflowId(kept[0], raw.seed_workflow_id) + const turns = raw.turns.map((turn, index) => + assembleTurn( + turn, + ids[index], + kept[index], + rows[index], + workflowId, + turnLabel(index) + ) + ) - const calls = rows.parents.map((row) => parentToolCall(row, workflowId)) - checkTurnAgreement(kept, rows.parents) - const appliedOps = calls.flatMap((call) => call.appliedOps) - const draft = checkDraft(rows.draft, seedIds, appliedOps, workflowId) + const appliedOps = turns.flatMap((turn) => turn.appliedOps) + const draft = checkDraft(rows.at(-1)!.draft, seedIds, appliedOps, workflowId) checkAddedClasses(appliedOps, workflow.catalog) return { capture: buildCapture({ input, - ids, + threadId, workflowId, seedMessageId: seedTurn?.messageId ?? null, - kept, - toolCalls: calls.map((call) => call.toolCall) + turns: turns.map((turn) => turn.capture) }), receipt: buildReceipt({ input, - ids, + threadId, workflowId, - frames: { kept: kept.length, dropped }, - parents: { - count: rows.parents.length, - mutating: calls.filter((call) => call.appliedOps.length > 0).length, - children: childStatuses(rows.parents) - }, + turns: turns.map((turn) => turn.receipt), + framesDropped: dropped, draft }) } @@ -675,19 +753,23 @@ async function recordTurns( redisExec: string[] timeoutMs: number seed: WorkflowJSON + prompts: string[] } ): Promise { const stop = openStream(raw, options.redisExec, (frame) => raw.frames.push({ ...frame, at_ms: Date.now() }) ) - const postTurn = async (turn: unknown): Promise => { - const response = await fetch(`${raw.base}/agent/threads/new/messages`, { - method: 'POST', - headers: options.headers, - body: JSON.stringify(turn), - signal: AbortSignal.timeout(30_000) - }) + const postTurn = async (thread: string, turn: unknown): Promise => { + const response = await fetch( + `${raw.base}/agent/threads/${thread}/messages`, + { + method: 'POST', + headers: options.headers, + body: JSON.stringify(turn), + signal: AbortSignal.timeout(30_000) + } + ) const payload = await response.text() const body = zJsonObject.safeParse(safeJson(payload)) if (response.status === 202 && !body.success) @@ -732,7 +814,7 @@ async function recordTurns( refuse(`frame source did not open: ${raw.frame_source}`) // Turn A mints and seeds the workflow; it is thrown away, not recorded. - const seedTurn = await postTurn({ + const seedTurn = await postTurn('new', { content: 'Hello. Please do nothing to the workflow yet; just acknowledge.', draft: { content: options.seed } @@ -744,24 +826,37 @@ async function recordTurns( raw.seed_workflow_id = seedAck.data.workflow_id await waitDone(seedAck.data.message_id, 'seed turn') - // Turn B opens on a fresh workflow so switch_tab publishes agent_active_tab. - const recorded = await postTurn({ - content: raw.prompt, - open_tabs: [ - { workflow_id: seedAck.data.workflow_id, name: raw.seed_name } - ] - }) - raw.accepted = recorded - const ack = zAck.safeParse(recorded.body) - if (recorded.status !== 202 || !ack.success) - refuse(`recorded turn not accepted: ${JSON.stringify(recorded)}`) - await waitDone(ack.data.message_id, 'recorded turn') - raw.saw_done = true + // Turn 1 opens on a fresh workflow so switch_tab publishes + // agent_active_tab; the rest continue that thread the way the panel does. + let thread = 'new' + let opening: string | null = null + for (const [index, prompt] of options.prompts.entries()) { + const posted = await postTurn(thread, { + content: prompt, + open_tabs: [ + { workflow_id: seedAck.data.workflow_id, name: raw.seed_name } + ] + }) + const turn: RecordedTurn = { + prompt, + accepted: posted, + saw_done: false + } + raw.turns.push(turn) + const ack = zAck.safeParse(posted.body) + if (posted.status !== 202 || !ack.success) + refuse(`${turnLabel(index)} not accepted: ${JSON.stringify(posted)}`) + thread = ack.data.thread_id + opening ??= ack.data.message_id + await waitDone(ack.data.message_id, turnLabel(index)) + turn.saw_done = true + } + if ( !raw.frames.some( (frame) => frame.type === 'agent_active_tab' && - frame.data.message_id === ack.data.message_id && + frame.data.message_id === opening && frame.data.workflow_id === seedAck.data.workflow_id ) ) @@ -776,15 +871,19 @@ async function recordTurns( async function main(argv: string[]): Promise { const positional: string[] = [] const flags: Record = {} + const prompts: string[] = [] for (let index = 0; index < argv.length; index += 1) { const arg = argv[index] - if (arg === '--out' || arg === '--work') + if (arg === '--prompt') prompts.push(argv[(index += 1)] ?? '') + else if (arg === '--out' || arg === '--work') flags[arg] = argv[(index += 1)] ?? '' else positional.push(arg) } - const [caseId, prompt, seedPath] = positional + const [caseId, seedPath] = positional const outPath = flags['--out'] - if (!caseId || !prompt || !seedPath || !outPath) throw new Error(USAGE) + if (!caseId || !seedPath || !outPath || prompts.length === 0) + throw new Error(USAGE) + if (prompts.some((turn) => !turn.trim())) throw new Error(USAGE) const env = parseOrRefuse( zEnv, @@ -816,7 +915,6 @@ async function main(argv: string[]): Promise { const raw: RawCapture = { case_id: caseId, attempt, - prompt, base: env.AGENT_FULLSTACK_URL.replace(/\/$/, ''), // The note carries this shape; the concrete channel stays in the receipt. frame_source: 'redis SUBSCRIBE channel:ws::u:', @@ -828,11 +926,10 @@ async function main(argv: string[]): Promise { stream_closed: false, seed_turn: null, seed_workflow_id: null, - accepted: null, - saw_done: false, + turns: [], timed_out: false, frames: [], - rows_artifact: null, + rows_artifacts: [], retrieval: null, error: null } @@ -850,17 +947,25 @@ async function main(argv: string[]): Promise { }, redisExec: env.AGENT_REDIS_EXEC.split(' '), timeoutMs: env.AGENT_TURN_TIMEOUT, - seed: seed.workflow.seed + seed: seed.workflow.seed, + prompts }) - const ids = zAck.parse(raw.accepted?.body) - const rows = readRows(env.AGENT_PG_EXEC.split(' '), sidecar('rows.json'), { - threadId: ids.thread_id, - messageId: ids.message_id, - workflowId: String(raw.seed_workflow_id) + // One row set per turn; the last one also carries the final draft. + const rows = raw.turns.map((turn, index) => { + const ids = zAck.parse(turn.accepted?.body) + return readRows( + env.AGENT_PG_EXEC.split(' '), + sidecar(`rows.${index + 1}.json`), + { + threadId: ids.thread_id, + messageId: ids.message_id, + workflowId: String(raw.seed_workflow_id) + } + ) }) - raw.rows_artifact = rows.path - raw.retrieval = rows.retrieval + raw.rows_artifacts = rows.map((set) => set.path) + raw.retrieval = rows[0].retrieval writeJson(rawPath, raw) const { capture, receipt } = assembleCapture({ @@ -878,8 +983,14 @@ async function main(argv: string[]): Promise { writeJson(sidecar('capture.json'), capture) writeJson(outPath, exportAgentConversation(capture)) writeJson(sidecar('receipt.json'), receipt) + const perTurn = receipt.turns + .map( + (turn, index) => + `t${index + 1}[frames=${turn.frames_kept} parents=${turn.parents} mutating=${turn.mutating_parents}]` + ) + .join(' ') process.stdout.write( - `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} frames=${receipt.frames_kept} dropped=${JSON.stringify(receipt.frames_dropped)} parents=${receipt.parents} mutating=${receipt.mutating_parents} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` + `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} turns=${receipt.turns.length} ${perTurn} dropped=${JSON.stringify(receipt.frames_dropped)} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` ) } catch (error) { raw.error = error instanceof Error ? error.message : String(error) From 4828f068af35f0abdd34c884e304862028e1df01 Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 09:10:54 -0700 Subject: [PATCH 48/49] test(agent): type the exporter test capture against the capture input Readonly literals from as const rejected the per-test overrides under a typecheck that includes scripts/. --- scripts/agentConversationCapture.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts index 53bd09f73eb4..366275751427 100644 --- a/scripts/agentConversationCapture.test.ts +++ b/scripts/agentConversationCapture.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from 'vitest' import { zAgentConversation } from '../browser_tests/fixtures/data/agent/agentConversation' +import type { AgentBackendCapture } from './agentConversationCapture' import { exportAgentConversation } from './agentConversationCapture' -const turn = { +const turn: AgentBackendCapture['turns'][number] = { message_id: 'message-1', request: { content: 'Add a node' }, frames: [ @@ -42,7 +43,7 @@ const turn = { } } ] -} as const +} const capture = { schema_version: 'agent-backend-capture.v2', @@ -63,7 +64,7 @@ const capture = { seed: { nodes: [], links: [] } }, turns: [turn] -} as const +} const withTurn = (patch: Partial) => ({ ...capture, From 86d55e7b82af5fd54e0d0c566352e88eed14c54d Mon Sep 17 00:00:00 2001 From: Nathaniel Parson Koroso Date: Thu, 3 Sep 2026 09:20:09 -0700 Subject: [PATCH 49/49] docs(agent): point the recording recipe at the launcher's record mode --- browser_tests/fixtures/data/agent/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md index 1cbf8c9c3088..d2cd229fdbed 100644 --- a/browser_tests/fixtures/data/agent/README.md +++ b/browser_tests/fixtures/data/agent/README.md @@ -39,6 +39,22 @@ nothing; later turns inherit that subscription, so only the first one needs it. Turn 1 opens the thread, and every later turn posts to it the way the panel does. +One command brings that stack up (launcher record mode, #16781): with the +cloud repo's own local stack running (`cloud up`, or `scripts/start-all.sh`, in +the cloud checkout), run + +```bash +pnpm exec tsx scripts/dev-agent-integration.ts --record \ + --catalog browser_tests/fixtures/data/agent/conversations/.json \ + --cloud-repo ../cloud --agent-port 8087 --doc-host-port 8096 +``` + +It asserts Postgres on 54331 and Redis on 6379, starts the doc host the cloud +repo ships, starts the agent non-standalone with `AGENT_CRDT_MODE=on` and +`AGENT_TARGET=local`, seeds the recorder's workspace and user, and prints the +recorder command with the secret by path. The manual recipe below is the +fallback. + Recording runs against the cloud agent in its non-standalone mode, with Postgres, Redis and the doc host beside it (the lean stack): frames come from its Redis channel and rows from Postgres. That is the only path. The same agent