diff --git a/browser_tests/README.md b/browser_tests/README.md index 14080d2a449..ef3cf551277 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -612,6 +612,33 @@ test.afterEach(async ({ comfyPage }) => { `debugShowCanvasOverlay()`, `debugGetCanvasDataURL()` are for local debugging only. Never commit them. +## Replay coverage for agent bug fixes + +When a fix changes how the agent's turns affect the app (graph edits, +CRDT frames, panel state), add a conversation replay case alongside the +fix so the bug stays fixed: + +1. **Record the conversation.** Reproduce the bug's turn with + `scripts/agentConversationRecord.ts` against the non-standalone local + stack (Postgres + doc host), which is the only place the doc host + writes the audit rows a replay asserts. That one command records the + turn and writes the conversation JSON under + `browser_tests/fixtures/data/agent/conversations/` through the + exporter, marked `response_side: 'recorded'`; see + `fixtures/data/agent/README.md` for the stack recipe, the command and + the capture format. Never write `graph_ops` by hand and never relabel + a synthesized response as recorded. +2. **Add the replay case.** Drive the fixture through the conversation + replay fixture (`agentConversationFixture`), asserting the + canvas-observable outcome the bug corrupted (graph end-state or + panel state - not mock call counts, not model text). +3. **Prove it bites.** Before merging, run the case once against the + fix's parent commit (red) and once at the fix (green); paste both + run lines in the PR description. A replay case that never went red + against the bug does not count as regression coverage. + +Name the case after the behavior it protects (`agent-.json`, with the fix PR cited in the spec header and the fixture's `source.note`). The exporter writes the provenance the replay keeps: `source.capture` (thread and message ids) and `source.note` (row ids and the raw capture hash), which is enough to re-export the recording. + ## Test Data & Typed API Mocks Mock data in `fixtures/data/` exports **typed** objects that conform to diff --git a/browser_tests/fixtures/agentConversationFixture.ts b/browser_tests/fixtures/agentConversationFixture.ts new file mode 100644 index 00000000000..a70c3826161 --- /dev/null +++ b/browser_tests/fixtures/agentConversationFixture.ts @@ -0,0 +1,643 @@ +import type { Locator, Page, WebSocketRoute } from '@playwright/test' +import { expect } from '@playwright/test' + +import { applyOps, mint, readGraph } from '@comfyorg/comfy-multi-player' +import type { + GraphSnapshot, + WidgetCatalog, + WorkflowJSON +} from '@comfyorg/comfy-multi-player' +import * as Y from 'yjs' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' +import { mintWireOps } from '@/workbench/extensions/agent/crdt/opEnvelope' +import type { + AgentCancelAccepted, + AgentMessages, + AgentWsEvent +} from '@/workbench/extensions/agent/schemas/agentApiSchema' +import { parseAgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema' + +import { agentTest, bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import type { + AgentConversation, + AgentConversationTurn, + RecordedWsEvent +} from '@e2e/fixtures/data/agent/agentConversation' +import { loadAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +import { compareNodeIds, toNodeId } from '@/types/nodeId' +import { knownTool } from '@/workbench/extensions/agent/services/agent/agentToolGlyph' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +const THREAD_ID = 'e9a2f3d1-7c44-4b2e-9a01-5f6d8c7b3a10' +// One synthetic message id per turn; the recorded ids never reach the page. +const turnId = (turn: number): string => + `0c5b1e77-2d4a-4f9e-8b63-1a2c3d4e5${turn.toString(16).padStart(3, '0')}` +// useAgentSession.ts keeps THREAD_STORAGE_KEY module-private. +const THREAD_STORAGE_KEY = 'Comfy.Agent.ThreadId' +const SOCKET_SID = '7d1f2e3a-4b5c-4d6e-8f90-1a2b3c4d5e6f' +const HOST_ACTOR = 'agent:comfy' +const DOC_PROTOCOL_VERSION = 1 +const PANEL_MOUNT_TIMEOUT = 30_000 +const SUBSCRIBE_TIMEOUT = 15_000 + +const OPEN_AGENT_LABEL = enMessages.agent.askComfyAgent +const SEND_LABEL = enMessages.agent.send +const STOP_LABEL = enMessages.agent.stop + +interface DocFrame { + type: 'doc_subscribed' | 'doc_update' | 'doc_ops_result' + data: Record +} + +function toBase64(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('base64') +} + +function fromBase64(value: string): Uint8Array { + return new Uint8Array(Buffer.from(value, 'base64')) +} + +class HostDoc { + private readonly doc: Y.Doc + private seq = 1 + + constructor( + private readonly workflowId: string, + seed: WorkflowJSON, + private readonly catalog: WidgetCatalog + ) { + this.doc = mint(seed, catalog) + } + + graph(): GraphSnapshot { + return readGraph(this.doc) + } + + subscribed(): DocFrame { + return { + type: 'doc_subscribed', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + ok: true, + seq: this.seq + } + } + } + + catchUp(stateVectorB64: string): DocFrame { + const update = Y.encodeStateAsUpdate(this.doc, fromBase64(stateVectorB64)) + return this.updateFrame(update, HOST_ACTOR, []) + } + + // The real host folds client batches into the same doc, so the expected graph must see them. + applyClient(ops: unknown[]): string[] { + const result = applyOps( + this.doc, + ops as Parameters[1], + this.catalog + ) + this.seq += 1 + return result.outcomes.flatMap((outcome, index) => + outcome.outcome === 'applied' + ? [String((ops[index] as { op_id?: unknown }).op_id ?? index)] + : [] + ) + } + + apply(operations: GraphOperation[]): DocFrame { + const before = Y.encodeStateVector(this.doc) + const ops = mintWireOps(operations, { + actor: HOST_ACTOR, + baseVersion: this.seq + }) + const result = applyOps(this.doc, ops, this.catalog) + const rejected = result.outcomes.filter((o) => o.outcome !== 'applied') + if (rejected.length > 0) + throw new Error( + `conversation graph_ops did not apply: ${JSON.stringify(rejected)}` + ) + this.seq += 1 + return this.updateFrame( + Y.encodeStateAsUpdate(this.doc, before), + HOST_ACTOR, + ops.map((op) => op.op_id) + ) + } + + private updateFrame( + update: Uint8Array, + actor: string, + opIds: string[] + ): DocFrame { + return { + type: 'doc_update', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id: this.workflowId, + seq: this.seq, + update_b64: toBase64(update), + actor, + op_ids: opIds + } + } + } +} + +type NodeBody = { + id: number | string + type: string + title?: string + inputs?: Array<{ name: string }> + outputs?: Array<{ name: string }> +} + +interface RecordedToolCall { + name: string + ok: boolean +} + +interface RecordedWidgetValue { + nodeId: string + widget: string + value: string | number +} + +// The panel's row label: the known-tool table, else the humanized tool name (ToolCallCard.vue). +function toolRowLabel(name: string): string { + const known = knownTool(name)?.labelKey + const label = known + ? enMessages.agent[ + known.replace('agent.', '') as keyof typeof enMessages.agent + ] + : undefined + if (typeof label === 'string') return label + const spaced = name.replaceAll('_', ' ') + return spaced.charAt(0).toUpperCase() + spaced.slice(1) +} + +interface GraphNodeSnapshot { + id: string + title: string + inputs: boolean[] + outputs: boolean[] +} + +// Runs one recorded prompt/response through the real panel over a routed /ws socket. +class AgentConversationHarness { + readonly postedMessages: string[] = [] + // Human-op minting is observable only on the client side of the socket. + readonly clientFrames: { type?: unknown; data?: unknown }[] = [] + readonly panel: Locator + + private readonly host: HostDoc + private socket: WebSocketRoute | null = null + private resolveSubscribed: (() => void) | null = null + private readonly subscribed = new Promise((resolve) => { + this.resolveSubscribed = resolve + }) + + constructor( + private readonly page: Page, + readonly conversation: AgentConversation, + readonly replayTiming: ReplayTiming + ) { + const { workflow } = conversation + this.host = new HostDoc(workflow.id, workflow.seed, workflow.catalog) + this.panel = page.locator('#agent-panel-root') + } + + async boot(agentFlag: boolean): Promise { + await this.mockAgentApi() + // The follower re-drives a pending subscribe only on a status frame, which every real connect sends. + await this.page.routeWebSocket(/\/ws/, (socket) => { + this.socket = socket + socket.onMessage((raw) => this.onClientFrame(raw)) + socket.send( + JSON.stringify({ + type: 'status', + data: { + status: { exec_info: { queue_remaining: 0 } }, + sid: SOCKET_SID + } + }) + ) + }) + await bootAgentApp(this.page, agentFlag, { + // Only the Vue node renderer projects follower edits onto the canvas. + settings: { 'Comfy.VueNodes.Enabled': true } + }) + + await this.page.getByRole('button', { name: OPEN_AGENT_LABEL }).click() + await expect(this.panel).toBeVisible({ timeout: PANEL_MOUNT_TIMEOUT }) + } + + async sendPrompt(turn = 0): Promise { + const { content } = this.conversation.turns[turn].request + const composer = this.panel.getByRole('textbox', { + name: /^Describe ideas/ + }) + await composer.fill(content) + await this.panel.getByRole('button', { name: SEND_LABEL }).click() + await expect.poll(() => this.postedMessages.length).toBeGreaterThan(turn) + expect(this.postedMessages[turn]).toContain(content) + // Replay frames are dropped until the page has applied the ack's thread id. + await expect + .poll(() => + this.page.evaluate( + (key) => localStorage.getItem(key), + THREAD_STORAGE_KEY + ) + ) + .toBe(THREAD_ID) + } + + async replayResponse( + turn = 0, + afterEntry?: (index: number) => Promise + ): Promise { + const startedAt = Date.now() + const entries = this.conversation.turns[turn].response.entries() + for (const [index, entry] of entries) { + // A timer can fire a millisecond early, so wait until the offset has really passed. + while ( + this.replayTiming === 'recorded' && + entry.at_ms !== undefined && + Date.now() - startedAt < entry.at_ms + ) + await new Promise((resolve) => + setTimeout(resolve, entry.at_ms! - (Date.now() - startedAt)) + ) + if (entry.kind === 'event') { + this.send(this.stampTurn(entry.event, turn)) + continue + } + await this.waitForSubscribe() + this.send(this.host.apply(entry.ops)) + await afterEntry?.(index) + } + this.replayElapsedMs = Date.now() - startedAt + } + + replayElapsedMs = 0 + + async runTurns(): Promise { + for (const turn of this.conversation.turns.keys()) { + await this.sendPrompt(turn) + await this.replayResponse(turn) + await this.waitForTurnComplete() + } + } + + private entries(): AgentConversationTurn['response'] { + return this.conversation.turns.flatMap((turn) => turn.response) + } + + async waitForTurnComplete(): Promise { + await expect( + this.panel.getByRole('button', { name: SEND_LABEL }) + ).toBeVisible() + await expect( + this.panel.getByRole('button', { name: STOP_LABEL }) + ).toHaveCount(0) + } + + // Doc-id filter: a stray template node must not pin the template here. + private nodeBodies(): NodeBody[] { + const seed = this.conversation.workflow.seed.nodes as NodeBody[] + const added = this.entries().flatMap((entry) => + entry.kind === 'graph_ops' + ? entry.ops.flatMap((op) => + op.op === 'add_node' ? [op.node as NodeBody] : [] + ) + : [] + ) + return [...seed, ...added] + } + + // The doc keeps no titles or slot names; those come from the node bodies the turn started from or added. + expectedGraph(): GraphNodeSnapshot[] { + const graph = this.host.graph() + const bodies = this.nodeBodies() + const byType = new Map(bodies.map((body) => [body.type, body])) + const byId = new Map(bodies.map((body) => [String(body.id), body])) + const catalog = this.conversation.workflow.catalog.types as Record< + string, + { widget_order: string[] } + > + const links = Object.values(graph.links) as Array< + [unknown, unknown, number, unknown, number, string] + > + return Object.entries(graph.nodes) + .map(([id, node]) => { + const type = String(node.type) + const body = byId.get(id) ?? byType.get(type) + const widgets = new Set(catalog[type]?.widget_order ?? []) + return { + id, + title: byId.get(id)?.title ?? type, + // Widget-backed inputs render as widgets, not slot rows. + inputs: (body?.inputs ?? []) + .map((slot, index) => ({ name: slot.name, index })) + .filter(({ name }) => !widgets.has(name)) + .map(({ index }) => + links.some((link) => String(link[3]) === id && link[4] === index) + ), + outputs: (body?.outputs ?? []).map((_, index) => + links.some((link) => String(link[1]) === id && link[2] === index) + ) + } + }) + .sort((a, b) => compareNodeIds(toNodeId(a.id), toNodeId(b.id))) + } + + // The panel starts a new tool group whenever thinking or text interrupts the calls. + // ToolCallGroup.vue merges consecutive calls of the same tool into one row. + // One row per run of same-tool calls; the card shows ×N for a run longer than one. + toolRowCounts(): Array<{ label: string; times: number; rows: number }> { + const rows = new Map< + string, + { label: string; times: number; rows: number } + >() + for (const group of this.toolCallGroups()) { + const runs: Array<{ name: string; times: number }> = [] + for (const call of group) { + const previous = runs.at(-1) + if (previous?.name === call.name) previous.times += 1 + else runs.push({ name: call.name, times: 1 }) + } + for (const { name, times } of runs) { + const key = `${name}/${times}` + const row = rows.get(key) ?? { + label: toolRowLabel(name), + times, + rows: 0 + } + row.rows += 1 + rows.set(key, row) + } + } + return [...rows.values()] + } + + toolCallGroups(): RecordedToolCall[][] { + const groups: RecordedToolCall[][] = [] + for (const turn of this.conversation.turns) { + let current: RecordedToolCall[] = [] + for (const entry of turn.response) { + if (entry.kind !== 'event') continue + const { type, data } = entry.event + if (type === 'agent_tool_call') { + if (data.status !== 'running') + current.push({ + name: String(data.tool_name), + ok: data.status === 'success' + }) + continue + } + if ( + (type === 'agent_thinking' || type === 'agent_message_delta') && + current.length > 0 + ) { + groups.push(current) + current = [] + } + } + if (current.length > 0) groups.push(current) + } + return groups + } + + // Last write wins per widget; the rendered control shows only the final value. + recordedWidgetValues(): RecordedWidgetValue[] { + const graph = this.host.graph() + const latest = new Map() + for (const entry of this.entries()) { + if (entry.kind !== 'graph_ops') continue + for (const op of entry.ops) { + if (op.op !== 'set_widget') continue + const nodeId = String(op.node_id) + const widget = op.widget + const widgets = graph.nodes[nodeId]?.widgets as + | Record + | undefined + const value = widgets?.[widget] + if (typeof value === 'string' || typeof value === 'number') + latest.set(`${nodeId}/${widget}`, { nodeId, widget, value }) + } + } + return [...latest.values()] + } + + addedNodeIds(): string[] { + const graph = this.host.graph() + const seedIds = new Set( + this.conversation.workflow.seed.nodes.map((node) => String(node.id)) + ) + return this.nodeBodies() + .map((body) => String(body.id)) + .filter((id) => !seedIds.has(id) && id in graph.nodes) + } + + // One update per ops entry plus the subscribe catch-up. + expectedUpdateCount(): number { + return ( + this.entries().filter((entry) => entry.kind === 'graph_ops').length + 1 + ) + } + + hostGraph(): GraphSnapshot { + return this.host.graph() + } + + async graphSnapshot(): Promise { + const docNodeIds = new Set(Object.keys(this.host.graph().nodes)) + const snapshot = await this.page + .locator('[data-node-id]') + .evaluateAll((nodes) => + nodes.map((node) => { + const connected = (selector: string) => + Array.from(node.querySelectorAll(selector)).map((slot) => + slot.classList.contains('lg-slot--connected') + ) + return { + id: node.getAttribute('data-node-id') ?? '', + title: + node + .querySelector('[data-testid="node-title"]') + ?.textContent?.trim() ?? '', + inputs: connected('.lg-slot--input'), + outputs: connected('.lg-slot--output') + } + }) + ) + return snapshot + .filter((node) => docNodeIds.has(node.id)) + .sort((a, b) => compareNodeIds(toNodeId(a.id), toNodeId(b.id))) + } + + private async mockAgentApi(): Promise { + const { page } = this + await page.route('**/api/agent/threads', (route) => + route.fulfill(jsonRoute({ threads: [] })) + ) + await page.route('**/api/agent/threads/*/messages', (route) => { + const request = route.request() + if (request.method() === 'POST') { + this.postedMessages.push(request.postData() ?? '') + return route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify({ + thread_id: THREAD_ID, + message_id: turnId(this.postedMessages.length - 1), + workflow_id: this.conversation.workflow.id + }) + }) + } + const history: AgentMessages = [] + return route.fulfill(jsonRoute(history)) + }) + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => { + const cancelled: AgentCancelAccepted = { status: 'cancelling' } + return route.fulfill(jsonRoute(cancelled)) + }) + await page.route('**/api/workflows**', (route) => + route.fulfill( + jsonRoute({ + data: [], + pagination: { has_more: false, limit: 100, offset: 0, total: 0 } + }) + ) + ) + } + + private stampTurn(event: RecordedWsEvent, turn: number): AgentWsEvent { + const stamped = { + type: event.type, + data: { ...event.data, message_id: turnId(turn), thread_id: THREAD_ID } + } + const parsed = parseAgentWsEvent(stamped) + if (!parsed.success) + throw new Error( + `recorded ${event.type} frame is not a valid agent event: ${parsed.error.message}` + ) + return parsed.data + } + + private send(frame: AgentWsEvent | DocFrame): void { + if (!this.socket) throw new Error('the app has not opened /ws yet') + this.socket.send(JSON.stringify(frame)) + } + + outboundOps(): { op?: unknown }[] { + return this.clientFrames + .filter((frame) => frame.type === 'doc_ops') + .flatMap((frame) => { + const ops = (frame.data as { ops?: unknown } | undefined)?.ops + return Array.isArray(ops) ? (ops as { op?: unknown }[]) : [] + }) + } + + private onClientFrame(raw: string | Buffer): void { + const frame: unknown = JSON.parse(raw.toString()) + if (typeof frame !== 'object' || frame === null) return + const { type, data } = frame as { type?: unknown; data?: unknown } + this.clientFrames.push({ type, data }) + if (type === 'doc_ops' && typeof data === 'object' && data !== null) { + // The client keeps one op batch in flight until the host acks it. + const { workflow_id, ops } = data as { + workflow_id?: unknown + ops?: unknown + } + if (workflow_id === this.conversation.workflow.id && Array.isArray(ops)) { + const applied = this.host.applyClient(ops) + this.send({ + type: 'doc_ops_result', + data: { + v: DOC_PROTOCOL_VERSION, + workflow_id, + ok: true, + applied, + skipped: [] + } + }) + } + return + } + if (type !== 'doc_subscribe' || typeof data !== 'object' || data === null) + return + const { workflow_id, state_vector_b64 } = data as { + workflow_id?: unknown + state_vector_b64?: unknown + } + if ( + workflow_id !== this.conversation.workflow.id || + typeof state_vector_b64 !== 'string' + ) + return + this.send(this.host.subscribed()) + this.send(this.host.catchUp(state_vector_b64)) + this.resolveSubscribed?.() + } + + private async waitForSubscribe(): Promise { + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + 'the follower never subscribed to the conversation workflow; graph_ops need an agent_active_tab (or a bound tab) first' + ) + ), + SUBSCRIBE_TIMEOUT + ) + }) + try { + await Promise.race([this.subscribed, timeout]) + } finally { + clearTimeout(timer) + } + } +} + +export type ReplayTiming = 'immediate' | 'recorded' + +function defaultReplayTiming(): ReplayTiming { + const value = process.env.AGENT_REPLAY_TIMING + if (value === undefined || value === 'immediate') return 'immediate' + if (value === 'recorded') return 'recorded' + throw new Error( + `AGENT_REPLAY_TIMING must be immediate or recorded, got ${value}` + ) +} + +interface ConversationFixtures { + conversationCase: string + // 'recorded' replays the fixture's at_ms gaps; the default follows AGENT_REPLAY_TIMING. + replayTiming: ReplayTiming + agentConversation: AgentConversationHarness +} + +export const agentConversationTest = agentTest.extend({ + conversationCase: ['', { option: true }], + replayTiming: [defaultReplayTiming(), { option: true }], + agentConversation: async ( + { page, agentFlagEnabled, conversationCase, replayTiming }, + use + ) => { + if (conversationCase.length === 0) + throw new Error('test.use({ conversationCase }) names the conversation') + const harness = new AgentConversationHarness( + page, + loadAgentConversation(conversationCase), + replayTiming + ) + await harness.boot(agentFlagEnabled) + await use(harness) + } +}) diff --git a/browser_tests/fixtures/agentPanelFixture.ts b/browser_tests/fixtures/agentPanelFixture.ts index 033c519adb7..4ee0352f537 100644 --- a/browser_tests/fixtures/agentPanelFixture.ts +++ b/browser_tests/fixtures/agentPanelFixture.ts @@ -23,15 +23,21 @@ function agentFeatures(agentFlag: boolean): RemoteConfig { } } +interface BootAgentAppOptions { + /** Extra `/api/settings` entries layered over the panel defaults. */ + settings?: Record +} + async function mockAgentBoot( page: Page, - { agentFlag }: { agentFlag: boolean } + { agentFlag, settings }: { agentFlag: boolean } & BootAgentAppOptions ): Promise { await mockCloudBoot(page, { features: agentFeatures(agentFlag), settings: { 'Comfy.TutorialCompleted': true, - 'Comfy.RightSidePanel.ShowErrorsTab': false + 'Comfy.RightSidePanel.ShowErrorsTab': false, + ...settings } }) await mockBilling(page) @@ -65,14 +71,15 @@ export const agentTest = cloudAppFixture.extend({ export async function bootAgentApp( page: Page, - agentFlag: boolean + agentFlag: boolean, + options: BootAgentAppOptions = {} ): Promise { // The shell's onboarding coach is a modal; pre-seed its dismissal so the // panel chrome is interactable, as the canonical agent suite does. await page.addInitScript(() => { localStorage.setItem('Comfy.AgentPanel.onboarded', 'true') }) - await mockAgentBoot(page, { agentFlag }) + await mockAgentBoot(page, { agentFlag, ...options }) await bootCloud(page) await page.goto(APP_URL) await waitForCloudApp(page) diff --git a/browser_tests/fixtures/data/agent/README.md b/browser_tests/fixtures/data/agent/README.md new file mode 100644 index 00000000000..d2cd229fdbe --- /dev/null +++ b/browser_tests/fixtures/data/agent/README.md @@ -0,0 +1,149 @@ +# Agent conversation capture + +Conversation replays must use recorded cloud-agent responses. A fixture holds +one thread, and each of its turns lands on the graph the previous turn left. The +exporter combines two records per turn: + +1. Websocket `agent_*` frames, in received order. +2. The cloud backend's durable parent `agent_tool_calls.result` plus the + accepted operation IDs from its child audit rows. + +Do not write `graph_ops` by hand or relabel a synthesized response. The fixture +schema rejects `response_side: recorded` without a cloud thread ID, a per-turn +message ID and an export timestamp. + +## Recording a conversation + +One command records a whole thread against a running agent, applies the gates +below, and writes the fixture through the exporter. Repeat `--prompt` once per +turn; they run in order on one thread: + +```bash +AGENT_CLOUD_SHA=$(git -C ../cloud rev-parse HEAD) AGENT_MODEL=claude-opus-5 \ +AGENT_M2M_SECRET_FILE=/path/to/m2m.secret \ +REC_WORKSPACE_ID=... REC_USER_ID=... \ +pnpm exec tsx scripts/agentConversationRecord.ts \ + agent-rec- \ + browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json \ + --prompt "Switch to the tab 'Text to image', then ." \ + --prompt "" \ + --out browser_tests/fixtures/data/agent/conversations/agent-rec-.json +``` + +The second positional argument is an existing conversation fixture, used only +for its `workflow` block: the recorder seeds a throwaway turn with that graph +and catalog, then records the prompted turns against it. The first prompt must +switch tabs, because the replay subscribes to the document only on an +`agent_active_tab` frame and switching to an already-focused tab publishes +nothing; later turns inherit that subscription, so only the first one needs it. +Turn 1 opens the thread, and every later turn posts to it the way the panel +does. + +One command brings that stack up (launcher record mode, #16781): with the +cloud repo's own local stack running (`cloud up`, or `scripts/start-all.sh`, in +the cloud checkout), run + +```bash +pnpm exec tsx scripts/dev-agent-integration.ts --record \ + --catalog browser_tests/fixtures/data/agent/conversations/.json \ + --cloud-repo ../cloud --agent-port 8087 --doc-host-port 8096 +``` + +It asserts Postgres on 54331 and Redis on 6379, starts the doc host the cloud +repo ships, starts the agent non-standalone with `AGENT_CRDT_MODE=on` and +`AGENT_TARGET=local`, seeds the recorder's workspace and user, and prints the +recorder command with the secret by path. The manual recipe below is the +fallback. + +Recording runs against the cloud agent in its non-standalone mode, with +Postgres, Redis and the doc host beside it (the lean stack): frames come from +its Redis channel and rows from Postgres. That is the only path. The same agent +in standalone mode (SQLite, no doc host) never writes the audit rows that carry +the graph operations a replay asserts, so it can only produce text-only or +tool-error turns. + +The recording agent shells out to `comfy-cli` for its read and edit tools inside +a sandbox that overrides `HOME`, so point `COMFY_BIN` at an installation that +works without a home directory. A turn recorded without it produces an answer +reporting tool failures rather than a usable fixture. + +Environment, all optional except the two provenance values: + +| Variable | Purpose | +| ------------------------------------- | ------------------------------------------------------------------ | +| `AGENT_CLOUD_SHA`, `AGENT_MODEL` | recorded into the fixture note; both required | +| `AGENT_FULLSTACK_URL` | agent base URL, default `http://127.0.0.1:8086` | +| `AGENT_M2M_SECRET_FILE` | path to the shared secret, read at runtime and never printed | +| `AGENT_WORKSPACE_ID`, `AGENT_USER_ID` | identity headers; `REC_WORKSPACE_ID` / `REC_USER_ID` also accepted | +| `AGENT_REDIS_EXEC`, `AGENT_PG_EXEC` | commands that reach the stack's Redis and Postgres | +| `AGENT_ATTEMPT` | attempt label in every artifact name; defaults to a UTC stamp | +| `AGENT_TURN_TIMEOUT` | milliseconds to wait for the turn, default 180000 | + +Alongside the fixture the command writes a `recordings/` directory holding the +raw frames, one retrieved row set per turn, the intermediate +`agent-backend-capture.v2` +document and a receipt (turn IDs, parent rows, applied ops, dropped-frame +counts, artifact hashes). Those are provenance for the recording, not committed +fixtures. Use `--work ` to put them elsewhere. + +A refused recording names the gate it failed and appends the reason to +`recordings/.refused.jsonl`. Re-record with a new `AGENT_ATTEMPT` +rather than editing a capture by hand. + +Then run the replay, which picks up every recorded fixture in this directory: + +```bash +PLAYWRIGHT_TEST_URL=http://localhost:5173 DISTRIBUTION=cloud \ + pnpm exec playwright test browser_tests/tests/agent/agentConversationReplay.spec.ts \ + --project=cloud -g '' +``` + +## Capture + +Record the `/ws` frames for one eval turn and remove unrelated frame types. Keep +the original `thread_id`, `message_id`, `tool_call_id`, and ordering. Export the +matching backend rows with this query (bind `$1` to the thread ID and `$2` to +the assistant message ID): + +```sql +SELECT + parent.tool_call_id, + parent.result, + COALESCE( + json_agg(child.op_id ORDER BY child.op_index) + FILTER (WHERE child.status = 'ok' AND child.op_id IS NOT NULL), + '[]'::json + ) AS applied_op_ids +FROM agent_tool_calls AS parent +LEFT JOIN agent_tool_calls AS child ON child.parent_call_id = parent.id +WHERE parent.thread_id = $1 + AND parent.message_id = $2 + AND parent.parent_call_id IS NULL +GROUP BY parent.id +ORDER BY parent.started_at, parent.id; +``` + +Put those values in an `agent-backend-capture.v2` JSON document: the eval +source, the seed workflow and widget catalog, a `capture` block naming the +backend, the thread ID and the export timestamp, and a `turns` array carrying, +per turn, its `message_id`, its `request`, its captured `frames` and its +`tool_calls`. Run the query once per turn, binding `$2` to that turn's message +ID. A frame belongs to the turn whose message ID it carries. + +## Export + +```bash +pnpm exec tsx scripts/agentConversationCapture.ts \ + /path/to/backend-capture.json \ + browser_tests/fixtures/data/agent/conversations/.json +``` + +The recorder above calls this exporter for you; run it directly when you +already have a capture document. + +The exporter emits one conversation turn per captured turn. It strips turn +identity from frames (the replay mints its own), inserts each durably accepted +op before its terminal tool-call frame, and fails if any frame belongs to +another turn, an accepted op is missing from the recorded parent result, or a +mutating call has no terminal frame. Then run the conversation replay test for +the case. diff --git a/browser_tests/fixtures/data/agent/agentConversation.ts b/browser_tests/fixtures/data/agent/agentConversation.ts new file mode 100644 index 00000000000..b81b9ce11e0 --- /dev/null +++ b/browser_tests/fixtures/data/agent/agentConversation.ts @@ -0,0 +1,132 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs' +import { fileURLToPath } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import type { WidgetCatalog, WorkflowJSON } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import type { GraphOperation } from '@/workbench/extensions/agent/crdt/graphOperations' + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export const zRecordedWsEvent = z.object({ + type: z.string(), + data: z.record(z.string(), z.unknown()), + at_ms: z.number().int().nonnegative().optional() +}) +export type RecordedWsEvent = z.infer + +// The applier validates op payloads at replay time; only the vocabulary is pinned here. +const zGraphOperation = z.custom( + (value) => + isRecord(value) && + typeof value.op === 'string' && + (FROZEN_OPS as readonly string[]).includes(value.op) +) + +const zWorkflowJson = z.custom( + (value) => + isRecord(value) && Array.isArray(value.nodes) && Array.isArray(value.links) +) + +const zWidgetCatalog = z.custom( + (value) => isRecord(value) && isRecord(value.types) +) + +export const zAgentConversationWorkflow = z.object({ + id: z.string().uuid(), + name: z.string().min(1), + catalog: zWidgetCatalog, + seed: zWorkflowJson +}) + +export const zAgentConversationRequest = z.object({ + content: z.string().min(1) +}) + +// Offset from the turn's first frame, so replays can reproduce real gaps. +const zAtMs = z.number().int().nonnegative().optional() + +const zResponseEntry = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('event'), event: zRecordedWsEvent, at_ms: zAtMs }), + z.object({ + kind: z.literal('graph_ops'), + ops: z.array(zGraphOperation).min(1), + at_ms: zAtMs + }) +]) + +const zTurn = z.object({ + message_id: z.string().min(1).optional(), + request: zAgentConversationRequest, + response: z.array(zResponseEntry).min(1) +}) +export type AgentConversationTurn = z.infer + +export const zAgentConversation = z + .object({ + schema_version: z.literal('agent-conversation.v2'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + response_side: z.enum(['recorded', 'synthesized']), + note: z.string().optional(), + capture: z + .object({ + backend: z.literal('Comfy-Org/cloud'), + thread_id: z.string().min(1), + exported_at: z.string().datetime() + }) + .optional() + }), + workflow: zAgentConversationWorkflow, + // One thread; each turn lands on the graph the previous turn left. + turns: z.array(zTurn).min(1) + }) + .superRefine((conversation, ctx) => { + if (conversation.source.response_side !== 'recorded') return + if (conversation.source.capture === undefined) + ctx.addIssue({ + code: 'custom', + path: ['source', 'capture'], + message: 'recorded responses require backend capture provenance' + }) + for (const [index, turn] of conversation.turns.entries()) + if (turn.message_id === undefined) + ctx.addIssue({ + code: 'custom', + path: ['turns', index, 'message_id'], + message: 'recorded turns carry the message id they were captured from' + }) + }) +export type AgentConversation = z.infer + +export function listRecordedConversations(): string[] { + const dir = fileURLToPath(new URL('./conversations/', import.meta.url)) + if (!existsSync(dir)) return [] + return readdirSync(dir) + .filter((file) => file.endsWith('.json')) + .map((file) => file.slice(0, -'.json'.length)) + .filter( + (caseId) => + loadAgentConversation(caseId).source.response_side === 'recorded' + ) + .sort() +} + +// Specs pinned to one recording skip on a branch that carries the code but not the data. +export function hasAgentConversation(caseId: string): boolean { + return existsSync( + fileURLToPath(new URL(`./conversations/${caseId}.json`, import.meta.url)) + ) +} + +export function loadAgentConversation(caseId: string): AgentConversation { + const file = fileURLToPath( + new URL(`./conversations/${caseId}.json`, import.meta.url) + ) + return zAgentConversation.parse(JSON.parse(readFileSync(file, 'utf-8'))) +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json new file mode 100644 index 00000000000..55e16cb9a37 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-l4-zimage-string-node-prompt.json @@ -0,0 +1,554 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-l4-zimage-string-node-prompt", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 91a905f3-b619-44e3-a6e5-5a047d8d0066; message 4293701d-cd6f-41cf-b03d-ea73df6af846; workflow 891d1304-09ae-4d51-9bea-aaa4756822d4 (seeded by throwaway turn 62bbe9f8-bcb1-4a84-9275-97384a9482f2; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ea9a0cdc-498e-4c8e-ba09-e487cbf02e12', '3893c0d2-605b-43bc-8d77-ff249b267087', '3ae90aa2-14e3-4001-a3c9-90eadef4ad86', '5656b627-7364-4759-8471-f0dedbf3b125', '7e4bd8e9-223d-4a79-985e-7bd262e8b5b9', 'c7f90263-0a57-4a48-8573-75bdb1746a6e', 'acc3faea-b08b-4e4d-8f4b-cb999a8cc6d5', 'a928d523-16e8-4d26-81ad-1b7cdd884c68', '42abc204-3da4-410c-bf83-1b206d567401']; rows agent-l4-zimage-string-node-prompt.t-a1.rows.json; raw capture sha256 f4a2076ae60f8e5eefc7799fd0a14d58d4c3829796ff89f88a0e4548c952b3fe", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "91a905f3-b619-44e3-a6e5-5a047d8d0066", + "exported_at": "2026-09-03T12:40:57Z" + } + }, + "workflow": { + "id": "891d1304-09ae-4d51-9bea-aaa4756822d4", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "4293701d-cd6f-41cf-b03d-ea73df6af846", + "request": { + "content": "Switch to the tab 'Text to image', then add a text node that feeds the prompt of this workflow, put 'a red bicycle on a pier at dusk' in it, and run." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look at that workflow." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "891d1304-09ae-4d51-9bea-aaa4756822d4" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 6, + "status": "success", + "tool_call_id": "toolu_01KZvQdg1MdhKgbWQsvScSpN", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 774, + "status": "success", + "tool_call_id": "toolu_015MGXQKWWD64ZWnv7jWQNLw", + "tool_name": "print_workflow" + } + }, + "at_ms": 2809 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 308, + "status": "success", + "tool_call_id": "toolu_015ngjjFSSFFFRrsQvJwbnut", + "tool_name": "find_nodes" + } + }, + "at_ms": 7587 + }, + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "The prompt node is easy — but this graph is also missing a couple of connections it needs to run, so I'll complete those too." + } + }, + "at_ms": 13907 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [395, 280], + "node": { + "id": 3876406316923056, + "pos": [395, 280], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b41f1bc5b13f44efb10bb51847859813", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876406316923056, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "90d12d67f1a54617af277c333f9880be", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a red bicycle on a pier at dusk", + "widget": "value", + "node_id": 3876406316923056, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c0b8ec72e8d64de7bcf9b45125f6c207", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1887935066439213, + "to_node": 6, + "to_slot": 1, + "from_node": 3876406316923056, + "from_slot": 0, + "link_type": "STRING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3880387162517873, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5da691f2d07486aa354c5ee7c7fb6f1", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3880387162517873, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "d02e085ef78d41109f09ed846715fdfa", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "", + "widget": "text", + "node_id": 3880387162517873, + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "54508721c33e478b94db088a93568f4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2636276550287764, + "to_node": 3880387162517873, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "98f104d28a664643b45551a76f4b6733", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2833535472085255, + "to_node": 3, + "to_slot": 2, + "from_node": 3880387162517873, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [395, 406], + "node": { + "id": 3876084544359242, + "pos": [395, 406], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "9e977850c86447acb7359b7a9b484a83", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3876084544359242, + "class_type": "EmptyLatentImage", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c9e0686dff2245fa90d9e720c0133928", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2211554611670965, + "to_node": 3, + "to_slot": 3, + "from_node": 3876084544359242, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + } + ], + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 447, + "status": "success", + "tool_call_id": "toolu_0126og8icdp1EUnSmtrcSZv6", + "tool_name": "apply_ops" + } + }, + "at_ms": 14347 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 249, + "status": "error", + "tool_call_id": "toolu_01RQTp9JXwFc1AdgmAKBiMYm", + "tool_name": "validate" + } + }, + "at_ms": 16172 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 232, + "status": "success", + "tool_call_id": "toolu_01Lq8eZseF7AhmTd9UuxvcNb", + "tool_name": "show_node" + } + }, + "at_ms": 19599 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 248, + "error_code": "knowledge_unavailable", + "status": "error", + "tool_call_id": "toolu_01VcWs7EVFuk18ct7nX9nEoi", + "tool_name": "list_model_picks" + } + }, + "at_ms": 24061 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 887, + "status": "success", + "tool_call_id": "toolu_01AB6XXLQrdF3EFbDRV9GtSx", + "tool_name": "list_generate_models" + } + }, + "at_ms": 28706 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 324, + "status": "success", + "tool_call_id": "toolu_01U2bR9WL8Pc5H35X2JERBwr", + "tool_name": "print_workflow" + } + }, + "at_ms": 38928 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab, and the text node is in and wired: your prompt \"a red bicycle on a pier at dusk\" now feeds the positive prompt. I also connected two things the graph was missing (an empty negative prompt and the blank canvas the sampler starts from) — without those it couldn't have run at all.\n\nI couldn't run it, though. The image model this workflow is built on isn't installed on this server — the checkpoint slot reports no available files at all, so there's nothing valid to point it at, and I'd rather tell you than silently swap in a different model and hand you an image that isn't the one you set up.\n\nTwo ways forward, your call:\n- Install the checkpoint this workflow expects, then hit Run — the graph is otherwise complete and correct.\n- Let me rebuild it on one of the hosted image models that are available here (Flux, Ideogram, Stable Image and others), keeping your prompt. That's a different model than your current setup, so I'd want your OK first." + } + }, + "at_ms": 44823 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 4467, + "cache_read_input_tokens": 395126, + "input_tokens": 20, + "output_tokens": 2467, + "total_tokens": 2487 + } + } + }, + "at_ms": 44827 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json new file mode 100644 index 00000000000..89acc9502c7 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-batched-ops.json @@ -0,0 +1,384 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-batched-ops", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread a6fd73be-e957-45b7-a5e0-5edd091670fb; message 9f88da92-5074-4091-850b-7cbc65005cd1; workflow e539bb63-bc5d-4f97-b918-d85862c0f591 (seeded by throwaway turn 365de61d-2131-4966-af0b-fe7ccda288bf; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['ae918468-c270-4d05-b0ef-31198a63cf0e', '069a03fa-3094-4fa4-821d-797672439db7', '044e431e-059e-45b1-b82d-45e25a92f2bf']; rows agent-rec-batched-ops.t-a1.rows.json; raw capture sha256 92757967c82950945ea1a183dbcfcb8d111ae768b3ea16a72348bb0e216b709e", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "a6fd73be-e957-45b7-a5e0-5edd091670fb", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "e539bb63-bc5d-4f97-b918-d85862c0f591", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "9f88da92-5074-4091-850b-7cbc65005cd1", + "request": { + "content": "Switch to the tab 'Text to image', then in ONE apply_ops call add two CLIPTextEncode nodes and wire both of their CLIP inputs from the checkpoint loader. Use a single batched tool call for all four operations." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "e539bb63-bc5d-4f97-b918-d85862c0f591" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 7, + "status": "success", + "tool_call_id": "toolu_01UJ5hWx9Rxkvwon9F7nMGaK", + "tool_name": "switch_tab" + } + }, + "at_ms": 2 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_011QfP8raZRXJbK5WZYpKTkv", + "tool_name": "print_workflow" + } + }, + "at_ms": 2139 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 2559261621971657, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7a130c22fe934bc98e2ea2fed6bd10ad", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2559261621971657, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "add_node", + "pos": [715, 406], + "node": { + "id": 2214070360209981, + "pos": [715, 406], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "00b500c587744366b85d5c71ddbc9fdf", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2214070360209981, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e9e5c529eb574496a516dd6014ffd5e6", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 879068618000012, + "to_node": 2559261621971657, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c7f256bd0cd34e92830062aeea907b4c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3832075311322831, + "to_node": 2214070360209981, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + } + ], + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01PhJ8AyewzRU6PHk7aFGfbC", + "tool_name": "apply_ops" + } + }, + "at_ms": 5208 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm now on your \"Text to image\" tab, and both new text prompt nodes are on the canvas with their CLIP inputs wired from the checkpoint loader. They're not connected to anything downstream yet, so tell me if you'd like one hooked up as a negative prompt." + } + }, + "at_ms": 7784 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1437, + "cache_read_input_tokens": 153211, + "input_tokens": 8, + "output_tokens": 574, + "total_tokens": 582 + } + } + }, + "at_ms": 7786 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json new file mode 100644 index 00000000000..b5ca120cf8b --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-refiner-between.json @@ -0,0 +1,448 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-refiner-between", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a; message b363781f-6c9a-4087-9190-d478781bbcd2; workflow b7d1e09f-682a-48f5-a388-d74a33ad5184 (seeded by throwaway turn 52d56bcf-7eb0-4131-a287-223e0f15e2e9; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2f4c246a-f0c7-4ca4-ad48-814bc2fe9ab6', '9727d8d7-19f4-48eb-98e6-5c054d3d6cf0', '22b43a2d-5fe8-49c0-8339-e2b3b52bf184', '33953182-42ff-46a6-a0d5-313c5a55d630', '349abb83-7fed-424e-8fb5-dc94e412aa90']; rows agent-rec-refiner-between.t-a1.rows.json; raw capture sha256 242b1c822b8b7697bdcdf1448030181ef0bb1049806870e9075077ca44897768", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "5208dc27-0c2c-4ff3-ad1d-bf7e9ad79e9a", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "b7d1e09f-682a-48f5-a388-d74a33ad5184", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "b363781f-6c9a-4087-9190-d478781bbcd2", + "request": { + "content": "Switch to the tab 'Text to image', then add a second KSampler as a refiner: feed it the first sampler's LATENT output, and wire its own LATENT output into the VAE decoder's samples input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and read what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "b7d1e09f-682a-48f5-a388-d74a33ad5184" + } + }, + "at_ms": 5 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 5, + "status": "success", + "tool_call_id": "toolu_01Bq7cRyMamrEnmo4yHuMphE", + "tool_name": "switch_tab" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 747, + "status": "success", + "tool_call_id": "toolu_01FEPdyTv3FtKtL8S6zTKj33", + "tool_name": "print_workflow" + } + }, + "at_ms": 3053 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1565, 280], + "node": { + "id": 2092962753103866, + "pos": [1565, 280], + "mode": 0, + "size": [240, 290], + "type": "KSampler", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "model", + "type": "MODEL" + }, + { + "link": null, + "name": "positive", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "negative", + "type": "CONDITIONING" + }, + { + "link": null, + "name": "latent_image", + "type": "LATENT" + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [0, "fixed", 20, 8, "euler", "simple", 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a3b5b79791364f75b1d89391a7a87564", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2092962753103866, + "class_type": "KSampler", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "a5d9733c860c40798848bf92d649988f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 4414751122447181, + "to_node": 2092962753103866, + "to_slot": 3, + "from_node": 3, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ced254ebc4a417589c32b5a1797452d", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 863234800411381, + "to_node": 2092962753103866, + "to_slot": 0, + "from_node": 4, + "from_slot": 0, + "link_type": "MODEL", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "57585fd740934c15835cef60c216c84c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3220756471207314, + "to_node": 2092962753103866, + "to_slot": 1, + "from_node": 6, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "bb113cee68314103861361151c93ad36", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 619699595909791, + "to_node": 8, + "to_slot": 0, + "from_node": 2092962753103866, + "from_slot": 0, + "link_type": "LATENT", + "base_version": 2 + }, + { + "op": "set_widget", + "old": 1, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8ba837eb610e4e0db4561b789bb44953", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 0.5, + "widget": "denoise", + "node_id": 2092962753103866, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "e83bac70162244fa96c7d4633dd5fccc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 20, + "widget": "steps", + "node_id": 2092962753103866, + "base_version": 2 + } + ], + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 198, + "status": "success", + "tool_call_id": "toolu_01LqYJ1bi41MTaeDXdzC9A9j", + "tool_name": "apply_ops" + } + }, + "at_ms": 11793 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 379, + "status": "success", + "tool_call_id": "toolu_01DAfeiUo97WETr7TRCrv4Cp", + "tool_name": "print_workflow" + } + }, + "at_ms": 14468 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "error", + "tool_call_id": "toolu_01G8xCCA9NuoEur1n8RySCWK", + "tool_name": "validate" + } + }, + "at_ms": 19339 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "The refiner is in and wired: the first sampler's latent feeds it, and its own latent now drives the VAE decoder into your save node.\n\nTwo things were already unfinished in this graph before I touched it, and they'll block a run:\n\n- Neither sampler has a negative prompt connected, and there's no empty-latent node feeding the first sampler.\n- The checkpoint it points at isn't installed here, so no model file is actually available.\n\nI set the refiner to 20 steps at 0.5 denoise, which is a sensible second-pass default. Want me to add the missing negative prompt and latent-size nodes and pick an installed checkpoint so it can actually run?" + } + }, + "at_ms": 29476 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2906, + "cache_read_input_tokens": 232638, + "input_tokens": 12, + "output_tokens": 1422, + "total_tokens": 1434 + } + } + }, + "at_ms": 29482 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json new file mode 100644 index 00000000000..964ce2952c2 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-replace-prompt-encoder.json @@ -0,0 +1,411 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-replace-prompt-encoder", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 2ce25f4b-d581-4348-ab9c-3f4170222f6f; message 37504547-e90d-4c14-91c7-b8f8fb65a0eb; workflow 5a6b35c4-8902-479d-87ab-8a9a07f08b90 (seeded by throwaway turn 3add9fbe-1dbb-4d5f-b255-d0fde410ed5e; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['d6e50c4a-5ef2-46bb-a48d-46473a36e13f', '6471d9bc-2e6f-41f1-b2c9-b417e7b9ecea', '3bda2337-3236-4585-9803-f8cbc1625d0a', '7cba8d9d-ebe4-481f-82e9-354377d2826e', '9d9678dd-2f0c-47eb-98da-0ec3720ff91d']; rows agent-rec-replace-prompt-encoder.t-a1.rows.json; raw capture sha256 eabe622a684331aa98da2a1a797cf0f761e3c866bcce8feaf2e1083a6fb7ff26", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "2ce25f4b-d581-4348-ab9c-3f4170222f6f", + "exported_at": "2026-09-03T12:40:55Z" + } + }, + "workflow": { + "id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "37504547-e90d-4c14-91c7-b8f8fb65a0eb", + "request": { + "content": "Switch to the tab 'Text to image', then delete the positive prompt node (node 6) and add a fresh CLIPTextEncode wired the same way: CLIP from the checkpoint loader, conditioning into the KSampler's positive input." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5a6b35c4-8902-479d-87ab-8a9a07f08b90" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 10, + "status": "success", + "tool_call_id": "toolu_01PbXSHPoAUnHhX2dqzJXiLB", + "tool_name": "switch_tab" + } + }, + "at_ms": 11 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 295, + "status": "success", + "tool_call_id": "toolu_01C2g2RACFUykeibS1je1Tbh", + "tool_name": "print_workflow" + } + }, + "at_ms": 3673 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 4181654812796082, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4c4d739392f14272939cf46a335652c8", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 4181654812796082, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "f1cd3aef4b984b04a9611a53f69763ba", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3320191292493608, + "to_node": 4181654812796082, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "4f4de6968f124e23a3114f8c88a1a766", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2686273519871711, + "to_node": 3, + "to_slot": 1, + "from_node": 4181654812796082, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "279c55a82f09416f9fa68e47ee11fafc", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "a photo of a pier", + "widget": "text", + "node_id": 4181654812796082, + "base_version": 2 + } + ], + "at_ms": 10451 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 223, + "status": "success", + "tool_call_id": "toolu_018Ao8cDcxmDBaz9qV7PWpGx", + "tool_name": "apply_ops" + } + }, + "at_ms": 10451 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "delete_node", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "b141d8503ee54cde94d3a4e3f6b5204f", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 6, + "base_version": 3, + "removed_links": [1] + } + ], + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 202, + "status": "success", + "tool_call_id": "toolu_01QkN5N4e8fkpBrT9aWbb4aK", + "tool_name": "delete_node" + } + }, + "at_ms": 15515 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01QUTSXXBfU7oShpWRkxtLzD", + "tool_name": "print_workflow" + } + }, + "at_ms": 18022 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — I'm on the \"Text to image\" tab, the old positive prompt node is gone, and a fresh one is in its place, taking CLIP from the checkpoint loader and feeding the sampler's positive input. I carried over the same prompt text (\"a photo of a pier\").\n\nOne thing unrelated to this swap: the sampler's negative prompt and empty latent inputs were already unconnected before I started, so the graph won't run until those are filled in. Want me to add them?" + } + }, + "at_ms": 22947 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2216, + "cache_read_input_tokens": 232121, + "input_tokens": 12, + "output_tokens": 885, + "total_tokens": 897 + } + } + }, + "at_ms": 22951 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json new file mode 100644 index 00000000000..695b4c38a9e --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-set-widget-existing.json @@ -0,0 +1,346 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-set-widget-existing", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread b5d11187-d40e-4be1-aeab-39869516773b; message 19230873-0f07-4f85-a0ae-e9f1d740cb3a; workflow 5733b17e-39f4-40f5-aa1c-468a4f355b55 (seeded by throwaway turn d00f4278-8e50-4671-a49b-8a237f1797b4; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['2383f8e0-be34-4bb1-88c5-bbe0fd34ee95', '2b9744df-c52a-4b45-b0e8-96b64642075d', 'c93c9e49-1d3d-4507-9c3d-481a0910d2c4', 'c2b5e940-a8f3-49eb-b043-e6f8360c686c', 'b538294c-6add-4ebe-b351-c09cce102a0f']; rows agent-rec-set-widget-existing.t3-a1.rows.json; raw capture sha256 6a5a92d256bb83cfd9fb95be1f182300da2361bb15a4325ce2365de6f79ad0a9", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "b5d11187-d40e-4be1-aeab-39869516773b", + "exported_at": "2026-09-03T12:38:52Z" + } + }, + "workflow": { + "id": "5733b17e-39f4-40f5-aa1c-468a4f355b55", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "19230873-0f07-4f85-a0ae-e9f1d740cb3a", + "request": { + "content": "Switch to the tab 'Text to image', then set the KSampler's steps to 30 and its cfg to 5. Change nothing else." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5733b17e-39f4-40f5-aa1c-468a4f355b55" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 12, + "status": "success", + "tool_call_id": "toolu_0127MABi8i6cpznwzfMSk2b6", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 498, + "status": "success", + "tool_call_id": "toolu_01DTMkDXHKWELHfKUTXUynfK", + "tool_name": "ls_nodes" + } + }, + "at_ms": 3903 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 425, + "status": "success", + "tool_call_id": "toolu_018Ck46iqTkvLjT12cDdcyH6", + "tool_name": "list_slots" + } + }, + "at_ms": 9834 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "set_widget", + "old": 20, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "5ee9143999c148439b6fc848bc8d309a", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 30, + "widget": "steps", + "node_id": 3, + "base_version": 2 + }, + { + "op": "set_widget", + "old": 7, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7ad6e06be4f84086bec999cfc5177816", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": 5, + "widget": "cfg", + "node_id": 3, + "base_version": 2 + } + ], + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 253, + "status": "success", + "tool_call_id": "toolu_01BaTP7o2iAhJbBjQ9tChoEG", + "tool_name": "apply_ops" + } + }, + "at_ms": 16908 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 199, + "status": "success", + "tool_call_id": "toolu_01MLp2wvSBojXfwPYKqT7dvB", + "tool_name": "list_slots" + } + }, + "at_ms": 21478 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "You're now on the \"Text to image\" tab, with steps at 30 and cfg at 5. Nothing else was touched." + } + }, + "at_ms": 23927 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1549, + "cache_read_input_tokens": 230832, + "input_tokens": 12, + "output_tokens": 509, + "total_tokens": 521 + } + } + }, + "at_ms": 23931 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json new file mode 100644 index 00000000000..66fc1ca04a2 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-text-only-answer.json @@ -0,0 +1,279 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-text-only-answer", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread f36ef047-82b4-4f2b-b480-6df821df46da; message 7de6039c-c1b4-437c-871c-2f1fc67e1750; workflow d762837d-c64d-4829-9aba-05e71b64adc3 (seeded by throwaway turn c3727b7b-569e-4eda-983a-7ccfffbf4496; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['7622ee7b-b97a-4899-8931-99488b82bee5', '5547227c-9bef-4e95-9bd1-053fb658e7b4']; rows agent-rec-text-only-answer.t-a1.rows.json; raw capture sha256 24906bcf7826bc7860dd7b109ce8d243a1c6322f56a646ea53e5645cc795c387", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "f36ef047-82b4-4f2b-b480-6df821df46da", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "d762837d-c64d-4829-9aba-05e71b64adc3", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "7de6039c-c1b4-437c-871c-2f1fc67e1750", + "request": { + "content": "Switch to the tab 'Text to image', then tell me what this workflow does." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch over and take a look." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d762837d-c64d-4829-9aba-05e71b64adc3" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_013WLWbbrNwTqV9UguY6rEu6", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 368, + "status": "success", + "tool_call_id": "toolu_01PjsdjzNXWZ8oDYHQAbjo5o", + "tool_name": "print_workflow" + } + }, + "at_ms": 2364 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now.\n\nIt's a standard SDXL text-to-image setup: it loads the SDXL base model, turns the prompt — currently \"a photo of a pier\" — into guidance, generates an image over 20 steps, and saves the result.\n\nOne thing to flag: it isn't runnable as-is. Two inputs on the generation step are still empty — there's no negative prompt wired in, and no blank canvas/image-size node telling it what dimensions to make. Want me to add those so you can run it?" + } + }, + "at_ms": 6244 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 960, + "cache_read_input_tokens": 114249, + "input_tokens": 6, + "output_tokens": 349, + "total_tokens": 355 + } + } + }, + "at_ms": 6246 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json new file mode 100644 index 00000000000..9b3462f2590 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-three-sequential-adds.json @@ -0,0 +1,416 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-three-sequential-adds", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 3710f44b-9941-4bc4-a806-5c15d00cd807; message d0d6b5dc-b189-491a-804e-f52fdb9495cb; workflow 7afd1f94-6cb5-4923-9388-18851b0fba5d (seeded by throwaway turn 9285739d-3bbe-475f-a681-6f9761c7a621; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['c27f6e4e-220a-42fc-8fd1-97c30d6d3c98', 'f020586d-9444-457f-85e9-8c96b2be6292', '3832f4c1-77c3-44a9-b890-83b82db5513a', 'df9c490c-5a9a-43b3-b6d7-931c40084eaf']; rows agent-rec-three-sequential-adds.cc-a1.rows.json; raw capture sha256 6a1dfc3c7d4961130ecc8e1b96edd76c9ec288602e38d86432b16e2ae3d342b6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "3710f44b-9941-4bc4-a806-5c15d00cd807", + "exported_at": "2026-09-03T14:05:21Z" + } + }, + "workflow": { + "id": "7afd1f94-6cb5-4923-9388-18851b0fba5d", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "d0d6b5dc-b189-491a-804e-f52fdb9495cb", + "request": { + "content": "Switch to the tab 'Text to image', then add three nodes one at a time, each with its own separate tool call and nothing else in that call: first a PrimitiveStringMultiline, then an EmptyLatentImage, then a CLIPTextEncode. Do not connect or configure them." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch focus to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "7afd1f94-6cb5-4923-9388-18851b0fba5d" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_0199bXc7oQVbgJA4aY56VcfY", + "tool_name": "switch_tab" + } + }, + "at_ms": 9 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1845, 0], + "node": { + "id": 2478057798252548, + "pos": [1845, 0], + "mode": 0, + "size": [240, 86], + "type": "PrimitiveStringMultiline", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "cf6a6fdb84ee4f7dbec557f999d2fa8f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2478057798252548, + "class_type": "PrimitiveStringMultiline", + "base_version": 2 + } + ], + "at_ms": 2686 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 964, + "status": "success", + "tool_call_id": "toolu_014VjtpNeXB9q2rBXxK9bkm9", + "tool_name": "add_node" + } + }, + "at_ms": 2686 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2165, 0], + "node": { + "id": 2772376668635982, + "pos": [2165, 0], + "mode": 0, + "size": [240, 134], + "type": "EmptyLatentImage", + "flags": {}, + "order": 0, + "inputs": [], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [] + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "48a4c2f2ea8647d1be5b0817985bda87", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2772376668635982, + "class_type": "EmptyLatentImage", + "base_version": 3 + } + ], + "at_ms": 4441 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 220, + "status": "success", + "tool_call_id": "toolu_01M6Jpuof94JLfgBij4etkaN", + "tool_name": "add_node" + } + }, + "at_ms": 4441 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [2485, 0], + "node": { + "id": 2514973844700532, + "pos": [2485, 0], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "da3e136d6b224cc8838a0d9aa569ede3", + "stamp": [4, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 2514973844700532, + "class_type": "CLIPTextEncode", + "base_version": 4 + } + ], + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 192, + "status": "success", + "tool_call_id": "toolu_01FzkN53SHp6ePAFbmX2H8oX", + "tool_name": "add_node" + } + }, + "at_ms": 7816 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "All three nodes are on the \"Text to image\" tab now — a multiline text box, an empty latent, and a text encoder — left unconnected and unconfigured as you asked." + } + }, + "at_ms": 9873 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1944, + "cache_read_input_tokens": 192480, + "input_tokens": 10, + "output_tokens": 322, + "total_tokens": 332 + } + } + }, + "at_ms": 9875 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json new file mode 100644 index 00000000000..0508a4b8c07 --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-tool-error.json @@ -0,0 +1,280 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-tool-error", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386; message a5ec5ead-1909-4322-9d62-edbb03a3ded9; workflow 5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5 (seeded by throwaway turn 788de1fc-829b-4636-b715-2879479effbc; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['39a22dd2-4aaa-4d7d-9505-779ab6386ee8', 'aeb6b2f3-8d0c-4131-a913-144322ca42c8']; rows agent-rec-tool-error.t-a1.rows.json; raw capture sha256 83bd2c2c30c00883df39527e5f482deccc5e10c34a0d6554c34bbff8d725c0e6", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "934d7b9d-72d9-40ab-a3d9-3cdb6fa5a386", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "a5ec5ead-1909-4322-9d62-edbb03a3ded9", + "request": { + "content": "Switch to the tab 'Text to image', then add a node of class DoesNotExist." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "5c1db1ce-4ca3-4db9-b559-4d3463d0c6e5" + } + }, + "at_ms": 9 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01YWm16Ejys8FJBofYwL7k9E", + "tool_name": "switch_tab" + } + }, + "at_ms": 12 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 346, + "error_code": "node_not_found", + "status": "error", + "tool_call_id": "toolu_01XqvtmCXKboog1UTZ3MGNJR", + "tool_name": "add_node" + } + }, + "at_ms": 2654 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm now on the \"Text to image\" tab, but there's no node called \"DoesNotExist\" — nothing was added, and the canvas is unchanged.\n\nIf you tell me what you actually want that node to do (e.g. upscale, add a LoRA, save output), I'll find the real one and wire it in." + } + }, + "at_ms": 5701 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 752, + "cache_read_input_tokens": 114291, + "input_tokens": 6, + "output_tokens": 295, + "total_tokens": 301 + } + } + }, + "at_ms": 5703 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json new file mode 100644 index 00000000000..7cd2dbd9d7c --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-rec-two-turn-dependent-edit.json @@ -0,0 +1,449 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-rec-two-turn-dependent-edit", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 6cafa19d-11f0-45e9-bcf1-e52529eb9ebb; messages 445f7340-ea9c-4456-8ffa-5d37ea6c7b98, 07fc770a-df43-42d8-8f2c-07bf9c2db9f8; workflow d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447 (seeded by throwaway turn af193614-a407-4987-99a5-97a04d52eb35; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows 42fbcd91-26cb-415a-b471-8daa44a8bd6c, 55c090e2-11f7-48b7-93dc-5fa9fd73e1ac, abbd53d0-2352-4d5d-bfd0-0bd2601a23ce, b5bde96e-939a-4b5b-992d-5fc9ece40a96, da2d0551-1fcb-435d-9dd1-745a451d75f9, f7084475-8dd0-4789-a14f-0a2cd47379dc; rows agent-rec-two-turn-dependent-edit.m1.rows.1.json, agent-rec-two-turn-dependent-edit.m1.rows.2.json; raw capture sha256 49a64b8ac694b4778b78b3df03a372eaed59e60baca70299ee5cee867b23be43", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "6cafa19d-11f0-45e9-bcf1-e52529eb9ebb", + "exported_at": "2026-09-03T16:03:42.552Z" + } + }, + "workflow": { + "id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447", + "name": "Text to image", + "catalog": { + "types": { + "CheckpointLoaderSimple": { + "widget_order": ["ckpt_name"] + }, + "CLIPTextEncode": { + "widget_order": ["text"] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "PrimitiveStringMultiline": { + "widget_order": ["value"] + }, + "EmptyLatentImage": { + "widget_order": ["width", "height", "batch_size"] + } + } + }, + "seed": { + "nodes": [ + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [0, 0], + "size": [315, 98], + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [3] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [1] + }, + { + "name": "VAE", + "type": "VAE", + "links": [5] + } + ], + "widgets_values": ["sd_xl_base_1.0.safetensors"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "title": "Positive prompt", + "pos": [400, 0], + "size": [400, 200], + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": null + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [2] + } + ], + "widgets_values": ["a photo of a pier"] + }, + { + "id": 3, + "type": "KSampler", + "pos": [850, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 3 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [4] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1200, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 5 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [6] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1450, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 6 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 4, 1, 6, 0, "CLIP"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 4, 0, 3, 0, "MODEL"], + [4, 3, 0, 8, 0, "LATENT"], + [5, 4, 2, 8, 1, "VAE"], + [6, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "445f7340-ea9c-4456-8ffa-5d37ea6c7b98", + "request": { + "content": "Switch to the tab 'Text to image', then add a second CLIPTextEncode node fed from the checkpoint loader's CLIP output and give it the text 'blurry, low quality'." + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at the graph first." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Text to image", + "workflow_id": "d8c3e6c4-dd9f-4e19-8c7c-8cdcbb491447" + } + }, + "at_ms": 6 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 8, + "status": "success", + "tool_call_id": "toolu_01WdrFCAxyg6dbYLfwVHfPTD", + "tool_name": "switch_tab" + } + }, + "at_ms": 8 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 752, + "status": "success", + "tool_call_id": "toolu_01E4tYHkXrhtb6ktKRki4z4r", + "tool_name": "print_workflow" + } + }, + "at_ms": 2435 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [715, 280], + "node": { + "id": 3337990138003756, + "pos": [715, 280], + "mode": 0, + "size": [240, 86], + "type": "CLIPTextEncode", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "clip", + "type": "CLIP" + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [] + } + ], + "properties": {}, + "widgets_values": [null] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "c00397c5ac4a411bafa8c29389fb1515", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 3337990138003756, + "class_type": "CLIPTextEncode", + "base_version": 2 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "720d36f953794c1a8d05c3558c8c7a0f", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2557345504960691, + "to_node": 3337990138003756, + "to_slot": 0, + "from_node": 4, + "from_slot": 1, + "link_type": "CLIP", + "base_version": 2 + }, + { + "op": "set_widget", + "old": null, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "959f6c6e6cbc4dd3a39f784437208f1c", + "stamp": [2, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "value": "blurry, low quality", + "widget": "text", + "node_id": 3337990138003756, + "base_version": 2 + } + ], + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 195, + "status": "success", + "tool_call_id": "toolu_013FPeYq4Ya1M9VJKsMPJE4j", + "tool_name": "apply_ops" + } + }, + "at_ms": 7788 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 196, + "status": "success", + "tool_call_id": "toolu_01RL1yma6QfBbL3cZQs3qsf6", + "tool_name": "print_workflow" + } + }, + "at_ms": 11731 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "I'm on the \"Text to image\" tab now, and the second text box is added with \"blurry, low quality\" and fed from the checkpoint loader.\n\nOne thing worth flagging: it isn't connected to anything downstream yet, so it has no effect on the image. The sampler's negative input is currently empty (and its latent input too) — say the word and I'll wire this new text into the negative slot." + } + }, + "at_ms": 15233 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 2013, + "cache_read_input_tokens": 192537, + "input_tokens": 10, + "output_tokens": 822, + "total_tokens": 832 + } + } + }, + "at_ms": 15235 + } + ] + }, + { + "message_id": "07fc770a-df43-42d8-8f2c-07bf9c2db9f8", + "request": { + "content": "Now connect that new CLIPTextEncode node to the KSampler's negative input." + }, + "response": [ + { + "kind": "graph_ops", + "ops": [ + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "0a73a31fd4c54bf8a2bf0746d41710cd", + "stamp": [3, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 1795569972720099, + "to_node": 3, + "to_slot": 2, + "from_node": 3337990138003756, + "from_slot": 0, + "link_type": "CONDITIONING", + "base_version": 3 + } + ], + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 298, + "status": "success", + "tool_call_id": "toolu_016EPZQoQXsKsbm5qAzKPHGD", + "tool_name": "connect" + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 200, + "status": "success", + "tool_call_id": "toolu_01RZH4jfk5SdpGS1xF2dMfvw", + "tool_name": "print_workflow" + } + }, + "at_ms": 4272 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Connected — the \"blurry, low quality\" text now feeds the negative input.\n\nThe graph still won't run as-is: nothing is feeding the sampler's latent input, so it needs an empty latent (which is also where the image size gets set). Want me to add one?" + } + }, + "at_ms": 8606 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 1542, + "cache_read_input_tokens": 115114, + "input_tokens": 6, + "output_tokens": 376, + "total_tokens": 382 + } + } + }, + "at_ms": 8610 + } + ] + } + ] +} diff --git a/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json new file mode 100644 index 00000000000..7547ef59dcb --- /dev/null +++ b/browser_tests/fixtures/data/agent/conversations/agent-workflow-editing-05.json @@ -0,0 +1,424 @@ +{ + "schema_version": "agent-conversation.v2", + "source": { + "repo": "Comfy-Org/ComfyUI_frontend", + "suite": "agent", + "case_id": "agent-workflow-editing-05", + "response_side": "recorded", + "note": "RECORDED from Comfy-Org/cloud services/agent running NON-standalone local full stack (Postgres + doc host, M2M identity headers) at http://127.0.0.1:8086 (frames: redis SUBSCRIBE channel:ws::u:); NOT a production capture. cloud commit 9dc1da7dae2a1d40a9ca1f2a1f91cd9fd9e10900; model claude-opus-5; thread 8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3; message b199d9ab-1800-49b2-8a43-458cda7f104d; workflow f47e55ab-2cd5-414e-90d3-23a5fc7eee22 (seeded by throwaway turn 3d61dd80-dd03-4957-8506-734392c9583c; the recorded turn opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame, see plan Q4); agent_tool_calls parent rows ['6206d0ef-f781-41db-81bd-4138950ccd2e', 'fb873a4b-59b4-4088-ae32-f43e0a7382c5', '473ea6be-f9c3-4a75-8776-0e7823cd5054', 'e6ec2a79-8f0b-408a-b01e-f9362ae8fde4', '47d7581e-2d55-49ea-ac3e-1d6520d80454', 'abd6c054-6b5b-49a2-8633-881078e9b07c', '3f83a246-0181-4d1a-b2b5-c1fcfac38fe0']; rows agent-workflow-editing-05.t-a1.rows.json; raw capture sha256 982353ccbf5d92ca70a7857eb7da662aa2d19b0fcebb27181a71c3a95b762658", + "capture": { + "backend": "Comfy-Org/cloud", + "thread_id": "8f2ed7bf-7e42-49dc-a2af-0a15e4f55db3", + "exported_at": "2026-09-03T12:40:56Z" + } + }, + "workflow": { + "id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22", + "name": "Image edit", + "catalog": { + "types": { + "LoadImage": { + "widget_order": ["image", "upload"] + }, + "VAEEncode": { + "widget_order": [] + }, + "KSampler": { + "widget_order": [ + "seed", + "control_after_generate", + "steps", + "cfg", + "sampler_name", + "scheduler", + "denoise" + ] + }, + "VAEDecode": { + "widget_order": [] + }, + "SaveImage": { + "widget_order": ["filename_prefix"] + }, + "ImageStitch": { + "widget_order": [ + "direction", + "match_image_size", + "spacing_width", + "spacing_color" + ] + }, + "PreviewImage": { + "widget_order": [] + } + } + }, + "seed": { + "nodes": [ + { + "id": 1, + "type": "LoadImage", + "pos": [0, 0], + "size": [315, 314], + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [1] + }, + { + "name": "MASK", + "type": "MASK", + "links": [] + } + ], + "widgets_values": ["example.png", "image"] + }, + { + "id": 2, + "type": "VAEEncode", + "pos": [400, 0], + "size": [210, 46], + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 1 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2] + } + ], + "widgets_values": [] + }, + { + "id": 3, + "type": "KSampler", + "pos": [700, 0], + "size": [315, 262], + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": null + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [3] + } + ], + "widgets_values": [0, "randomize", 20, 7, "euler", "normal", 0.6] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1100, 0], + "size": [210, 46], + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 3 + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [4] + } + ], + "widgets_values": [] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1400, 0], + "size": [315, 270], + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "widgets_values": ["ComfyUI"] + } + ], + "links": [ + [1, 1, 0, 2, 0, "IMAGE"], + [2, 2, 0, 3, 3, "LATENT"], + [3, 3, 0, 8, 0, "LATENT"], + [4, 8, 0, 9, 0, "IMAGE"] + ] + } + }, + "turns": [ + { + "message_id": "b199d9ab-1800-49b2-8a43-458cda7f104d", + "request": { + "content": "Switch to the tab 'Image edit', then use a stitch images node to show the output next to the input" + }, + "response": [ + { + "kind": "event", + "event": { + "type": "agent_thinking", + "data": { + "delta": "I'll switch to that tab and take a look at what's there." + } + }, + "at_ms": 0 + }, + { + "kind": "event", + "event": { + "type": "agent_active_tab", + "data": { + "name": "Image edit", + "workflow_id": "f47e55ab-2cd5-414e-90d3-23a5fc7eee22" + } + }, + "at_ms": 7 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 9, + "status": "success", + "tool_call_id": "toolu_01RMDkomGUUpDL8wixWq9Fhe", + "tool_name": "switch_tab" + } + }, + "at_ms": 14 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 366, + "status": "success", + "tool_call_id": "toolu_01JECLyu71YJ1niLhepMB63u", + "tool_name": "print_workflow" + } + }, + "at_ms": 2905 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 194, + "status": "success", + "tool_call_id": "toolu_01Sttwp3yTjiF6LdrL5GdmVy", + "tool_name": "find_nodes" + } + }, + "at_ms": 7034 + }, + { + "kind": "graph_ops", + "ops": [ + { + "op": "add_node", + "pos": [1710, 280], + "node": { + "id": 1177948620476144, + "pos": [1710, 280], + "mode": 0, + "size": [240, 178], + "type": "ImageStitch", + "flags": {}, + "order": 0, + "inputs": [ + { + "link": null, + "name": "image1", + "type": "IMAGE" + }, + { + "link": null, + "name": "image2", + "type": "IMAGE" + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + } + ], + "properties": {}, + "widgets_values": ["right", true, 0, "white"] + }, + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "8e44f8dcf31e46f6bf6d487c1bec2676", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "node_id": 1177948620476144, + "class_type": "ImageStitch", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "792213bf2f824cfd8866f157117795a4", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3298141227647238, + "to_node": 1177948620476144, + "to_slot": 0, + "from_node": 1, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "7c91fdf62a4d437dac4ada8d3ae13661", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 2572689854678654, + "to_node": 1177948620476144, + "to_slot": 1, + "from_node": 8, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + }, + { + "op": "connect", + "actor": "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4", + "op_id": "3eaf0c9e52cb4f0994962ff4b1e341ff", + "stamp": [1, "u-a02b5a87-26d8-451a-a5fc-07b54cbe6db4"], + "link_id": 3588278739910384, + "to_node": 9, + "to_slot": 0, + "from_node": 1177948620476144, + "from_slot": 0, + "link_type": "IMAGE", + "base_version": 1 + } + ], + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 208, + "status": "success", + "tool_call_id": "toolu_01DNrpeJQDjW2v32xme5eyDq", + "tool_name": "apply_ops" + } + }, + "at_ms": 13620 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 210, + "status": "success", + "tool_call_id": "toolu_01VFMK15TdDdT4zFK3w8bRqE", + "tool_name": "list_slots" + } + }, + "at_ms": 20740 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 206, + "error_code": "conflict", + "status": "error", + "tool_call_id": "toolu_01GZUZLG9RtoeGym25Hy5p5P", + "tool_name": "set_widget" + } + }, + "at_ms": 25170 + }, + { + "kind": "event", + "event": { + "type": "agent_tool_call", + "data": { + "duration_ms": 216, + "status": "error", + "tool_call_id": "toolu_01FCgU1tuBtusNgYiPMRaUQL", + "tool_name": "validate" + } + }, + "at_ms": 29613 + }, + { + "kind": "event", + "event": { + "type": "agent_message_delta", + "data": { + "delta": "Done — the side-by-side comparison is on your canvas: the original image and the edited result now go into a stitch node (edited placed to the right of the original, sizes matched), and that combined image is what gets saved.\n\nTwo things worth knowing:\n\n- The gap/spacing setting on that node can't be changed from here, so the two images sit flush against each other. You can set a spacing value directly on the node if you want a divider.\n- The rest of this workflow was already incomplete before I touched it — the sampler has no model loaded and no prompt text, and the encode/decode steps have no VAE connected. It won't run until those are filled in. Want me to wire up a checkpoint and prompt boxes for you?" + } + }, + "at_ms": 38675 + }, + { + "kind": "event", + "event": { + "type": "agent_message_done", + "data": { + "usage": { + "cache_creation_input_tokens": 3425, + "cache_read_input_tokens": 313028, + "input_tokens": 16, + "output_tokens": 1658, + "total_tokens": 1674 + } + } + }, + "at_ms": 38677 + } + ] + } + ] +} diff --git a/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts new file mode 100644 index 00000000000..060ac4db564 --- /dev/null +++ b/browser_tests/tests/agent/agentConversationConcurrentEdits.spec.ts @@ -0,0 +1,64 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { hasAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +const HUMAN_TEXT = 'a lighthouse at dawn, typed while the agent works' + +// The recording lands with the data PR; without it there is nothing to replay. +if (hasAgentConversation('agent-rec-three-sequential-adds')) + test.describe( + 'Agent conversation replay with concurrent human edits', + { tag: '@cloud' }, + () => { + test.use({ conversationCase: 'agent-rec-three-sequential-adds' }) + + test('keeps a human widget edit made between the agent adds', async ({ + agentConversation, + page + }) => { + test.setTimeout(60_000) + const { conversation } = agentConversation + const opsEntries = conversation.turns[0].response + .map((entry, index) => (entry.kind === 'graph_ops' ? index : -1)) + .filter((index) => index >= 0) + expect(opsEntries.length).toBeGreaterThanOrEqual(3) + const promptText = page + .locator('[data-node-id="6"]') + .getByLabel('text', { exact: true }) + + await agentConversation.sendPrompt() + await agentConversation.replayResponse(0, async (index) => { + if (index !== opsEntries[0]) return + await promptText.fill(HUMAN_TEXT) + await expect + .poll(() => + agentConversation + .outboundOps() + .some((op) => op.op === 'set_widget') + ) + .toBe(true) + }) + await agentConversation.waitForTurnComplete() + + for (const id of agentConversation.addedNodeIds()) { + await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() + } + await expect(promptText).toHaveValue(HUMAN_TEXT) + expect( + String( + agentConversation.hostGraph().nodes['6']?.widgets && + ( + agentConversation.hostGraph().nodes['6'].widgets as Record< + string, + unknown + > + ).text + ) + ).toBe(HUMAN_TEXT) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + } + ) diff --git a/browser_tests/tests/agent/agentConversationReplay.spec.ts b/browser_tests/tests/agent/agentConversationReplay.spec.ts new file mode 100644 index 00000000000..562291c38d1 --- /dev/null +++ b/browser_tests/tests/agent/agentConversationReplay.spec.ts @@ -0,0 +1,91 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { listRecordedConversations } from '@e2e/fixtures/data/agent/agentConversation' + +test.describe('Agent conversation replay', { tag: '@cloud' }, () => { + for (const conversationCase of listRecordedConversations()) { + test.describe(`recorded ${conversationCase}`, () => { + test.use({ conversationCase }) + + test('replays the recorded turn onto the panel and the canvas', async ({ + agentConversation, + page + }) => { + test.setTimeout(60_000) + const { panel } = agentConversation + + await agentConversation.runTurns() + + const groups = agentConversation.toolCallGroups() + const groupButtons = panel.getByRole('button', { + name: /^Ran \d+ tool call/ + }) + await expect(groupButtons).toHaveCount(groups.length) + const toolRows = panel.getByRole('listitem') + for (const [index, calls] of groups.entries()) { + const button = groupButtons.nth(index) + const succeeded = calls.every((call) => call.ok) + await expect(button).toHaveText( + new RegExp(`^Ran ${calls.length} tool call`) + ) + // A failed group stays open at turn end; a clean one collapses. + await expect(button).toHaveAttribute( + 'aria-expanded', + succeeded ? 'false' : 'true' + ) + if (succeeded) await button.click() + } + for (const { + label, + times, + rows + } of agentConversation.toolRowCounts()) { + const named = toolRows.filter({ + has: page.getByText(label, { exact: true }) + }) + await expect( + named.filter( + times > 1 ? { hasText: `×${times}` } : { hasNotText: '×' } + ) + ).toHaveCount(rows) + } + + await expect( + panel.getByRole('button', { + name: `Open ${agentConversation.conversation.workflow.name}` + }) + ).toBeVisible() + await expect( + panel.getByTestId('markdown-stream').first() + ).not.toBeEmpty() + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + 'connected' + ) + await expect(panel.getByTestId('agent-crdt-status')).toContainText( + new RegExp(`\\b${agentConversation.expectedUpdateCount()} updates`) + ) + + for (const id of agentConversation.addedNodeIds()) { + await expect(page.locator(`[data-node-id="${id}"]`)).toBeVisible() + } + for (const { + nodeId, + widget, + value + } of agentConversation.recordedWidgetValues()) { + const field = page + .locator(`[data-node-id="${nodeId}"]`) + .getByLabel(widget, { exact: true }) + const control = + typeof value === 'string' ? field : field.locator('input') + await expect(control).toHaveValue(String(value)) + } + + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + }) + } +}) diff --git a/browser_tests/tests/agent/agentConversationTiming.spec.ts b/browser_tests/tests/agent/agentConversationTiming.spec.ts new file mode 100644 index 00000000000..8a6494b8458 --- /dev/null +++ b/browser_tests/tests/agent/agentConversationTiming.spec.ts @@ -0,0 +1,33 @@ +import { expect } from '@playwright/test' + +import { agentConversationTest as test } from '@e2e/fixtures/agentConversationFixture' +import { hasAgentConversation } from '@e2e/fixtures/data/agent/agentConversation' + +// The recording lands with the data PR; without it there is nothing to replay. +if (hasAgentConversation('agent-rec-set-widget-existing')) + test.describe('Agent conversation replay timing', { tag: '@cloud' }, () => { + test.use({ + conversationCase: 'agent-rec-set-widget-existing', + replayTiming: 'recorded' + }) + + test('replays the frames at their recorded offsets', async ({ + agentConversation + }) => { + test.setTimeout(120_000) + const offsets = agentConversation.conversation.turns[0].response.flatMap( + (entry) => (entry.at_ms === undefined ? [] : [entry.at_ms]) + ) + const span = offsets.length ? Math.max(...offsets) : undefined + expect(span, 'the fixture carries at_ms offsets').toBeDefined() + + await agentConversation.sendPrompt() + await agentConversation.replayResponse() + await agentConversation.waitForTurnComplete() + + expect(agentConversation.replayElapsedMs).toBeGreaterThanOrEqual(span!) + await expect + .poll(() => agentConversation.graphSnapshot()) + .toEqual(agentConversation.expectedGraph()) + }) + }) diff --git a/scripts/agentConversationCapture.test.ts b/scripts/agentConversationCapture.test.ts new file mode 100644 index 00000000000..36627575142 --- /dev/null +++ b/scripts/agentConversationCapture.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from 'vitest' + +import { zAgentConversation } from '../browser_tests/fixtures/data/agent/agentConversation' +import type { AgentBackendCapture } from './agentConversationCapture' +import { exportAgentConversation } from './agentConversationCapture' + +const turn: AgentBackendCapture['turns'][number] = { + message_id: 'message-1', + request: { content: 'Add a node' }, + frames: [ + { + type: 'agent_tool_call', + data: { + thread_id: 'thread-1', + message_id: 'message-1', + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'running' + } + }, + { + type: 'agent_tool_call', + data: { + thread_id: 'thread-1', + message_id: 'message-1', + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'success' + } + } + ], + tool_calls: [ + { + tool_call_id: 'tool-1', + applied_op_ids: ['op-accepted'], + result: { + data: { + ops: [ + { op: 'add_node', op_id: 'op-accepted', node_id: 1 }, + { op: 'add_node', op_id: 'op-rejected', node_id: 2 } + ] + } + } + } + ] +} + +const capture = { + schema_version: 'agent-backend-capture.v2', + source: { + repo: 'Comfy-Org/evals', + suite: 'agent', + case_id: 'recorded-case' + }, + capture: { + backend: 'Comfy-Org/cloud', + thread_id: 'thread-1', + exported_at: '2026-09-02T16:00:00.000Z' + }, + workflow: { + id: '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01', + name: 'Captured workflow', + catalog: { types: {} }, + seed: { nodes: [], links: [] } + }, + turns: [turn] +} + +const withTurn = (patch: Partial) => ({ + ...capture, + turns: [{ ...turn, ...patch }] +}) + +describe('exportAgentConversation', () => { + it('interleaves only durably applied backend ops before the terminal frame', () => { + const conversation = exportAgentConversation(capture) + + expect(conversation.source).toMatchObject({ + response_side: 'recorded', + capture: capture.capture + }) + expect(conversation.turns[0].response).toEqual([ + { + kind: 'event', + event: { + type: 'agent_tool_call', + data: { + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'running' + } + } + }, + { + kind: 'graph_ops', + ops: [{ op: 'add_node', op_id: 'op-accepted', node_id: 1 }] + }, + { + kind: 'event', + event: { + type: 'agent_tool_call', + data: { + tool_call_id: 'tool-1', + tool_name: 'add_node', + status: 'success' + } + } + } + ]) + }) + + it('emits each entry at its offset from the first recorded frame', () => { + const [running, success] = turn.frames + const exported = exportAgentConversation( + withTurn({ + frames: [ + { ...running, at_ms: 1_700_000_001_000 }, + { ...success, at_ms: 1_700_000_001_750 } + ] + }) + ) + const [{ response }] = exported.turns + expect(response.map((entry) => entry.at_ms)).toEqual([0, 750, 750]) + expect( + response.every( + (entry) => entry.kind !== 'event' || !('at_ms' in entry.event) + ) + ).toBe(true) + }) + + it('leaves the offset out when the frames carry no receipt time', () => { + const [{ response }] = exportAgentConversation(capture).turns + expect(response.every((entry) => entry.at_ms === undefined)).toBe(true) + }) + + it('refuses a tool-call frame with a missing status', () => { + const [running, success] = turn.frames + const { status: _status, ...withoutStatus } = success.data + expect(() => + exportAgentConversation( + withTurn({ frames: [running, { ...success, data: withoutStatus }] }) + ) + ).toThrow(/status undefined/) + }) + + it('refuses a tool-call frame with an unknown status', () => { + const [running, success] = turn.frames + expect(() => + exportAgentConversation( + withTurn({ + frames: [ + running, + { ...success, data: { ...success.data, status: 'done' } } + ] + }) + ) + ).toThrow(/status "done"/) + }) + + it('refuses an accepted op missing from the recorded result', () => { + expect(() => + exportAgentConversation( + withTurn({ + tool_calls: [ + { ...turn.tool_calls[0], applied_op_ids: ['op-not-in-result'] } + ] + }) + ) + ).toThrow('accepted op op-not-in-result') + }) + + it('refuses to silently omit a mutating backend call', () => { + expect(() => + exportAgentConversation(withTurn({ frames: [turn.frames[0]] })) + ).toThrow('no terminal websocket frame for tool call(s): tool-1') + }) + + it('refuses a websocket frame from another turn', () => { + expect(() => + exportAgentConversation( + withTurn({ + frames: [ + { + ...turn.frames[0], + data: { ...turn.frames[0].data, message_id: 'another-message' } + } + ] + }) + ) + ).toThrow('does not belong to turn thread-1/message-1') + }) + + it('exports every turn of the thread in order with its own frames', () => { + const second = { + ...turn, + message_id: 'message-2', + request: { content: 'Connect it' }, + frames: turn.frames.map((frame) => ({ + ...frame, + data: { ...frame.data, message_id: 'message-2' } + })) + } + const exported = exportAgentConversation({ + ...capture, + turns: [turn, second] + }) + expect( + exported.turns.map((exportedTurn) => exportedTurn.message_id) + ).toEqual(['message-1', 'message-2']) + expect(exported.turns[1].request).toEqual({ content: 'Connect it' }) + expect(exported.turns[1].response).toHaveLength(3) + }) + + it('refuses a recorded turn without the message id it came from', () => { + const conversation = exportAgentConversation(capture) + const { message_id: _messageId, ...anonymous } = conversation.turns[0] + expect(() => + zAgentConversation.parse({ ...conversation, turns: [anonymous] }) + ).toThrow('recorded turns carry the message id') + }) + + it('refuses a recorded label without backend provenance', () => { + const conversation = exportAgentConversation(capture) + const { capture: _capture, ...source } = conversation.source + + expect(() => zAgentConversation.parse({ ...conversation, source })).toThrow( + 'recorded responses require backend capture provenance' + ) + }) +}) diff --git a/scripts/agentConversationCapture.ts b/scripts/agentConversationCapture.ts new file mode 100644 index 00000000000..5ccf7d86e90 --- /dev/null +++ b/scripts/agentConversationCapture.ts @@ -0,0 +1,190 @@ +#!/usr/bin/env tsx + +import { readFileSync, writeFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import { + zAgentConversation, + zAgentConversationRequest, + zAgentConversationWorkflow, + zRecordedWsEvent +} from '../browser_tests/fixtures/data/agent/agentConversation' + +const zOperation = z + .record(z.string(), z.unknown()) + .refine( + (operation) => + typeof operation.op === 'string' && + (FROZEN_OPS as readonly string[]).includes(operation.op), + 'unknown graph operation' + ) + +const zToolCall = z.object({ + tool_call_id: z.string().min(1), + result: z.record(z.string(), z.unknown()), + applied_op_ids: z.array(z.string().min(1)) +}) + +const zBackendTurn = z.object({ + message_id: z.string().min(1), + request: zAgentConversationRequest, + frames: z.array(zRecordedWsEvent).min(1), + tool_calls: z.array(zToolCall) +}) + +const zBackendCapture = z.object({ + schema_version: z.literal('agent-backend-capture.v2'), + source: z.object({ + repo: z.string(), + suite: z.string(), + case_id: z.string(), + note: z.string().optional() + }), + capture: z.object({ + backend: z.literal('Comfy-Org/cloud'), + thread_id: z.string().min(1), + exported_at: z.string().datetime() + }), + workflow: zAgentConversationWorkflow, + turns: z.array(zBackendTurn).min(1) +}) +export type AgentBackendCapture = z.input + +function operationsFromResult(toolCall: z.infer) { + const data = toolCall.result.data + if (typeof data !== 'object' || data === null || Array.isArray(data)) + return [] + + const result = data as Record + const candidates = Array.isArray(result.ops) + ? result.ops + : result.op === undefined + ? [] + : [result.op] + const operations = candidates.map((operation) => zOperation.parse(operation)) + const byId = new Map( + operations.flatMap((operation) => + typeof operation.op_id === 'string' + ? [[operation.op_id, operation] as const] + : [] + ) + ) + + return toolCall.applied_op_ids.map((opId) => { + const operation = byId.get(opId) + if (operation === undefined) { + throw new Error( + `tool call ${toolCall.tool_call_id} accepted op ${opId}, but its recorded result does not contain that op` + ) + } + return operation + }) +} + +// Ops are inserted before their terminal tool-call frame and must exist in the accepted-op rows; nothing is authored here. +function exportTurn(turn: z.infer, threadId: string) { + const toolCalls = new Map( + turn.tool_calls.map((toolCall) => [toolCall.tool_call_id, toolCall]) + ) + const emittedToolCalls = new Set() + const response: Array< + | { kind: 'event'; event: z.infer; at_ms?: number } + | { kind: 'graph_ops'; ops: Array>; at_ms?: number } + > = [] + const firstAt = turn.frames[0].at_ms + const relativeAt = (frame: { at_ms?: number }) => + firstAt === undefined || frame.at_ms === undefined + ? undefined + : frame.at_ms - firstAt + + for (const frame of turn.frames) { + if ( + frame.data.thread_id !== threadId || + frame.data.message_id !== turn.message_id + ) { + throw new Error( + `recorded ${frame.type} frame does not belong to turn ${threadId}/${turn.message_id}` + ) + } + const at_ms = relativeAt(frame) + const event = { + type: frame.type, + data: { ...frame.data } + } + delete event.data.thread_id + delete event.data.message_id + + if (event.type === 'agent_tool_call') { + const toolCallId = event.data.tool_call_id + const status = event.data.status + if (status !== 'running' && status !== 'success' && status !== 'error') { + throw new Error( + `recorded agent_tool_call frame carries status ${JSON.stringify(status)}; only running, success or error are known` + ) + } + if ( + typeof toolCallId === 'string' && + status !== 'running' && + toolCalls.has(toolCallId) + ) { + const operations = operationsFromResult(toolCalls.get(toolCallId)!) + if (operations.length > 0) { + response.push({ kind: 'graph_ops', ops: operations, at_ms }) + } + emittedToolCalls.add(toolCallId) + } + } + response.push({ kind: 'event', event, at_ms }) + } + + const omitted = turn.tool_calls.filter( + (toolCall) => + toolCall.applied_op_ids.length > 0 && + !emittedToolCalls.has(toolCall.tool_call_id) + ) + if (omitted.length > 0) { + throw new Error( + `recorded response has no terminal websocket frame for tool call(s): ${omitted.map((call) => call.tool_call_id).join(', ')}` + ) + } + return { message_id: turn.message_id, request: turn.request, response } +} + +export function exportAgentConversation(input: unknown) { + const capture = zBackendCapture.parse(input) + return zAgentConversation.parse({ + schema_version: 'agent-conversation.v2', + source: { + ...capture.source, + response_side: 'recorded', + capture: capture.capture + }, + workflow: capture.workflow, + turns: capture.turns.map((turn) => + exportTurn(turn, capture.capture.thread_id) + ) + }) +} + +function main(args: string[]): void { + if (args.length !== 2) { + throw new Error( + 'usage: pnpm exec tsx scripts/agentConversationCapture.ts ' + ) + } + const [inputPath, outputPath] = args + const conversation = exportAgentConversation( + JSON.parse(readFileSync(inputPath, 'utf8')) + ) + writeFileSync(outputPath, `${JSON.stringify(conversation, null, 2)}\n`) +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv.slice(2)) +} diff --git a/scripts/agentConversationRecord.test.ts b/scripts/agentConversationRecord.test.ts new file mode 100644 index 00000000000..9ffcd6527f2 --- /dev/null +++ b/scripts/agentConversationRecord.test.ts @@ -0,0 +1,835 @@ +import { describe, expect, it } from 'vitest' +import type { z } from 'zod' + +import { exportAgentConversation } from './agentConversationCapture' +import type { + AssembleInput, + NormalizedRows, + RawCapture, + RecordedFrame, + RecordedTurn, + SeedFixture +} from './agentConversationRecord' +import { assembleCapture, zRowsDump } from './agentConversationRecord' + +const THREAD = 'thread-1' +const MESSAGE = 'message-1' +const SEED_MESSAGE = 'seed-message-1' +const WORKFLOW = '6f1c2c1e-3b1c-4c88-9d9c-0d6e9b8e1a01' + +const turnFrame = ( + type: string, + data: Record, + at_ms: number +): RecordedFrame => ({ + type, + data: { thread_id: THREAD, message_id: MESSAGE, ...data }, + at_ms +}) + +const frames = (): RecordedFrame[] => [ + turnFrame('agent_thinking', { delta: 'switching' }, 1_700_000_000_000), + turnFrame( + 'agent_active_tab', + { workflow_id: WORKFLOW, name: 'Text to image' }, + 1_700_000_000_100 + ), + turnFrame( + 'agent_tool_call', + { tool_call_id: 'tool-1', tool_name: 'apply_ops', status: 'success' }, + 1_700_000_000_200 + ), + turnFrame('agent_message_done', {}, 1_700_000_000_300) +] + +const seed = (): SeedFixture => ({ + workflow: { + id: WORKFLOW, + name: 'Text to image', + catalog: { types: { KSampler: { widget_order: ['steps'] } } }, + seed: { + nodes: [ + { id: 3, type: 'CheckpointLoaderSimple' }, + { id: 4, type: 'KSampler' } + ], + links: [] + } + } +}) + +const addNodeOp = { + op: 'add_node', + op_id: 'op-1', + node_id: 10, + class_type: 'KSampler' +} + +const parent = ( + overrides: Partial = {} +): RowsInput['parents'][number] => ({ + id: 'parent-1', + tool_call_id: 'tool-1', + tool_name: 'apply_ops', + status: 'ok', + workflow_id: WORKFLOW, + result: { ok: true, data: { ops: [addNodeOp] } }, + children: [{ op_id: 'op-1', status: 'ok' }], + ...overrides +}) + +const raw = (overrides: Partial = {}): RawCapture => ({ + case_id: 'agent-rec-example', + attempt: 'a1', + base: 'http://127.0.0.1:8086', + frame_source: 'redis SUBSCRIBE channel:ws::u:', + channel: 'channel:ws:w-secret:u:u-secret', + seed_sha256: 'a'.repeat(64), + seed_name: 'Text to image', + seed_node_ids: [3, 4], + saw_stream: true, + stream_closed: false, + seed_turn: { + status: 202, + body: { thread_id: 'seed-thread', message_id: SEED_MESSAGE } + }, + seed_workflow_id: WORKFLOW, + turns: [ + { + prompt: "Switch to the tab 'Text to image', then add a sampler.", + accepted: { + status: 202, + body: { + thread_id: THREAD, + message_id: MESSAGE, + workflow_id: 'blank-wf' + } + }, + saw_done: true + } + ], + timed_out: false, + frames: frames(), + rows_artifacts: ['rows.json'], + retrieval: { kind: 'postgres-json' }, + error: null, + ...overrides +}) + +type RowsInput = z.input + +const rows = (overrides: Partial = {}): NormalizedRows => { + const { parents, draft } = zRowsDump.parse({ + source: 'postgres', + parents: [parent()], + draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] }, + ...overrides + }) + return { + parents, + draft, + retrieval: { kind: 'postgres-json' }, + path: '/tmp/agent-rec-example.a1.rows.json', + sha256: 'b'.repeat(64) + } +} + +const input = ( + overrides: Partial> & { + rows?: NormalizedRows | NormalizedRows[] + } = {} +): AssembleInput => { + const { rows: given, ...rest } = overrides + return { + raw: raw(), + seed: { json: seed(), path: 'seed.json', sha256: 'a'.repeat(64) }, + provenance: { + cloudSha: '9dc1da7', + model: 'claude-opus-5', + exportedAt: '2026-09-03T01:00:00.000Z' + }, + rawSha256: 'c'.repeat(64), + rawPath: '/tmp/agent-rec-example.a1.raw.json', + ...rest, + rows: given === undefined ? [rows()] : [given].flat() + } +} + +describe('assembleCapture', () => { + it('keeps the turn frames with their receipt times and emits the applied ops', () => { + const { capture, receipt } = assembleCapture(input()) + + expect(capture.workflow.id).toBe(WORKFLOW) + expect(capture.schema_version).toBe('agent-backend-capture.v2') + expect(capture.capture).toMatchObject({ + backend: 'Comfy-Org/cloud', + thread_id: THREAD + }) + expect(capture.turns).toHaveLength(1) + expect(capture.turns[0].message_id).toBe(MESSAGE) + expect(capture.turns[0].frames.map((frame) => frame.type)).toEqual([ + 'agent_thinking', + 'agent_active_tab', + 'agent_tool_call', + 'agent_message_done' + ]) + expect(capture.turns[0].frames.map((frame) => frame.at_ms)).toEqual([ + 1_700_000_000_000, 1_700_000_000_100, 1_700_000_000_200, 1_700_000_000_300 + ]) + expect(capture.turns[0].tool_calls).toEqual([ + { + tool_call_id: 'tool-1', + result: { ok: true, data: { ops: [addNodeOp] } }, + applied_op_ids: ['op-1'] + } + ]) + expect(receipt).toMatchObject({ + added_nodes: 1, + deleted_nodes: 0, + unexplained_draft_nodes: 0 + }) + expect(receipt.turns).toEqual([ + { + message_id: MESSAGE, + frames_kept: 4, + parents: 1, + mutating_parents: 1, + child_statuses: { ok: 1 }, + rows: '/tmp/agent-rec-example.a1.rows.json', + rows_sha256: 'b'.repeat(64) + } + ]) + }) + + it('produces a capture the exporter turns into a recorded conversation', () => { + const { capture } = assembleCapture(input()) + const conversation = exportAgentConversation(capture) + + expect(conversation.source.response_side).toBe('recorded') + expect(conversation.turns).toHaveLength(1) + expect(conversation.turns[0].message_id).toBe(MESSAGE) + expect( + conversation.turns[0].response.filter( + (entry) => entry.kind === 'graph_ops' + ) + ).toHaveLength(1) + }) + + it('records the note without the concrete redis channel', () => { + const { capture, receipt } = assembleCapture(input()) + + expect(capture.source.note).toContain('channel:ws::u:') + expect(capture.source.note).not.toContain('w-secret') + expect(receipt.channel).toContain('w-secret') + }) + + it('accepts a delete_node whose node id is zero', () => { + const { receipt } = assembleCapture( + input({ + seed: { + json: { + workflow: { + ...seed().workflow, + seed: { nodes: [{ id: 0, type: 'KSampler' }], links: [] } + } + }, + path: 'seed.json', + sha256: 'a'.repeat(64) + }, + raw: raw({ seed_node_ids: [0] }), + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-1', node_id: 0 }] + } + } + }) + ], + draft: { nodes: [], links: [] } + }) + }) + ) + + expect(receipt.deleted_nodes).toBe(1) + }) + + it('accepts a batch the document rejected as zero applied ops', () => { + const { receipt } = assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + status: 'error', + children: [{ op_id: 'op-1', status: 'error' }] + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }], links: [] } + }) + }) + ) + + expect(receipt.turns[0].mutating_parents).toBe(0) + expect(receipt.turns[0].child_statuses).toEqual({ error: 1 }) + }) + + it('accepts a read tool that echoes ops without writing child rows', () => { + const { capture, receipt } = assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + tool_name: 'plan_path', + children: [], + result: { ok: true, data: { ops: [addNodeOp] } } + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }], links: [] } + }) + }) + ) + + expect(capture.turns[0].tool_calls[0].applied_op_ids).toEqual([]) + expect(receipt.turns[0].mutating_parents).toBe(0) + }) + + it('drops the ambient heartbeat and a non-replay turn frame into buckets', () => { + const { receipt } = assembleCapture( + input({ + raw: raw({ + frames: [ + { + type: 'draft_version', + data: { workflow_id: WORKFLOW, version: 3 } + }, + turnFrame('draft_patch', { seq: 1 }, 1_700_000_000_050), + { + type: 'agent_message_delta', + data: { + thread_id: 'seed-thread', + message_id: SEED_MESSAGE, + delta: 'ack' + } + }, + ...frames() + ] + }) + }) + ) + + expect(receipt.frames_dropped).toEqual({ + 'type:draft_version': 1, + 'type:draft_patch': 1, + seed_turn: 1 + }) + expect(receipt.turns[0].frames_kept).toBe(4) + }) + + it('refuses a mutating call whose mutation never reached the doc host', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ children: [] })] }) }) + ) + ).toThrow('reports a document mutation but has NO audit child rows') + }) + + it('refuses an applied add_node whose class is outside the seed catalog', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ ...addNodeOp, class_type: 'LatentUpscaleBy' }] + } + } + }) + ] + }) + }) + ) + ).toThrow('are not in the seed catalog') + }) + + it('refuses an applied parent row that names another workflow', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ workflow_id: null })] }) }) + ) + ).toThrow('not the seeded workflow') + }) + + it('refuses an unparseable socket payload', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: [ + { type: '__raw__', data: { payload: 'not json' } }, + ...frames() + ] + }) + }) + ) + ).toThrow('unparseable socket payload') + }) + + it('refuses rows whose tool calls disagree with the recorded frames', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent(), + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + children: [], + result: { ok: true } + }) + ] + }) + }) + ) + ).toThrow('disagree; the rows are not this turn') + }) + + it('refuses an applied op that its parent result never echoed', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ children: [{ op_id: 'op-missing', status: 'ok' }] }) + ] + }) + }) + ) + ).toThrow('are not echoed in its result') + }) + + it('refuses applied ops whose parent recorded no result at all', () => { + expect(() => + assembleCapture( + input({ rows: rows({ parents: [parent({ result: null })] }) }) + ) + ).toThrow('has applied ops but a NULL result') + }) + + it('refuses an echoed op kind outside the exporter frozen set', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [addNodeOp, { op: 'reset_doc', op_id: 'op-2' }] + } + } + }) + ] + }) + }) + ) + ).toThrow('outside the exporter frozen set') + }) + + it('refuses a non-object echoed op entry', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { ok: true, data: { ops: [addNodeOp, 'bogus'] } } + }) + ] + }) + }) + ) + ).toThrow('non-object op entry') + }) + + it('refuses an applied delete_node without a node id', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { ops: [{ op: 'delete_node', op_id: 'op-1' }] } + } + }) + ] + }) + }) + ) + ).toThrow('applied delete_node without node_id') + }) + + it('refuses an attempt label that would collide across attempts', () => { + expect(() => assembleCapture(input({ raw: raw({ attempt: '' }) }))).toThrow( + 'is not [A-Za-z0-9_-]+' + ) + }) + + it('refuses a seed the driver did not use', () => { + expect(() => + assembleCapture(input({ raw: raw({ seed_node_ids: [3, 4, 99] }) })) + ).toThrow('but the seed fixture given here has') + }) + + it('refuses a turn with no active tab frame', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: frames().filter( + (frame) => frame.type !== 'agent_active_tab' + ) + }) + }) + ) + ).toThrow('no agent_active_tab frame in this turn') + }) + + it('refuses a turn whose last frame is not the done frame', () => { + expect(() => + assembleCapture(input({ raw: raw({ frames: frames().slice(0, -1) }) })) + ).toThrow('not agent_message_done') + }) + + it('refuses an agent frame type the replay cannot validate', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + frames: [turnFrame('agent_ask', {}, 1_700_000_000_050), ...frames()] + }) + }) + ) + ).toThrow('outside the replay union') + }) + + it('refuses a draft that lost a seed node nothing deleted', () => { + expect(() => + assembleCapture( + input({ rows: rows({ draft: { nodes: [{ id: 3 }], links: [] } }) }) + ) + ).toThrow('lacks seed node ids') + }) + + it('refuses a draft that still holds a deleted node', () => { + expect(() => + assembleCapture( + input({ + rows: rows({ + parents: [ + parent({ + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-1', node_id: 3 }] + } + } + }) + ] + }) + }) + ) + ).toThrow('still holds node ids') + }) + + it('refuses a turn the backend never accepted', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ + turns: [ + { ...raw().turns[0], accepted: { status: 500, body: null } } + ] + }) + }) + ) + ).toThrow('turn 1 not accepted') + }) + + it('refuses a recording whose frame stream never opened', () => { + expect(() => + assembleCapture(input({ raw: raw({ saw_stream: false }) })) + ).toThrow('frame stream never opened') + }) + + it('refuses a recording with no terminal done frame observed', () => { + expect(() => + assembleCapture( + input({ raw: raw({ turns: [{ ...raw().turns[0], saw_done: false }] }) }) + ) + ).toThrow('never arrived') + }) + + it('refuses a seeded workflow the active tab never named', () => { + expect(() => + assembleCapture( + input({ + raw: raw({ seed_workflow_id: '11111111-2222-4333-8444-555555555555' }) + }) + ) + ).toThrow('is not the seeded workflow') + }) +}) + +const MESSAGE_2 = 'message-2' + +const connectOp = { op: 'connect', op_id: 'op-2', from: 10, to: 3 } + +const secondTurnFrames = (): RecordedFrame[] => [ + { + type: 'agent_thinking', + data: { thread_id: THREAD, message_id: MESSAGE_2, delta: 'wiring' }, + at_ms: 1_700_000_001_000 + }, + { + type: 'agent_tool_call', + data: { + thread_id: THREAD, + message_id: MESSAGE_2, + tool_call_id: 'tool-2', + tool_name: 'connect', + status: 'success' + }, + at_ms: 1_700_000_001_100 + }, + { + type: 'agent_message_done', + data: { thread_id: THREAD, message_id: MESSAGE_2 }, + at_ms: 1_700_000_001_200 + } +] + +const secondTurn = (overrides: Partial = {}): RecordedTurn => ({ + prompt: 'Now connect it to the sampler.', + accepted: { + status: 202, + body: { thread_id: THREAD, message_id: MESSAGE_2 } + }, + saw_done: true, + ...overrides +}) + +const secondRows = (): NormalizedRows => + rows({ + parents: [ + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + tool_name: 'connect', + result: { ok: true, data: { ops: [connectOp] } }, + children: [{ op_id: 'op-2', status: 'ok' }] + }) + ] + }) + +const twoTurns = (overrides: Partial = {}): AssembleInput => + input({ + raw: raw({ + turns: [raw().turns[0], secondTurn()], + frames: [...frames(), ...secondTurnFrames()] + }), + rows: [rows(), secondRows()], + ...overrides + }) + +describe('assembleCapture across turns', () => { + it('buckets frames by the message id of the turn that owns them', () => { + const { capture, receipt } = assembleCapture(twoTurns()) + + expect(capture.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect( + capture.turns[0].frames.every( + (frame) => frame.data.message_id === MESSAGE + ) + ).toBe(true) + expect(capture.turns[1].frames.map((frame) => frame.type)).toEqual([ + 'agent_thinking', + 'agent_tool_call', + 'agent_message_done' + ]) + expect(capture.turns[1].request.content).toBe( + 'Now connect it to the sampler.' + ) + expect(receipt.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect(receipt.turns.map((turn) => turn.frames_kept)).toEqual([4, 3]) + }) + + it('exports one recorded conversation turn per recorded turn', () => { + const conversation = exportAgentConversation( + assembleCapture(twoTurns()).capture + ) + + expect(conversation.turns.map((turn) => turn.message_id)).toEqual([ + MESSAGE, + MESSAGE_2 + ]) + expect( + conversation.turns.map( + (turn) => + turn.response.filter((entry) => entry.kind === 'graph_ops').length + ) + ).toEqual([1, 1]) + }) + + it('requires the active tab frame only on the opening turn', () => { + const { capture } = assembleCapture(twoTurns()) + const tabFrames = (index: number) => + capture.turns[index].frames.filter( + (frame) => frame.type === 'agent_active_tab' + ) + + expect(tabFrames(0)).toHaveLength(1) + expect(tabFrames(1)).toHaveLength(0) + }) + + it('checks the draft once, against every applied op of the thread', () => { + expect(() => + assembleCapture( + twoTurns({ + rows: [ + rows(), + rows({ + parents: [ + parent({ + id: 'parent-2', + tool_call_id: 'tool-2', + tool_name: 'delete_node', + result: { + ok: true, + data: { + ops: [{ op: 'delete_node', op_id: 'op-2', node_id: 4 }] + } + }, + children: [{ op_id: 'op-2', status: 'ok' }] + }) + ], + draft: { nodes: [{ id: 3 }, { id: 4 }, { id: 10 }], links: [] } + }) + ] + }) + ) + ).toThrow('still holds node ids 4') + }) + + it('refuses a turn that landed on another thread', () => { + expect(() => + assembleCapture( + twoTurns({ + raw: raw({ + turns: [ + raw().turns[0], + secondTurn({ + accepted: { + status: 202, + body: { thread_id: 'thread-2', message_id: MESSAGE_2 } + } + }) + ], + frames: [...frames(), ...secondTurnFrames()] + }) + }) + ) + ).toThrow('turn 2 landed on thread thread-2') + }) + + it('refuses a turn whose audit rows were never read', () => { + expect(() => assembleCapture(twoTurns({ rows: [rows()] }))).toThrow( + 'recorded 2 turn(s) but read 1 audit row set(s)' + ) + }) + + it('names the turn whose last kept frame is not the done frame', () => { + expect(() => + assembleCapture( + twoTurns({ + raw: raw({ + turns: [raw().turns[0], secondTurn()], + frames: [...frames(), ...secondTurnFrames().slice(0, 2)] + }) + }) + ) + ).toThrow('last kept frame of turn 2 is agent_tool_call') + }) +}) + +describe('zRowsDump', () => { + const dump = (result: unknown, overrides: Record = {}) => ({ + source: 'postgres', + parents: [ + { + id: 'parent-1', + tool_call_id: 'tool-1', + tool_name: 'apply_ops', + status: 'ok', + workflow_id: WORKFLOW, + result, + children: [], + ...overrides + } + ], + draft: null + }) + + it('decodes a result psql handed back as a JSON string', () => { + const parsed = zRowsDump.parse( + dump(JSON.stringify({ ok: true, data: { ops: [addNodeOp] } })) + ) + + expect(parsed.parents[0].result).toEqual({ + ok: true, + data: { ops: [addNodeOp] } + }) + }) + + it('decodes the draft column and normalises its node ids to strings', () => { + const parsed = zRowsDump.parse({ + ...dump(null), + draft: JSON.stringify({ nodes: [{ id: 3 }, { id: '4' }], links: [] }) + }) + + expect(parsed.draft?.nodes.map((node) => node.id)).toEqual(['3', '4']) + }) + + it('refuses a result that is not a JSON object', () => { + expect(() => zRowsDump.parse(dump([1, 2]))).toThrow() + }) + + it('refuses a result string that is not JSON', () => { + expect(() => zRowsDump.parse(dump('{'))).toThrow('is not JSON') + }) + + it('refuses a parent row without a tool call id', () => { + expect(() => + zRowsDump.parse(dump({ ok: true }, { tool_call_id: null })) + ).toThrow() + }) + + it('refuses a dump that did not come from postgres', () => { + expect(() => + zRowsDump.parse({ ...dump({ ok: true }), source: 'sqlite' }) + ).toThrow() + }) +}) diff --git a/scripts/agentConversationRecord.ts b/scripts/agentConversationRecord.ts new file mode 100644 index 00000000000..bfa2a0a6716 --- /dev/null +++ b/scripts/agentConversationRecord.ts @@ -0,0 +1,1018 @@ +#!/usr/bin/env tsx + +import { execFileSync, spawn } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { basename, dirname, join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { FROZEN_OPS } from '@comfyorg/comfy-multi-player' +import type { WorkflowJSON } from '@comfyorg/comfy-multi-player' +import { z } from 'zod' + +import type { zRecordedWsEvent } from '../browser_tests/fixtures/data/agent/agentConversation' +import { zAgentConversationWorkflow } from '../browser_tests/fixtures/data/agent/agentConversation' +import type { AgentBackendCapture } from './agentConversationCapture' +import { exportAgentConversation } from './agentConversationCapture' + +const USAGE = + 'usage: pnpm exec tsx scripts/agentConversationRecord.ts --prompt "" [--prompt "" ...] --out [--work ]' + +const REPLAYED_FRAMES = + 'agent_thinking agent_tool_call agent_message_delta agent_message_done agent_active_tab'.split( + ' ' + ) + +// boundary.Classify's ActionEdit set minus the tab tools, which move focus +// rather than the document. +const MUTATING_TOOLS = + 'apply_ops add_node connect set_widget delete_node delete_nodes clear_canvas apply_recipe generate_workflow reset_doc open_workflow get_template use_asset_as_input'.split( + ' ' + ) + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +const STACK = + 'NON-standalone local full stack (Postgres + doc host, M2M identity headers)' + +const zJsonObject = z.record(z.string(), z.unknown()) + +// psql hands a json column back as an object, as a string, or as NULL. +const zJsonColumn = z.unknown().transform((value, ctx) => { + if (typeof value !== 'string') return value + try { + return JSON.parse(value) as unknown + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: 'is not JSON' }) + return z.NEVER + } +}) + +const zEnv = z.object({ + AGENT_CLOUD_SHA: z.string().regex(/^[0-9a-f]{7,40}$/), + AGENT_MODEL: z.string().trim().min(1), + AGENT_M2M_SECRET_FILE: z.string().min(1), + AGENT_WORKSPACE_ID: z.string().min(1), + AGENT_USER_ID: z.string().min(1), + AGENT_FULLSTACK_URL: z.string().default('http://127.0.0.1:8086'), + AGENT_REDIS_EXEC: z + .string() + .default('docker exec -i be11470-redis redis-cli'), + AGENT_PG_EXEC: z + .string() + .default('docker exec -i be11470-pg psql -U postgres -d postgres -At -c'), + AGENT_TURN_TIMEOUT: z.coerce.number().positive().default(180_000), + AGENT_ATTEMPT: z.string().default('') +}) + +const zAck = z.object({ + thread_id: z.string().min(1), + message_id: z.string().min(1) +}) + +const zSeedAck = zAck.extend({ workflow_id: z.string().min(1) }) + +export const zSeedFixture = z.object({ + workflow: zAgentConversationWorkflow +}) + +const zParentRow = z.object({ + id: z.coerce.string(), + tool_call_id: z.string().min(1), + tool_name: z.string().nullable(), + status: z.string().nullable(), + workflow_id: z.string().nullable(), + result: zJsonColumn.pipe(zJsonObject.nullable()), + children: z.array( + z.object({ + op_id: z.string().nullable(), + status: z.string().nullable() + }) + ) +}) + +export const zRowsDump = z.object({ + source: z.literal('postgres'), + parents: z.array(zParentRow), + draft: zJsonColumn.pipe( + z + .object({ nodes: z.array(z.object({ id: z.coerce.string() })) }) + .passthrough() + .nullable() + ) +}) + +// The socket carries frames without a data object; the replay union does not. +const zSocketFrame = z + .object({ + type: z.string(), + data: zJsonObject.optional().catch(undefined) + }) + .transform((frame) => ({ type: frame.type, data: frame.data ?? {} })) + +// The exporter's candidate set: data.ops when it is a list, else data.op. +const zOpsCarrier = z.object({ + data: z + .object({ ops: z.array(z.unknown()).optional().catch(undefined) }) + .passthrough() +}) + +export type RecordedFrame = z.infer +export type SeedFixture = z.infer +export type ParentRow = z.infer + +export type NormalizedRows = Omit, 'source'> & { + retrieval: Record + path: string + sha256: string +} + +export interface TurnAck { + status: number + body: unknown +} + +export interface RecordedTurn { + prompt: string + accepted: TurnAck | null + saw_done: boolean +} + +export interface RawCapture { + case_id: string + attempt: string + base: string + frame_source: string + channel: string + seed_sha256: string + seed_name: string + seed_node_ids: Array + saw_stream: boolean + stream_closed: boolean + seed_turn: TurnAck | null + seed_workflow_id: string | null + turns: RecordedTurn[] + timed_out: boolean + frames: RecordedFrame[] + rows_artifacts: string[] + retrieval: Record | null + error: string | null +} + +export interface AssembleInput { + raw: RawCapture + rows: NormalizedRows[] + seed: { json: SeedFixture; path: string; sha256: string } + provenance: { cloudSha: string; model: string; exportedAt: string } + rawSha256: string + rawPath: string +} + +interface DraftCounts { + draft_nodes: number + added_nodes: number + deleted_nodes: number + unexplained_draft_nodes: number +} + +interface TurnIds { + threadId: string + messageId: string +} + +// A refused recording is an expected outcome: main logs it and exits 1. +export class RecordRefusal extends Error {} + +function refuse(reason: string): never { + throw new RecordRefusal(reason) +} + +// A shape mismatch at a boundary is a refusal, named by its json path. +function parseOrRefuse( + schema: S, + value: unknown, + what: string +): z.output { + const parsed = schema.safeParse(value) + if (!parsed.success) + refuse( + `${what}: ${parsed.error.issues + .map((issue) => `${issue.path.join('.') || '(root)'} ${issue.message}`) + .join('; ')}` + ) + return parsed.data +} + +const safeJson = (text: string): unknown => { + try { + return JSON.parse(text) + } catch { + return undefined + } +} + +const list = (values: Iterable): string => + [...values].map(String).sort().join(', ') + +const sameSet = (left: Set, right: Set): boolean => + left.size === right.size && [...left].every((value) => right.has(value)) + +const sha256 = (bytes: Buffer | string): string => + createHash('sha256').update(bytes).digest('hex') + +const sha256OfFile = (path: string): string => sha256(readFileSync(path)) + +const writeJson = (path: string, value: unknown): void => + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`) + +const turnLabel = (index: number): string => `turn ${index + 1}` + +// Everything the recording as a whole must clear before a turn means anything. +function checkRecording( + raw: RawCapture, + seedIds: Set, + rowSets: number +): void { + if (!/^[A-Za-z0-9_-]+$/.test(raw.attempt)) + refuse(`attempt label ${JSON.stringify(raw.attempt)} is not [A-Za-z0-9_-]+`) + const driverIds = new Set(raw.seed_node_ids.map(String)) + if (!sameSet(driverIds, seedIds)) + refuse( + `the driver seeded ${list(driverIds)} but the seed fixture given here has ${list(seedIds)}` + ) + if (!raw.saw_stream) refuse('frame stream never opened') + if (raw.turns.length === 0) refuse('no turns recorded') + if (raw.turns.length !== rowSets) + refuse( + `recorded ${raw.turns.length} turn(s) but read ${rowSets} audit row set(s)` + ) +} + +function turnIds(turn: RecordedTurn, label: string, raw: RawCapture): TurnIds { + if (turn.accepted?.status !== 202) + refuse(`${label} not accepted: ${JSON.stringify(turn.accepted)}`) + const ack = zAck.safeParse(turn.accepted.body) + if (!ack.success) + refuse( + `${label} ack without ids: ${JSON.stringify(turn.accepted.body ?? {})}` + ) + if (!turn.saw_done) + refuse( + `${label}: agent_message_done never arrived (timed_out=${raw.timed_out}, error=${raw.error})` + ) + return { threadId: ack.data.thread_id, messageId: ack.data.message_id } +} + +function keepTurnFrames( + frames: RecordedFrame[], + ids: TurnIds[], + seedTurn: TurnIds | null +): { kept: RecordedFrame[][]; dropped: Record } { + const kept: RecordedFrame[][] = ids.map(() => []) + const dropped: Record = {} + const drop = (bucket: string): void => { + dropped[bucket] = (dropped[bucket] ?? 0) + 1 + } + + for (const frame of frames) { + const { type, data } = frame + const turn = ids.findIndex( + (id) => data.thread_id === id.threadId && data.message_id === id.messageId + ) + // Unparseable payloads belong to no turn; the heartbeat carries no ids. + if (type === '__raw__' || type === 'draft_version') + drop(type === '__raw__' ? 'unparseable' : 'type:draft_version') + else if (turn === -1) + drop( + seedTurn !== null && + data.thread_id === seedTurn.threadId && + data.message_id === seedTurn.messageId + ? 'seed_turn' + : 'foreign' + ) + else if (REPLAYED_FRAMES.includes(type)) kept[turn].push(frame) + else if (type.startsWith('agent_')) + refuse(`frame type ${type} is outside the replay union`) + else drop(`type:${type}`) + } + + if (dropped.unparseable) + refuse(`${dropped.unparseable} unparseable socket payload(s) recorded`) + for (const [index, turn] of kept.entries()) { + const last = turn.at(-1) + if (last?.type !== 'agent_message_done') + refuse( + `last kept frame of ${turnLabel(index)} is ${last?.type}, not agent_message_done` + ) + } + return { kept, dropped } +} + +function activeWorkflowId( + kept: RecordedFrame[], + seededId: string | null +): string { + const tabs = new Set( + kept + .filter((frame) => frame.type === 'agent_active_tab') + .map((frame) => frame.data.workflow_id) + ) + if (tabs.size === 0) refuse('no agent_active_tab frame in this turn') + if (tabs.size > 1) + refuse(`agent_active_tab frames disagree on workflow_id: ${list(tabs)}`) + const [tab] = [...tabs] + const workflowId = z.string().regex(UUID).safeParse(tab) + if (!workflowId.success) + refuse(`active_tab workflow id is not a uuid: ${String(tab)}`) + if (workflowId.data !== seededId) + refuse( + `active_tab workflow ${workflowId.data} is not the seeded workflow ${seededId}` + ) + return workflowId.data +} + +// Every op the exporter will read out of this result, shape-checked once. +function echoedOps(row: ParentRow): Array> { + const carrier = zOpsCarrier.safeParse(row.result ?? {}) + if (!carrier.success) return [] + const { data } = carrier.data + const ops = z + .array(zJsonObject) + .safeParse(data.ops ?? ('op' in data ? [data.op] : [])) + if (!ops.success) refuse(`parent row ${row.id} echoes a non-object op entry`) + const kinds = ops.data.map((op) => String(op.op)) + const offFrozen = new Set( + kinds.filter((kind) => !(FROZEN_OPS as readonly string[]).includes(kind)) + ) + if (offFrozen.size > 0) + refuse( + `parent row ${row.id} echoes op kinds ${list(offFrozen)} outside the exporter frozen set` + ) + return ops.data +} + +// Ops-shaped mutation record: the CRDT-on op echo or the CRDT-off ack summary. +// Bare count keys are excluded because read tools carry them. +function mutationReported(row: ParentRow): boolean { + const carrier = zOpsCarrier.safeParse(row.result ?? {}) + if (!carrier.success) return false + const { data } = carrier.data + return ( + (data.ops?.length ?? 0) > 0 || + zJsonObject.safeParse(data.op).success || + Boolean(data.ops_by_kind ?? data.nodes_added ?? data.nodes_deleted) + ) +} + +function appliedOps( + row: ParentRow, + applied: string[] +): Array> { + const byId = new Map( + echoedOps(row).flatMap((op) => { + const opId = z.string().safeParse(op.op_id) + return opId.success ? [[opId.data, op] as const] : [] + }) + ) + const missing = applied.filter((opId) => !byId.has(opId)) + if (missing.length > 0) + refuse( + `parent row ${row.id} applied op ids ${missing.join(', ')} are not echoed in its result` + ) + const ops = applied.map((opId) => byId.get(opId)!) + // A node id of 0 is a real id, so only a missing one refuses. + if ( + ops.some((op) => op.op === 'delete_node' && (op.node_id ?? null) === null) + ) + refuse(`parent row ${row.id} has an applied delete_node without node_id`) + return ops +} + +function parentToolCall( + row: ParentRow, + workflowId: string +): { + toolCall: AgentBackendCapture['turns'][number]['tool_calls'][number] + appliedOps: Array> +} { + const applied = row.children.flatMap((child) => + child.status === 'ok' && child.op_id ? [child.op_id] : [] + ) + if (row.result === null && applied.length > 0) + refuse(`parent row ${row.id} has applied ops but a NULL result`) + if ( + row.tool_name !== null && + MUTATING_TOOLS.includes(row.tool_name) && + mutationReported(row) && + row.children.length === 0 + ) + refuse( + `parent row ${row.id} (${row.tool_name}) reports a document mutation but has NO audit child rows` + ) + + if (applied.length > 0 && row.workflow_id !== workflowId) + refuse( + `parent row ${row.id} applied ops on ${row.workflow_id}, not the seeded workflow ${workflowId}` + ) + + return { + appliedOps: appliedOps(row, applied), + toolCall: { + tool_call_id: row.tool_call_id, + result: row.result ?? {}, + applied_op_ids: applied + } + } +} + +// The frames and the audit rows must describe the same turn's tool calls. +function checkTurnAgreement( + kept: RecordedFrame[], + rows: ParentRow[], + label: string +): void { + const frameCalls = new Set( + kept + .filter( + (frame) => + frame.type === 'agent_tool_call' && frame.data.status !== 'running' + ) + .flatMap( + (frame) => z.string().safeParse(frame.data.tool_call_id).data ?? [] + ) + ) + const rowCalls = new Set(rows.map((row) => row.tool_call_id)) + if (!sameSet(frameCalls, rowCalls)) + refuse( + `${label}: frames ${list(frameCalls)} and audit parent rows ${list(rowCalls)} disagree; the rows are not this turn` + ) +} + +// The draft is the only witness that the applied ops reached the document. +function checkDraft( + draft: NormalizedRows['draft'], + seedIds: Set, + appliedOps: Array>, + workflowId: string +): DraftCounts { + if (draft === null) + refuse(`no workflow_drafts row for ${workflowId}: the seed did not bind`) + const draftIds = new Set(draft.nodes.map((node) => node.id)) + const nodeIds = (kind: string): Set => + new Set( + appliedOps + .filter((op) => op.op === kind && (op.node_id ?? null) !== null) + .map((op) => String(op.node_id)) + ) + + const deleted = nodeIds('delete_node') + const added = nodeIds('add_node') + const expected = [...seedIds].filter((id) => !deleted.has(id)) + const missing = expected.filter((id) => !draftIds.has(id)) + if (missing.length > 0) + refuse( + `draft for ${workflowId} lacks seed node ids ${list(missing)} that no applied delete_node removed` + ) + const undeleted = [...deleted].filter((id) => draftIds.has(id)) + if (undeleted.length > 0) + refuse( + `draft for ${workflowId} still holds node ids ${list(undeleted)} that applied delete_node ops removed` + ) + + return { + draft_nodes: draftIds.size, + added_nodes: added.size, + deleted_nodes: deleted.size, + unexplained_draft_nodes: [...draftIds].filter( + (id) => !expected.includes(id) && !added.has(id) + ).length + } +} + +// An uncatalogued class is stored opaquely by the applier, so a later +// set_widget on it throws. +function checkAddedClasses( + appliedOps: Array>, + catalog: SeedFixture['workflow']['catalog'] +): void { + const types = new Set(Object.keys(catalog.types)) + const offCatalog = new Set( + appliedOps + .filter((op) => op.op === 'add_node') + .map((op) => String(op.class_type)) + .filter((type) => !types.has(type)) + ) + if (offCatalog.size > 0) + refuse( + `applied add_node classes ${list(offCatalog)} are not in the seed catalog ${list(types)}` + ) +} + +function buildCapture(options: { + input: AssembleInput + threadId: string + workflowId: string + seedMessageId: string | null + turns: AgentBackendCapture['turns'] +}): AgentBackendCapture { + const { input, threadId, workflowId, turns } = options + const { raw, rows, provenance } = input + const { workflow } = input.seed.json + const note = `RECORDED from Comfy-Org/cloud services/agent running ${STACK} at ${raw.base} (frames: ${raw.frame_source}); NOT a production capture. cloud commit ${provenance.cloudSha}; model ${provenance.model}; thread ${threadId}; messages ${turns.map((turn) => turn.message_id).join(', ')}; workflow ${workflowId} (seeded by throwaway turn ${options.seedMessageId}; turn 1 opens on a fresh workflow and switches to it first because the replay subscribes only on an agent_active_tab frame); agent_tool_calls parent rows ${list(rows.flatMap((set) => set.parents.map((row) => row.id)))}; rows ${rows.map((set) => basename(set.path)).join(', ')}; raw capture sha256 ${input.rawSha256}` + + return { + schema_version: 'agent-backend-capture.v2', + source: { + repo: 'Comfy-Org/ComfyUI_frontend', + suite: 'agent', + case_id: raw.case_id, + note + }, + capture: { + backend: 'Comfy-Org/cloud', + thread_id: threadId, + exported_at: provenance.exportedAt + }, + workflow: { + id: workflowId, + name: workflow.name, + catalog: workflow.catalog, + seed: workflow.seed + }, + turns + } +} + +interface TurnReceipt { + message_id: string + frames_kept: number + parents: number + mutating_parents: number + child_statuses: Record + rows: string + rows_sha256: string +} + +function buildReceipt(options: { + input: AssembleInput + threadId: string + workflowId: string + turns: TurnReceipt[] + framesDropped: Record + draft: DraftCounts +}) { + const { input, threadId, workflowId, turns } = options + const { raw, rows, seed, provenance } = input + return { + attempt: raw.attempt, + frame_source: raw.frame_source, + channel: raw.channel, + raw: input.rawPath, + raw_sha256: input.rawSha256, + seed: seed.path, + seed_sha256: seed.sha256, + retrieval: rows[0].retrieval, + thread_id: threadId, + workflow_id: workflowId, + turns, + frames_dropped: options.framesDropped, + ...options.draft, + cloud_sha: provenance.cloudSha, + model: provenance.model + } +} + +function childStatuses(parents: ParentRow[]): Record { + const tally: Record = {} + for (const child of parents.flatMap((row) => row.children)) + tally[child.status ?? 'null'] = (tally[child.status ?? 'null'] ?? 0) + 1 + return tally +} + +// One recorded turn: its own frames, its own rows, and the gates binding them. +function assembleTurn( + turn: RecordedTurn, + ids: TurnIds, + frames: RecordedFrame[], + rows: NormalizedRows, + workflowId: string, + label: string +): { + capture: AgentBackendCapture['turns'][number] + receipt: TurnReceipt + appliedOps: Array> +} { + const calls = rows.parents.map((row) => parentToolCall(row, workflowId)) + checkTurnAgreement(frames, rows.parents, label) + return { + capture: { + message_id: ids.messageId, + request: { content: turn.prompt }, + frames, + tool_calls: calls.map((call) => call.toolCall) + }, + receipt: { + message_id: ids.messageId, + frames_kept: frames.length, + parents: rows.parents.length, + mutating_parents: calls.filter((call) => call.appliedOps.length > 0) + .length, + child_statuses: childStatuses(rows.parents), + rows: rows.path, + rows_sha256: rows.sha256 + }, + appliedOps: calls.flatMap((call) => call.appliedOps) + } +} + +export function assembleCapture(input: AssembleInput) { + const { raw, rows } = input + const { workflow } = input.seed.json + const seedIds = new Set(workflow.seed.nodes.map((node) => String(node.id))) + checkRecording(raw, seedIds, rows.length) + + const seedAck = zAck.safeParse(raw.seed_turn?.body) + const seedTurn = seedAck.success + ? { threadId: seedAck.data.thread_id, messageId: seedAck.data.message_id } + : null + const ids = raw.turns.map((turn, index) => + turnIds(turn, turnLabel(index), raw) + ) + const { threadId } = ids[0] + const strayed = ids.findIndex((id) => id.threadId !== threadId) + if (strayed > 0) + refuse( + `${turnLabel(strayed)} landed on thread ${ids[strayed].threadId}, not ${threadId}` + ) + + const { kept, dropped } = keepTurnFrames(raw.frames, ids, seedTurn) + // The replay subscribes once, so only the opening turn must switch tabs. + const workflowId = activeWorkflowId(kept[0], raw.seed_workflow_id) + const turns = raw.turns.map((turn, index) => + assembleTurn( + turn, + ids[index], + kept[index], + rows[index], + workflowId, + turnLabel(index) + ) + ) + + const appliedOps = turns.flatMap((turn) => turn.appliedOps) + const draft = checkDraft(rows.at(-1)!.draft, seedIds, appliedOps, workflowId) + checkAddedClasses(appliedOps, workflow.catalog) + + return { + capture: buildCapture({ + input, + threadId, + workflowId, + seedMessageId: seedTurn?.messageId ?? null, + turns: turns.map((turn) => turn.capture) + }), + receipt: buildReceipt({ + input, + threadId, + workflowId, + turns: turns.map((turn) => turn.receipt), + framesDropped: dropped, + draft + }) + } +} + +function readRows( + exec: string[], + path: string, + ids: TurnIds & { workflowId: string } +): NormalizedRows { + const quote = (value: string): string => `'${value.replace(/'/g, "''")}'` + const sql = `select json_build_object('source', 'postgres', 'parents', coalesce((select json_agg(row_to_json(r) order by r.started_at, r.id) from (select parent.id, parent.tool_call_id, parent.tool_name, parent.status, parent.result, parent.started_at, parent.workflow_id, coalesce(json_agg(json_build_object('op_id', child.op_id, 'status', child.status, 'op_index', child.op_index) order by child.op_index) filter (where child.id is not null), '[]'::json) as children from agent_tool_calls as parent left join agent_tool_calls as child on child.parent_call_id = parent.id where parent.thread_id = ${quote(ids.threadId)} and parent.message_id = ${quote(ids.messageId)} and parent.parent_call_id is null group by parent.id) r), '[]'::json), 'draft', (select content from workflow_drafts where workflow_id = ${quote(ids.workflowId)} limit 1))` + const dump: unknown = JSON.parse( + execFileSync(exec[0], [...exec.slice(1), sql], { encoding: 'utf8' }).trim() + ) + writeJson(path, dump) + const { parents, draft } = parseOrRefuse(zRowsDump, dump, 'rows dump') + return { + parents, + draft, + retrieval: { kind: 'postgres-json', sql }, + path, + sha256: sha256OfFile(path) + } +} + +function openStream( + raw: RawCapture, + redisExec: string[], + onFrame: (frame: RecordedFrame) => void +): () => void { + const push = (payload: string): void => { + const frame = zSocketFrame.safeParse(safeJson(payload)) + onFrame(frame.success ? frame.data : { type: '__raw__', data: { payload } }) + } + + const child = spawn( + redisExec[0], + [...redisExec.slice(1), 'SUBSCRIBE', raw.channel], + { stdio: ['ignore', 'pipe', 'inherit'] } + ) + child.stdout.setEncoding('utf8') + child.on('error', (error) => { + raw.error ??= `frame source failed to start: ${error.message}` + raw.stream_closed = true + }) + child.on('exit', () => (raw.stream_closed = true)) + // redis-cli off a tty prints subscribe/message envelopes one line at a time. + let buffered = '' + let pending = 0 + child.stdout.on('data', (chunk: string) => { + buffered += chunk + for (const line of buffered.split('\n').slice(0, -1)) { + if (line === 'subscribe') raw.saw_stream = true + else if (line === 'message') pending = 2 + else if (pending === 2) pending = 1 + else if (pending === 1) { + pending = 0 + push(line) + } + } + buffered = buffered.slice(buffered.lastIndexOf('\n') + 1) + }) + return () => child.kill() +} + +const sleep = (ms: number): Promise => + new Promise((done) => setTimeout(done, ms)) + +async function recordTurns( + raw: RawCapture, + options: { + headers: Record + redisExec: string[] + timeoutMs: number + seed: WorkflowJSON + prompts: string[] + } +): Promise { + const stop = openStream(raw, options.redisExec, (frame) => + raw.frames.push({ ...frame, at_ms: Date.now() }) + ) + + const postTurn = async (thread: string, turn: unknown): Promise => { + const response = await fetch( + `${raw.base}/agent/threads/${thread}/messages`, + { + method: 'POST', + headers: options.headers, + body: JSON.stringify(turn), + signal: AbortSignal.timeout(30_000) + } + ) + const payload = await response.text() + const body = zJsonObject.safeParse(safeJson(payload)) + if (response.status === 202 && !body.success) + refuse(`202 with a non-JSON body: ${payload.slice(0, 200)}`) + return { status: response.status, body: body.data ?? null } + } + + const done = (messageId: string): boolean => + raw.frames.some( + (frame) => + frame.type === 'agent_message_done' && + frame.data.message_id === messageId + ) + + const waitDone = async (messageId: string, label: string): Promise => { + const started = Date.now() + while ( + !done(messageId) && + !raw.stream_closed && + Date.now() - started < options.timeoutMs + ) + await sleep(250) + await sleep(1_500) + if (done(messageId)) return + raw.timed_out = !raw.stream_closed + refuse( + raw.stream_closed + ? `frame stream closed before the ${label} agent_message_done` + : `no agent_message_done for the ${label} within ${options.timeoutMs}ms` + ) + } + + try { + const opened = Date.now() + while ( + !raw.saw_stream && + !raw.stream_closed && + Date.now() - opened < 10_000 + ) + await sleep(100) + if (!raw.saw_stream) + refuse(`frame source did not open: ${raw.frame_source}`) + + // Turn A mints and seeds the workflow; it is thrown away, not recorded. + const seedTurn = await postTurn('new', { + content: + 'Hello. Please do nothing to the workflow yet; just acknowledge.', + draft: { content: options.seed } + }) + raw.seed_turn = seedTurn + const seedAck = zSeedAck.safeParse(seedTurn.body) + if (seedTurn.status !== 202 || !seedAck.success) + refuse(`seed turn not accepted: ${JSON.stringify(seedTurn)}`) + raw.seed_workflow_id = seedAck.data.workflow_id + await waitDone(seedAck.data.message_id, 'seed turn') + + // Turn 1 opens on a fresh workflow so switch_tab publishes + // agent_active_tab; the rest continue that thread the way the panel does. + let thread = 'new' + let opening: string | null = null + for (const [index, prompt] of options.prompts.entries()) { + const posted = await postTurn(thread, { + content: prompt, + open_tabs: [ + { workflow_id: seedAck.data.workflow_id, name: raw.seed_name } + ] + }) + const turn: RecordedTurn = { + prompt, + accepted: posted, + saw_done: false + } + raw.turns.push(turn) + const ack = zAck.safeParse(posted.body) + if (posted.status !== 202 || !ack.success) + refuse(`${turnLabel(index)} not accepted: ${JSON.stringify(posted)}`) + thread = ack.data.thread_id + opening ??= ack.data.message_id + await waitDone(ack.data.message_id, turnLabel(index)) + turn.saw_done = true + } + + if ( + !raw.frames.some( + (frame) => + frame.type === 'agent_active_tab' && + frame.data.message_id === opening && + frame.data.workflow_id === seedAck.data.workflow_id + ) + ) + refuse( + `no agent_active_tab for ${seedAck.data.workflow_id}: the agent never switched tabs` + ) + } finally { + stop() + } +} + +async function main(argv: string[]): Promise { + const positional: string[] = [] + const flags: Record = {} + const prompts: string[] = [] + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + if (arg === '--prompt') prompts.push(argv[(index += 1)] ?? '') + else if (arg === '--out' || arg === '--work') + flags[arg] = argv[(index += 1)] ?? '' + else positional.push(arg) + } + const [caseId, seedPath] = positional + const outPath = flags['--out'] + if (!caseId || !seedPath || !outPath || prompts.length === 0) + throw new Error(USAGE) + if (prompts.some((turn) => !turn.trim())) throw new Error(USAGE) + + const env = parseOrRefuse( + zEnv, + { + ...process.env, + AGENT_WORKSPACE_ID: + process.env.AGENT_WORKSPACE_ID ?? process.env.REC_WORKSPACE_ID, + AGENT_USER_ID: process.env.AGENT_USER_ID ?? process.env.REC_USER_ID + }, + 'environment' + ) + const attempt = ( + env.AGENT_ATTEMPT.trim() || new Date().toISOString().replace(/[:.]/g, '-') + ).replace(/[^A-Za-z0-9_-]/g, '-') + + const seed = parseOrRefuse( + zSeedFixture, + JSON.parse(readFileSync(seedPath, 'utf8')), + `seed fixture ${seedPath}` + ) + // conversations/ is scanned for replay cases, so sidecars go one level down. + const workDir = flags['--work'] || join(dirname(outPath), 'recordings') + mkdirSync(workDir, { recursive: true }) + mkdirSync(dirname(resolve(outPath)), { recursive: true }) + const sidecar = (suffix: string): string => + join(workDir, `${caseId}.${attempt}.${suffix}`) + const rawPath = sidecar('raw.json') + + const raw: RawCapture = { + case_id: caseId, + attempt, + base: env.AGENT_FULLSTACK_URL.replace(/\/$/, ''), + // The note carries this shape; the concrete channel stays in the receipt. + frame_source: 'redis SUBSCRIBE channel:ws::u:', + channel: `channel:ws:${env.AGENT_WORKSPACE_ID}:u:${env.AGENT_USER_ID}`, + seed_sha256: sha256OfFile(seedPath), + seed_name: seed.workflow.name, + seed_node_ids: seed.workflow.seed.nodes.map((node) => node.id), + saw_stream: false, + stream_closed: false, + seed_turn: null, + seed_workflow_id: null, + turns: [], + timed_out: false, + frames: [], + rows_artifacts: [], + retrieval: null, + error: null + } + + try { + await recordTurns(raw, { + headers: { + 'content-type': 'application/json', + 'X-Comfy-M2M-Secret': readFileSync( + env.AGENT_M2M_SECRET_FILE, + 'utf8' + ).trim(), + 'X-Comfy-Workspace': env.AGENT_WORKSPACE_ID, + 'X-Comfy-User': env.AGENT_USER_ID + }, + redisExec: env.AGENT_REDIS_EXEC.split(' '), + timeoutMs: env.AGENT_TURN_TIMEOUT, + seed: seed.workflow.seed, + prompts + }) + + // One row set per turn; the last one also carries the final draft. + const rows = raw.turns.map((turn, index) => { + const ids = zAck.parse(turn.accepted?.body) + return readRows( + env.AGENT_PG_EXEC.split(' '), + sidecar(`rows.${index + 1}.json`), + { + threadId: ids.thread_id, + messageId: ids.message_id, + workflowId: String(raw.seed_workflow_id) + } + ) + }) + raw.rows_artifacts = rows.map((set) => set.path) + raw.retrieval = rows[0].retrieval + writeJson(rawPath, raw) + + const { capture, receipt } = assembleCapture({ + raw, + rows, + seed: { json: seed, path: seedPath, sha256: raw.seed_sha256 }, + provenance: { + cloudSha: env.AGENT_CLOUD_SHA, + model: env.AGENT_MODEL, + exportedAt: new Date().toISOString() + }, + rawSha256: sha256OfFile(rawPath), + rawPath + }) + writeJson(sidecar('capture.json'), capture) + writeJson(outPath, exportAgentConversation(capture)) + writeJson(sidecar('receipt.json'), receipt) + const perTurn = receipt.turns + .map( + (turn, index) => + `t${index + 1}[frames=${turn.frames_kept} parents=${turn.parents} mutating=${turn.mutating_parents}]` + ) + .join(' ') + process.stdout.write( + `recorded ${outPath} sha256=${sha256OfFile(outPath)} attempt=${attempt} turns=${receipt.turns.length} ${perTurn} dropped=${JSON.stringify(receipt.frames_dropped)} workflow=${receipt.workflow_id}\nsidecars in ${workDir}\n` + ) + } catch (error) { + raw.error = error instanceof Error ? error.message : String(error) + writeJson(rawPath, raw) + writeFileSync( + join(workDir, `${caseId}.refused.jsonl`), + `${JSON.stringify({ at: new Date().toISOString(), attempt, refused: raw.error })}\n`, + { flag: 'a' } + ) + throw error + } +} + +if ( + process.argv[1] !== undefined && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main(process.argv.slice(2)).catch((error: unknown) => { + const refused = error instanceof RecordRefusal + process.stderr.write( + `record: ${refused ? 'REFUSED' : 'FAILED'}: ${error instanceof Error ? error.message : String(error)}\n` + ) + process.exitCode = 1 + }) +} diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index 090f81832ad..5da50c3476d 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -2400,6 +2400,8 @@ describe('AgentPanelRoot workflow binding', () => { ) const minted = hostStores.workflow.tabs.get('workflows/Video test.json') expect(minted?.filename).toBe('Video test') + // Blank, not the default template: the follower keeps template links it does not displace. + expect(minted?.activeState).toMatchObject({ nodes: [], links: [] }) // The host minted the doc server-side; the follower fills the canvas. // Nothing loads, saves, or adopts here. expect(workflowService.saveWorkflowAs).not.toHaveBeenCalled() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 6dd86bb51ba..e06580d8098 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -47,6 +47,7 @@ import { ACTOR_CONFIG } from '@/renderer/core/layout/constants' import { LayoutSource } from '@/renderer/core/layout/types' import { api } from '@/scripts/api' import { app } from '@/scripts/app' +import { blankGraph } from '@/scripts/defaultGraph' import { useAgentNodeSelectionStore } from '@/stores/agentNodeSelectionStore' import { useExecutionErrorStore } from '@/stores/executionErrorStore' import { useWorkflowTabActivityStore } from '@/stores/workflowTabActivityStore' @@ -538,9 +539,7 @@ async function onAgentActiveTab( }) return } - // A new agent workflow opens as an EMPTY tab: the host minted its doc - // server-side (seed-at-bind), and the follower fills the canvas through - // the ordinary subscribe catch-up - no snapshot fetch, no draft apply. + // The tab starts blank: the host seeds the doc at bind and the default template's node ids collide with agent docs. const creatingStartedAt = Date.now() tabActivity.setCreating(true) const remainingCreatingTime = @@ -548,7 +547,10 @@ async function onAgentActiveTab( if (remainingCreatingTime > 0) await new Promise((resolve) => setTimeout(resolve, remainingCreatingTime)) if (stale()) return - const tab = workflowStore.createTemporary(agentTabFilename(data.name)) + const tab = workflowStore.createTemporary( + agentTabFilename(data.name), + blankGraph + ) tabActivity.setCreating(false) await workflowService.openWorkflow(tab) if (stale()) {