From 8a9e103f7d4193f430df85b74658550b935a38d1 Mon Sep 17 00:00:00 2001 From: kishore Date: Thu, 10 Sep 2026 16:59:03 -0700 Subject: [PATCH 01/64] fix(agent): let the canvas track a standalone agent's edits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A local agent could edit a workflow, report it accurately, and leave the canvas untouched. Three independent defects, any one of which alone breaks it. 1. The follower's transport is ComfyUI's same-origin socket, which carries agent document frames only because INGEST relays them there. A standalone agent has no ingest and that socket carries nothing from it, so the follower listened forever to a wire that could never deliver. 2. Nothing ever named the workflow. The agent fans document frames out on a workflow-scoped channel and joins that channel only for the ids in the connection's query string, so even the right socket would have received chat and nothing else. 3. The panel seeded the draft from the canvas on every message with NO version, which the backend documents as "authoritative" and applies unconditionally. Combined with (1) and (2) that is silent data loss: the canvas never caught up, so each message overwrote whatever the agent had just built, and the agent's next read honestly reported the old graph — which is how this looked like the agent lying about its own work. When a document is live it IS the source of truth and the draft is only its projection, so the fix is to send nothing rather than to send it versioned. The new standalone transport keeps the frame vocabulary DocFrameClient already speaks and swaps only the addressing: `doc_subscribe` becomes the agent's resync call whose answer is re-emitted as the frames the client expects, `doc_ops` becomes its ops call, and inbound frames come off the agent's own socket. IT OWNS A SEPARATE SOCKET FROM THE CHAT STREAM, and that is not incidental. A connection's workflow subscription is fixed at connect time, so following a workflow means reconnecting — and the workflow binds partway through the first turn. An earlier revision of this change followed on the CHAT socket, which tore it down at exactly that moment and dropped every progress frame published while it re-established: the panel went from showing partial progress to showing none at all. Reconnecting the document socket instead costs nothing, because the follower resubscribes and catches up through resync. Verified against a real standalone agent over its own HTTP surface, both halves at once: progress frames (thinking, two tool calls, completion) arrive on a chat socket that is still OPEN when the turn ends, while `doc_update` seq=2 arrives on the document socket. Verified negatively too — a connection naming no workflow receives zero document frames for the identical turn. Deliberately NOT paired with a backend guard. The obvious one — refusing an unversioned seed — would reject every turn the currently shipped panel sends, turning silent data loss into a total outage. It should land after clients stop depending on the unversioned path. --- .../extensions/agent/AgentPanelRoot.vue | 70 ++++- .../agent/crdt/standaloneDocFrameTransport.ts | 277 ++++++++++++++++++ .../agent/crdt/useAgentCrdtFollower.ts | 16 +- 3 files changed, 354 insertions(+), 9 deletions(-) create mode 100644 src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 1affee303d4..793ed48d0c8 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -102,6 +102,7 @@ import { } from './crdt/crdtDebugGate' import { attachMintPortWiring } from './crdt/mintPortWiring' import { useAgentCrdtFollower } from './crdt/useAgentCrdtFollower' +import { createStandaloneDocFrameTransport } from './crdt/standaloneDocFrameTransport' const CrdtDevPanel = defineAsyncComponent( () => import('./crdt/CrdtDevPanel.vue') @@ -141,10 +142,38 @@ const userName = computed( const rest = createAgentRestClient() -const events = - import.meta.env.VITE_AGENT_STANDALONE === 'true' - ? createStandaloneAgentEventSource() - : createAgentEventSource(api) +const isStandaloneAgent = import.meta.env.VITE_AGENT_STANDALONE === 'true' + +const standaloneEvents = isStandaloneAgent + ? createStandaloneAgentEventSource() + : null + +const events = standaloneEvents ?? createAgentEventSource(api) + +/** + * Standalone document transport. The cloud path relies on ingest relaying the + * agent's document frames onto ComfyUI's socket; a local agent has no ingest, + * so the follower must read them off the agent's own stream and push edits to + * its HTTP routes. + * + * It owns a SEPARATE socket from the chat stream above, deliberately. The agent + * fixes a connection's workflow subscription at connect time, so following a + * workflow means reconnecting — and the workflow binds partway through the + * first turn. Reconnecting the CHAT stream at that moment drops every progress + * frame published while it re-establishes, leaving the panel silent for the + * rest of the turn. Reconnecting this one costs nothing: the follower + * resubscribes and catches up through resync. + */ +const standaloneDocTransport = isStandaloneAgent + ? createStandaloneDocFrameTransport({ + onError: (error, context) => + reportError(error, { + errorType: 'agent_standalone_doc_transport', + tags: { context }, + level: 'warning' + }) + }) + : null function onPaywallAction(action: AgentPaywallAction): void { openAccountPrecondition(action === 'addCredits' ? 'credits' : 'subscription') @@ -368,6 +397,20 @@ function activeWorkflowDraft(origin?: TurnOrigin): DraftSnapshot | undefined { if (workflowDetached.value) return undefined const active = originWorkflow(origin) if (!active) return undefined + // A live document is the source of truth for this workflow; the draft is only + // its projection. Seeding it from the canvas is not just redundant there, it + // is destructive: this snapshot carries no version, which the backend treats + // as authoritative and applies unconditionally, so a canvas that has not yet + // caught up silently overwrites whatever the agent just built. Stay out of + // the way and let the document carry the state. + const documentId = cloudIdFor(active) + if ( + documentId !== undefined && + crdtStatus.value.connected && + crdtStatus.value.workflowId === documentId + ) { + return undefined + } if (active.path === workflowStore.activeWorkflow?.path) active.changeTracker?.prepareForSave() const content = active.activeState @@ -500,8 +543,25 @@ const { // `app.isGraphReady` is a plain getter; reading `canvasStore.canvas` (set // right after `app.setup()`) makes the follower's graph watch fire once the // root graph exists. - () => (canvasStore.canvas && app.isGraphReady ? app.rootGraph : null) + () => (canvasStore.canvas && app.isGraphReady ? app.rootGraph : null), + // Standalone has no ingest relaying agent frames onto ComfyUI's socket, so + // the default transport would listen forever to a socket that never carries + // them. Talk to the agent directly instead. + standaloneDocTransport ?? undefined ) + +// Standalone only: the agent sends a workflow's document frames ONLY to +// connections that named it, so the bound workflow has to reach the document +// socket's query string. Without this the canvas never moves. This follows on +// the DOCUMENT socket, never the chat one — see the transport's comment for why +// reconnecting the chat stream here would silence the panel mid-turn. +if (standaloneDocTransport) { + watch(boundWorkflowId, (id) => standaloneDocTransport.follow(id ?? null), { + immediate: true + }) + onBeforeUnmount(() => standaloneDocTransport.destroy()) +} + const mintPortWiring = attachMintPortWiring({ isEnabled: () => agentPanelStore.enabled, isDocBound: () => isBoundWorkflowActive.value, diff --git a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts new file mode 100644 index 00000000000..02f184d9411 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts @@ -0,0 +1,277 @@ +/** + * A `DocFrameTransport` for the LOCAL (standalone) agent. + * + * The cloud transport (`apiTransport` in useAgentCrdtFollower) rides ComfyUI's + * same-origin `/ws`, because ingest relays the agent's document frames onto it. + * A standalone agent has no ingest, and ComfyUI's socket carries nothing from + * it, so a follower wired that way can never see a document update. That is why + * an agent could edit a workflow, report it accurately, and leave the canvas + * untouched. + * + * The local agent already exposes everything needed, just addressed + * differently: + * + * inbound GET /api/agent/events?workflow_id= doc_update / doc_reset + * catch-up POST /api/agent/doc/resync whole doc when the + * state vector is empty + * outbound POST /api/agent/doc/ops human edits + * + * ITS OWN SOCKET, ON PURPOSE. The agent fixes a connection's workflow + * subscription at connect time from the query string, so following a different + * workflow means reconnecting. The chat stream must NOT be what reconnects: the + * workflow binds partway through the first turn, and tearing the chat socket + * down at that moment drops every progress frame published while it is + * re-establishing — the panel goes silent for the rest of the turn. Dropping + * THIS socket is harmless by comparison, because the follower resubscribes and + * catches up through resync, so no document state is lost. + */ +import type { DocFrameTransport } from './docFrameClient' + +/** Frames the agent sends us. Anything else on the socket is not ours. */ +const INBOUND_DOC_FRAMES = new Set([ + 'doc_update', + 'doc_subscribed', + 'doc_ops_result', + 'doc_reset', + 'awareness' +]) + +/** The wire envelope every agent frame arrives in. */ +interface AgentFrame { + type?: unknown + data?: unknown +} + +/** + * Shapes describing what the agent sends, not a guarantee. These are parsed + * from the wire, so every field a caller dereferences is optional here and the + * defaults below are load-bearing rather than decorative. + */ +interface ResyncResponse { + workflow_id?: string + seq?: number + missing_b64?: string +} + +interface OpsResponse { + workflow_id?: string + seq?: number + applied?: string[] + skipped?: string[] + failed?: unknown +} + +export interface StandaloneDocFrameTransportOptions { + /** Overridable for tests; defaults to the panel's same-origin agent routes. */ + basePath?: string + fetchImpl?: typeof fetch + createSocket?: (url: string) => WebSocket + reconnectDelayMs?: number + /** Reported instead of thrown: a transport must never break the follower. */ + onError?: (error: unknown, context: string) => void +} + +export interface StandaloneDocFrameTransport extends DocFrameTransport { + /** + * Follow this workflow's document channel. Reconnects only on a real change, + * so it is safe to call on every tab activation. `null` disconnects. + */ + follow(workflowId: string | null): void + destroy(): void +} + +export function createStandaloneDocFrameTransport({ + basePath = '/api/agent', + fetchImpl = fetch, + createSocket = (url) => new WebSocket(url), + reconnectDelayMs = 1000, + onError = () => {} +}: StandaloneDocFrameTransportOptions = {}): StandaloneDocFrameTransport { + const target = new EventTarget() + let followed: string | null = null + let socket: WebSocket | null = null + let reconnectTimer: number | null = null + let destroyed = false + + function emit(type: string, data: unknown): void { + target.dispatchEvent(new CustomEvent(type, { detail: data })) + } + + function socketUrl(workflowId: string): string { + const url = new URL(`${basePath}/events`, window.location.href) + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' + // Repeated parameter, matching the server: it reads every `workflow_id` and + // subscribes that workflow's document channel. A connection naming none + // receives chat only, which is exactly the bug this transport exists to fix. + url.searchParams.append('workflow_id', workflowId) + return url.toString() + } + + function disconnect(): void { + if (reconnectTimer !== null) { + window.clearTimeout(reconnectTimer) + reconnectTimer = null + } + const current = socket + socket = null + current?.close() + } + + function connect(): void { + if (destroyed || followed === null || socket !== null) return + const current = createSocket(socketUrl(followed)) + socket = current + current.addEventListener('message', (event) => { + if (socket !== current) return + if (typeof event.data !== 'string') return + let frame: AgentFrame + try { + frame = JSON.parse(event.data) as AgentFrame + } catch { + return + } + if (typeof frame.type !== 'string') return + if (!INBOUND_DOC_FRAMES.has(frame.type)) return + emit(frame.type, frame.data) + }) + current.addEventListener('error', () => { + if (socket === current) current.close() + }) + current.addEventListener('close', () => { + if (socket !== current) return + socket = null + if (destroyed || followed === null || reconnectTimer !== null) return + reconnectTimer = window.setTimeout(() => { + reconnectTimer = null + connect() + }, reconnectDelayMs) + }) + } + + async function post(path: string, body: unknown): Promise { + try { + const response = await fetchImpl(`${basePath}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + if (!response.ok) { + onError(new Error(`HTTP ${response.status}`), path) + return null + } + return (await response.json()) as T + } catch (error) { + onError(error, path) + return null + } + } + + /** + * A resync answers with the state we are missing. Re-emit it as the two + * frames a subscribing client expects: the acknowledgement, then the state + * itself as an ordinary update. An empty payload means we were already + * current, so only the acknowledgement is emitted. + */ + async function resync(data: Record): Promise { + const workflowId = String(data.workflow_id ?? '') + const answer = await post('/doc/resync', data) + if (!answer) { + emit('doc_subscribed', { + workflow_id: workflowId, + ok: false, + code: 'resync_failed' + }) + return + } + emit('doc_subscribed', { + workflow_id: answer.workflow_id ?? workflowId, + ok: true, + seq: answer.seq + }) + if (answer.missing_b64) { + emit('doc_update', { + workflow_id: answer.workflow_id ?? workflowId, + seq: answer.seq, + update_b64: answer.missing_b64 + }) + } + } + + async function sendOps(data: Record): Promise { + const workflowId = String(data.workflow_id ?? '') + const answer = await post('/doc/ops', data) + if (!answer) { + emit('doc_ops_result', { + workflow_id: workflowId, + ok: false, + applied: [], + skipped: [], + code: 'ops_failed' + }) + return + } + emit('doc_ops_result', { + workflow_id: answer.workflow_id ?? workflowId, + ok: answer.failed === undefined || answer.failed === null, + seq: answer.seq, + applied: answer.applied ?? [], + skipped: answer.skipped ?? [], + ...(answer.failed ? { failed: answer.failed } : {}) + }) + // The agent broadcasts the resulting update to followers of this workflow, + // and this socket is one of them, so the effect arrives on its own. + } + + return { + /** + * Optimistically true. These are HTTP calls rather than socket writes, so + * there is no "socket not open" state to report, and the real outcome + * arrives as a frame. Returning false would stop the bridge latching the + * workflow it just subscribed to, and it would retry forever. + */ + send(frame: string): boolean { + let parsed: AgentFrame + try { + parsed = JSON.parse(frame) as AgentFrame + } catch (error) { + onError(error, 'parse outbound frame') + return false + } + const data = (parsed.data ?? {}) as Record + switch (parsed.type) { + case 'doc_subscribe': + void resync(data) + return true + case 'doc_unsubscribe': + // Nothing to tear down: the subscription is per-connection and the + // agent drops it when this socket closes. + return true + case 'doc_ops': + void sendOps(data) + return true + default: + // Awareness and anything else the cloud multiplexes onto /ws has no + // local equivalent. Swallow it rather than reporting a failure the + // caller cannot act on. + return true + } + }, + addEventListener(type, listener) { + target.addEventListener(type, listener) + }, + removeEventListener(type, listener) { + target.removeEventListener(type, listener) + }, + follow(workflowId) { + if (workflowId === followed) return + followed = workflowId + disconnect() + connect() + }, + destroy() { + destroyed = true + followed = null + disconnect() + } + } +} diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 2926a4089b5..bc79ba05012 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -203,7 +203,15 @@ export function useAgentCrdtFollower( * reads inside the getter are tracked, so a `null` → graph flip triggers a * reconcile without waiting for the next remote frame. */ - getGraph: () => MaterializableGraph | null = () => null + getGraph: () => MaterializableGraph | null = () => null, + /** + * Where document frames travel. Defaults to ComfyUI's same-origin socket, + * which carries them only because ingest relays the agent's frames onto it. + * A STANDALONE agent has no ingest, and ComfyUI's socket carries nothing from + * it, so this default can never see an update there — pass the standalone + * transport instead (see standaloneDocFrameTransport). + */ + baseTransport: DocFrameTransport = apiTransport ) { const connected = ref(false) const updatesApplied = ref(0) @@ -224,7 +232,7 @@ export function useAgentCrdtFollower( // never-throw contract is covered by tests. const transport: DocFrameTransport = { send(frame) { - const delivered = apiTransport.send(frame) + const delivered = baseTransport.send(frame) let parsed: unknown = frame try { parsed = JSON.parse(frame) @@ -235,10 +243,10 @@ export function useAgentCrdtFollower( return delivered }, addEventListener(type, listener) { - apiTransport.addEventListener(type, listener) + baseTransport.addEventListener(type, listener) }, removeEventListener(type, listener) { - apiTransport.removeEventListener(type, listener) + baseTransport.removeEventListener(type, listener) } } const client = new DocFrameClient(transport) From 21d067c6ed193b0dca0722d64493f68ee9df1817 Mon Sep 17 00:00:00 2001 From: kishore Date: Fri, 11 Sep 2026 13:43:38 -0700 Subject: [PATCH 02/64] fix(agent): one socket for chat and canvas on the standalone agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standalone agent now speaks the same document protocol on its event socket that ingest speaks on the cloud's /ws, so the panel keeps one shape for both targets: - The doc transport rides the chat stream's socket and forwards frames verbatim, exactly as the cloud transport rides ComfyUI's socket. The second per-workflow socket, its reconnects, and the HTTP translation are gone — following a workflow is a frame, so the chat stream never reconnects mid-turn. - Canvas ops are stamped with the identity the agent reports (GET /api/agent/identity), which is the actor its writer accepts. - The turn-context decision is extracted (turnContextFor): standalone has no ingest workflow list, so an unbound saved tab is sent tab-only and adopts the workflow the agent mints for it by the turn's own tab, the same way a temporary tab always has. This replaces the binding-repair watch, which bound to whatever tab was active when the ack landed. - The unversioned canvas re-seed is skipped while a document is live for the workflow; without this a canvas that had not caught up overwrote the agent's work on the next message. --- .../extensions/agent/AgentPanelRoot.vue | 94 ++++--- .../crdt/standaloneDocFrameTransport.test.ts | 91 ++++++ .../agent/crdt/standaloneDocFrameTransport.ts | 262 ++---------------- .../agent/standaloneAgentEventSource.test.ts | 25 ++ .../agent/standaloneAgentEventSource.ts | 24 +- .../extensions/agent/turnContext.test.ts | 64 +++++ src/workbench/extensions/agent/turnContext.ts | 39 +++ 7 files changed, 327 insertions(+), 272 deletions(-) create mode 100644 src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts create mode 100644 src/workbench/extensions/agent/turnContext.test.ts create mode 100644 src/workbench/extensions/agent/turnContext.ts diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 793ed48d0c8..6726cf00b90 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -103,6 +103,7 @@ import { import { attachMintPortWiring } from './crdt/mintPortWiring' import { useAgentCrdtFollower } from './crdt/useAgentCrdtFollower' import { createStandaloneDocFrameTransport } from './crdt/standaloneDocFrameTransport' +import { turnContextFor } from './turnContext' const CrdtDevPanel = defineAsyncComponent( () => import('./crdt/CrdtDevPanel.vue') @@ -135,15 +136,47 @@ const paywallPresentation = computed(() => const sidebarTabStore = useSidebarTabStore() const { isBuilderMode } = useAppMode() +// Declared here, above its first use: a `const` read before initialisation +// throws and takes the whole panel down at setup. +const isStandaloneAgent = import.meta.env.VITE_AGENT_STANDALONE === 'true' + const { resolvedUserInfo, userDisplayName } = useCurrentUser() + +/** + * Standalone only: the identity the AGENT authenticated this client as. + * + * Attribution on `POST /doc/ops` is server-derived, and a batch whose ops claim + * a different actor is refused with 403. In the cloud the panel's session user + * matches what ingest forwards, so the claim agrees. Standalone bootstraps a + * fixed local user that the panel cannot see, so it fell back to "anonymous" + * and EVERY canvas edit was rejected — silently, which is why the agent kept + * reporting an empty canvas no matter what was on screen. + * + * Asking the agent avoids hardcoding its bootstrap constant here. Null until it + * answers; the follower simply attributes nothing until then. + */ +const standaloneUserId = ref(null) +if (isStandaloneAgent) { + void (async () => { + try { + const response = await fetch('/api/agent/identity') + if (!response.ok) return + const identity = (await response.json()) as { user_id?: string } + standaloneUserId.value = identity.user_id ?? null + } catch (error) { + reportError(error, { + errorType: 'agent_standalone_identity_failed', + level: 'warning' + }) + } + })() +} const userName = computed( () => userDisplayName.value?.trim().split(/\s+/)[0] || undefined ) const rest = createAgentRestClient() -const isStandaloneAgent = import.meta.env.VITE_AGENT_STANDALONE === 'true' - const standaloneEvents = isStandaloneAgent ? createStandaloneAgentEventSource() : null @@ -151,28 +184,14 @@ const standaloneEvents = isStandaloneAgent const events = standaloneEvents ?? createAgentEventSource(api) /** - * Standalone document transport. The cloud path relies on ingest relaying the - * agent's document frames onto ComfyUI's socket; a local agent has no ingest, - * so the follower must read them off the agent's own stream and push edits to - * its HTTP routes. - * - * It owns a SEPARATE socket from the chat stream above, deliberately. The agent - * fixes a connection's workflow subscription at connect time, so following a - * workflow means reconnecting — and the workflow binds partway through the - * first turn. Reconnecting the CHAT stream at that moment drops every progress - * frame published while it re-establishes, leaving the panel silent for the - * rest of the turn. Reconnecting this one costs nothing: the follower - * resubscribes and catches up through resync. + * Standalone document transport. The cloud follower rides ComfyUI's one socket + * because ingest multiplexes the agent's document frames onto it. The + * standalone agent speaks that same document protocol on its own socket, so + * the follower rides the chat stream's socket here — one socket, exactly as in + * the cloud, and following a workflow is a frame rather than a reconnect. */ -const standaloneDocTransport = isStandaloneAgent - ? createStandaloneDocFrameTransport({ - onError: (error, context) => - reportError(error, { - errorType: 'agent_standalone_doc_transport', - tags: { context }, - level: 'warning' - }) - }) +const standaloneDocTransport = standaloneEvents + ? createStandaloneDocFrameTransport(standaloneEvents) : null function onPaywallAction(action: AgentPaywallAction): void { @@ -385,12 +404,17 @@ function activeWorkflowTurnContext( if (workflowDetached.value) return undefined const active = originWorkflow(origin) if (!active) return undefined - const id = cloudIdFor(active) - if (id === undefined && !active.isTemporary && origin !== undefined) - return undefined - return id === undefined - ? { tabPath: active.path } - : { id, tabPath: active.path } + return turnContextFor({ + id: cloudIdFor(active), + tabPath: active.path, + isTemporary: active.isTemporary, + hasOrigin: origin !== undefined, + // Standalone has no ingest workflow list: the tab binding is the only way + // a tab resolves, so an unbound saved tab must still be sent (tab-only) + // or it can never acquire the workflow adoption would give it — and the + // agent mints a fresh one on every message instead. + bindingIsAuthoritative: isStandaloneAgent + }) } function activeWorkflowDraft(origin?: TurnOrigin): DraftSnapshot | undefined { @@ -538,7 +562,9 @@ const { } = useAgentCrdtFollower( boundWorkflowId, graphMutations, - () => resolvedUserInfo.value?.id ?? null, + // Standalone: the agent's own identity, so canvas edits are stamped with the + // actor the server derives rather than "anonymous", which it refuses. + () => standaloneUserId.value ?? resolvedUserInfo.value?.id ?? null, isBoundWorkflowActive, // `app.isGraphReady` is a plain getter; reading `canvasStore.canvas` (set // right after `app.setup()`) makes the follower's graph watch fire once the @@ -550,15 +576,7 @@ const { standaloneDocTransport ?? undefined ) -// Standalone only: the agent sends a workflow's document frames ONLY to -// connections that named it, so the bound workflow has to reach the document -// socket's query string. Without this the canvas never moves. This follows on -// the DOCUMENT socket, never the chat one — see the transport's comment for why -// reconnecting the chat stream here would silence the panel mid-turn. if (standaloneDocTransport) { - watch(boundWorkflowId, (id) => standaloneDocTransport.follow(id ?? null), { - immediate: true - }) onBeforeUnmount(() => standaloneDocTransport.destroy()) } diff --git a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts new file mode 100644 index 00000000000..0af3443474b --- /dev/null +++ b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts @@ -0,0 +1,91 @@ +/** + * The standalone doc transport rides the SAME socket as the chat stream, the + * way the cloud transport rides ComfyUI's one socket. It must forward outbound + * frames verbatim and surface inbound document frames as events, while leaving + * chat frames to the chat listeners. + */ +import { describe, expect, it, vi } from 'vitest' + +import type { StandaloneAgentEventSource } from '../services/agent/standaloneAgentEventSource' +import { createStandaloneDocFrameTransport } from './standaloneDocFrameTransport' + +function fakeSource() { + const listeners = new Set<(raw: unknown) => void>() + const send = vi.fn((_frame: string) => true) + const source: StandaloneAgentEventSource = { + subscribe(listener) { + listeners.add(listener) + return () => listeners.delete(listener) + }, + send + } + return { + source, + send, + receive(frame: unknown) { + listeners.forEach((listener) => listener(frame)) + }, + listenerCount: () => listeners.size + } +} + +describe('createStandaloneDocFrameTransport', () => { + it('sends outbound doc frames on the shared socket verbatim', () => { + const fake = fakeSource() + const transport = createStandaloneDocFrameTransport(fake.source) + + expect(transport.send('{"type":"doc_subscribe","data":{}}')).toBe(true) + expect(fake.send).toHaveBeenCalledWith('{"type":"doc_subscribe","data":{}}') + }) + + it('reports the socket state instead of throwing', () => { + const fake = fakeSource() + fake.send.mockReturnValue(false) + const transport = createStandaloneDocFrameTransport(fake.source) + + expect(() => transport.send('{}')).not.toThrow() + expect(transport.send('{}')).toBe(false) + }) + + it('surfaces an inbound doc_update as an event carrying its data', () => { + const fake = fakeSource() + const transport = createStandaloneDocFrameTransport(fake.source) + const seen = vi.fn() + transport.addEventListener('doc_update', (event) => + seen((event as CustomEvent).detail) + ) + + fake.receive({ + type: 'doc_update', + data: { workflow_id: 'wf-1', seq: 2, update_b64: 'AQID' } + }) + + expect(seen).toHaveBeenCalledWith({ + workflow_id: 'wf-1', + seq: 2, + update_b64: 'AQID' + }) + }) + + it('does not surface chat frames to doc listeners', () => { + const fake = fakeSource() + const transport = createStandaloneDocFrameTransport(fake.source) + const seen = vi.fn() + transport.addEventListener('doc_update', seen) + + fake.receive({ type: 'agent_message_delta', data: { delta: 'hi' } }) + + expect(seen).not.toHaveBeenCalled() + }) + + it('stops listening once destroyed', () => { + const fake = fakeSource() + const transport = createStandaloneDocFrameTransport(fake.source) + transport.addEventListener('doc_update', vi.fn()) + expect(fake.listenerCount()).toBe(1) + + transport.destroy() + + expect(fake.listenerCount()).toBe(0) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts index 02f184d9411..2830389f8fc 100644 --- a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts +++ b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts @@ -2,32 +2,24 @@ * A `DocFrameTransport` for the LOCAL (standalone) agent. * * The cloud transport (`apiTransport` in useAgentCrdtFollower) rides ComfyUI's - * same-origin `/ws`, because ingest relays the agent's document frames onto it. - * A standalone agent has no ingest, and ComfyUI's socket carries nothing from - * it, so a follower wired that way can never see a document update. That is why - * an agent could edit a workflow, report it accurately, and leave the canvas - * untouched. + * one same-origin socket, because ingest multiplexes the agent's document + * frames onto it. Standalone has no ingest — but the agent's own socket now + * speaks the identical document protocol (doc_subscribe / doc_unsubscribe / + * doc_ops inbound; doc_subscribed / doc_update / doc_ops_result / doc_reset + * outbound), so this transport is the same idea with the socket swapped: ride + * the chat stream's socket, forward outbound frames verbatim, surface inbound + * document frames as events. * - * The local agent already exposes everything needed, just addressed - * differently: - * - * inbound GET /api/agent/events?workflow_id= doc_update / doc_reset - * catch-up POST /api/agent/doc/resync whole doc when the - * state vector is empty - * outbound POST /api/agent/doc/ops human edits - * - * ITS OWN SOCKET, ON PURPOSE. The agent fixes a connection's workflow - * subscription at connect time from the query string, so following a different - * workflow means reconnecting. The chat stream must NOT be what reconnects: the - * workflow binds partway through the first turn, and tearing the chat socket - * down at that moment drops every progress frame published while it is - * re-establishing — the panel goes silent for the rest of the turn. Dropping - * THIS socket is harmless by comparison, because the follower resubscribes and - * catches up through resync, so no document state is lost. + * ONE socket, on purpose. An earlier cut opened a second socket per followed + * workflow because the agent fixed a connection's follows at connect time; + * reconnecting to follow tore the chat stream down mid-turn and lost every + * progress frame published while it re-established. With follows as frames, + * nothing reconnects. */ +import type { StandaloneAgentEventSource } from '../services/agent/standaloneAgentEventSource' import type { DocFrameTransport } from './docFrameClient' -/** Frames the agent sends us. Anything else on the socket is not ours. */ +/** Frames the agent sends followers. Anything else on the socket is chat. */ const INBOUND_DOC_FRAMES = new Set([ 'doc_update', 'doc_subscribed', @@ -36,242 +28,46 @@ const INBOUND_DOC_FRAMES = new Set([ 'awareness' ]) -/** The wire envelope every agent frame arrives in. */ interface AgentFrame { type?: unknown data?: unknown } -/** - * Shapes describing what the agent sends, not a guarantee. These are parsed - * from the wire, so every field a caller dereferences is optional here and the - * defaults below are load-bearing rather than decorative. - */ -interface ResyncResponse { - workflow_id?: string - seq?: number - missing_b64?: string -} - -interface OpsResponse { - workflow_id?: string - seq?: number - applied?: string[] - skipped?: string[] - failed?: unknown -} - -export interface StandaloneDocFrameTransportOptions { - /** Overridable for tests; defaults to the panel's same-origin agent routes. */ - basePath?: string - fetchImpl?: typeof fetch - createSocket?: (url: string) => WebSocket - reconnectDelayMs?: number - /** Reported instead of thrown: a transport must never break the follower. */ - onError?: (error: unknown, context: string) => void -} - export interface StandaloneDocFrameTransport extends DocFrameTransport { - /** - * Follow this workflow's document channel. Reconnects only on a real change, - * so it is safe to call on every tab activation. `null` disconnects. - */ - follow(workflowId: string | null): void destroy(): void } -export function createStandaloneDocFrameTransport({ - basePath = '/api/agent', - fetchImpl = fetch, - createSocket = (url) => new WebSocket(url), - reconnectDelayMs = 1000, - onError = () => {} -}: StandaloneDocFrameTransportOptions = {}): StandaloneDocFrameTransport { +export function createStandaloneDocFrameTransport( + source: StandaloneAgentEventSource +): StandaloneDocFrameTransport { const target = new EventTarget() - let followed: string | null = null - let socket: WebSocket | null = null - let reconnectTimer: number | null = null - let destroyed = false - - function emit(type: string, data: unknown): void { - target.dispatchEvent(new CustomEvent(type, { detail: data })) - } - - function socketUrl(workflowId: string): string { - const url = new URL(`${basePath}/events`, window.location.href) - url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' - // Repeated parameter, matching the server: it reads every `workflow_id` and - // subscribes that workflow's document channel. A connection naming none - // receives chat only, which is exactly the bug this transport exists to fix. - url.searchParams.append('workflow_id', workflowId) - return url.toString() - } - - function disconnect(): void { - if (reconnectTimer !== null) { - window.clearTimeout(reconnectTimer) - reconnectTimer = null - } - const current = socket - socket = null - current?.close() - } + let unsubscribe: (() => void) | null = null - function connect(): void { - if (destroyed || followed === null || socket !== null) return - const current = createSocket(socketUrl(followed)) - socket = current - current.addEventListener('message', (event) => { - if (socket !== current) return - if (typeof event.data !== 'string') return - let frame: AgentFrame - try { - frame = JSON.parse(event.data) as AgentFrame - } catch { - return - } + function tap(): void { + if (unsubscribe !== null) return + unsubscribe = source.subscribe((raw) => { + if (typeof raw !== 'object' || raw === null) return + const frame = raw as AgentFrame if (typeof frame.type !== 'string') return if (!INBOUND_DOC_FRAMES.has(frame.type)) return - emit(frame.type, frame.data) - }) - current.addEventListener('error', () => { - if (socket === current) current.close() - }) - current.addEventListener('close', () => { - if (socket !== current) return - socket = null - if (destroyed || followed === null || reconnectTimer !== null) return - reconnectTimer = window.setTimeout(() => { - reconnectTimer = null - connect() - }, reconnectDelayMs) - }) - } - - async function post(path: string, body: unknown): Promise { - try { - const response = await fetchImpl(`${basePath}${path}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body) - }) - if (!response.ok) { - onError(new Error(`HTTP ${response.status}`), path) - return null - } - return (await response.json()) as T - } catch (error) { - onError(error, path) - return null - } - } - - /** - * A resync answers with the state we are missing. Re-emit it as the two - * frames a subscribing client expects: the acknowledgement, then the state - * itself as an ordinary update. An empty payload means we were already - * current, so only the acknowledgement is emitted. - */ - async function resync(data: Record): Promise { - const workflowId = String(data.workflow_id ?? '') - const answer = await post('/doc/resync', data) - if (!answer) { - emit('doc_subscribed', { - workflow_id: workflowId, - ok: false, - code: 'resync_failed' - }) - return - } - emit('doc_subscribed', { - workflow_id: answer.workflow_id ?? workflowId, - ok: true, - seq: answer.seq - }) - if (answer.missing_b64) { - emit('doc_update', { - workflow_id: answer.workflow_id ?? workflowId, - seq: answer.seq, - update_b64: answer.missing_b64 - }) - } - } - - async function sendOps(data: Record): Promise { - const workflowId = String(data.workflow_id ?? '') - const answer = await post('/doc/ops', data) - if (!answer) { - emit('doc_ops_result', { - workflow_id: workflowId, - ok: false, - applied: [], - skipped: [], - code: 'ops_failed' - }) - return - } - emit('doc_ops_result', { - workflow_id: answer.workflow_id ?? workflowId, - ok: answer.failed === undefined || answer.failed === null, - seq: answer.seq, - applied: answer.applied ?? [], - skipped: answer.skipped ?? [], - ...(answer.failed ? { failed: answer.failed } : {}) + target.dispatchEvent(new CustomEvent(frame.type, { detail: frame.data })) }) - // The agent broadcasts the resulting update to followers of this workflow, - // and this socket is one of them, so the effect arrives on its own. } return { - /** - * Optimistically true. These are HTTP calls rather than socket writes, so - * there is no "socket not open" state to report, and the real outcome - * arrives as a frame. Returning false would stop the bridge latching the - * workflow it just subscribed to, and it would retry forever. - */ - send(frame: string): boolean { - let parsed: AgentFrame - try { - parsed = JSON.parse(frame) as AgentFrame - } catch (error) { - onError(error, 'parse outbound frame') - return false - } - const data = (parsed.data ?? {}) as Record - switch (parsed.type) { - case 'doc_subscribe': - void resync(data) - return true - case 'doc_unsubscribe': - // Nothing to tear down: the subscription is per-connection and the - // agent drops it when this socket closes. - return true - case 'doc_ops': - void sendOps(data) - return true - default: - // Awareness and anything else the cloud multiplexes onto /ws has no - // local equivalent. Swallow it rather than reporting a failure the - // caller cannot act on. - return true - } + send(frame) { + return source.send(frame) }, addEventListener(type, listener) { + tap() target.addEventListener(type, listener) }, removeEventListener(type, listener) { target.removeEventListener(type, listener) }, - follow(workflowId) { - if (workflowId === followed) return - followed = workflowId - disconnect() - connect() - }, destroy() { - destroyed = true - followed = null - disconnect() + unsubscribe?.() + unsubscribe = null } } } diff --git a/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.test.ts b/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.test.ts index 4716328a829..4403d2b5106 100644 --- a/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.test.ts +++ b/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.test.ts @@ -194,3 +194,28 @@ describe('createStandaloneAgentEventSource', () => { expect(sockets[0].readyState).toBe(WebSocket.CLOSED) }) }) + +describe('createStandaloneAgentEventSource send', () => { + it('reports false before the socket is open and writes nothing', () => { + const { source, sockets } = sourceHarness() + source.subscribe(vi.fn()) + const socket = sockets[0] + const send = vi.fn() + Object.assign(socket, { send }) + + expect(source.send('{"type":"doc_subscribe"}')).toBe(false) + expect(send).not.toHaveBeenCalled() + }) + + it('writes the frame verbatim on the open socket and reports true', () => { + const { source, sockets } = sourceHarness() + source.subscribe(vi.fn()) + const socket = sockets[0] + const send = vi.fn() + Object.assign(socket, { send }) + socket.open() + + expect(source.send('{"type":"doc_subscribe"}')).toBe(true) + expect(send).toHaveBeenCalledWith('{"type":"doc_subscribe"}') + }) +}) diff --git a/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.ts b/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.ts index 7fa6fea5427..45ce8c16e8c 100644 --- a/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.ts +++ b/src/workbench/extensions/agent/services/agent/standaloneAgentEventSource.ts @@ -1,5 +1,22 @@ import type { AgentEventSource } from '../../composables/agent/useAgentSession' +/** + * The standalone agent's ONE socket. Chat frames come down it, and — because + * the agent speaks the same document protocol on it that ingest speaks on the + * cloud's /ws — document frames go up and down it too. `send` is what lets the + * doc transport ride this socket instead of opening a second one: a + * connection's follows are then frames, not a query string, so following a + * workflow never reconnects the chat stream mid-turn. + */ +export interface StandaloneAgentEventSource extends AgentEventSource { + /** + * Best-effort send, never throws. `false` when the socket is not OPEN — + * a normal state while the panel mounts — so a caller retries rather than + * aborts. + */ + send(frame: string): boolean +} + interface StandaloneAgentEventSourceOptions { createSocket?: (url: string) => WebSocket endpoint?: string @@ -10,7 +27,7 @@ export function createStandaloneAgentEventSource({ createSocket = (url) => new WebSocket(url), endpoint = '/api/agent/events', reconnectDelayMs = 1000 -}: StandaloneAgentEventSourceOptions = {}): AgentEventSource { +}: StandaloneAgentEventSourceOptions = {}): StandaloneAgentEventSource { const listeners = new Set<(raw: unknown) => void>() const statusListeners = new Set<(live: boolean) => void>() let socket: WebSocket | null = null @@ -77,6 +94,11 @@ export function createStandaloneAgentEventSource({ } return { + send(frame) { + if (socket?.readyState !== WebSocket.OPEN) return false + socket.send(frame) + return true + }, subscribe(listener) { listeners.add(listener) connect() diff --git a/src/workbench/extensions/agent/turnContext.test.ts b/src/workbench/extensions/agent/turnContext.test.ts new file mode 100644 index 00000000000..30f6013f24a --- /dev/null +++ b/src/workbench/extensions/agent/turnContext.test.ts @@ -0,0 +1,64 @@ +/** + * Which workflow a turn is attributed to, and whether a tab may adopt the + * workflow the agent mints for it. + * + * In the cloud a saved tab resolves its workflow through ingest's workflow + * list; a saved tab that does NOT resolve means the list has not loaded, and + * sending a tab-only context would let the agent mint a second workflow for a + * tab that already has one. Standalone has no such list — the tab binding is + * the only source of truth — so an unbound saved tab is simply a tab the agent + * has not been given a workflow for yet, exactly like a temporary one. + */ +import { describe, expect, it } from 'vitest' + +import { turnContextFor } from './turnContext' + +describe('turnContextFor', () => { + it('names the resolved workflow when the tab has one', () => { + expect( + turnContextFor({ + id: 'wf-1', + tabPath: 'workflows/a.json', + isTemporary: false, + hasOrigin: true, + bindingIsAuthoritative: false + }) + ).toEqual({ id: 'wf-1', tabPath: 'workflows/a.json' }) + }) + + it('withholds the context for an unresolved saved tab when a cloud list is authoritative', () => { + expect( + turnContextFor({ + id: undefined, + tabPath: 'workflows/a.json', + isTemporary: false, + hasOrigin: true, + bindingIsAuthoritative: false + }) + ).toBeUndefined() + }) + + it('sends a tab-only context for an unresolved saved tab when the binding is authoritative', () => { + expect( + turnContextFor({ + id: undefined, + tabPath: 'workflows/a.json', + isTemporary: false, + hasOrigin: true, + bindingIsAuthoritative: true + }) + ).toEqual({ tabPath: 'workflows/a.json' }) + }) + + it('always sends a tab-only context for a temporary tab', () => { + expect( + turnContextFor({ + id: undefined, + tabPath: 'workflows/temp.json', + isTemporary: true, + hasOrigin: true, + bindingIsAuthoritative: false + }) + ).toEqual({ tabPath: 'workflows/temp.json' }) + }) +}) diff --git a/src/workbench/extensions/agent/turnContext.ts b/src/workbench/extensions/agent/turnContext.ts new file mode 100644 index 00000000000..132cd746220 --- /dev/null +++ b/src/workbench/extensions/agent/turnContext.ts @@ -0,0 +1,39 @@ +import type { WorkflowTurnContext } from './composables/agent/useAgentSession' + +export interface TurnContextInput { + /** The workflow the tab resolved to, if any. */ + id: string | undefined + tabPath: string + isTemporary: boolean + /** Whether the send named an origin tab (as opposed to a detached turn). */ + hasOrigin: boolean + /** + * True when the tab binding is the ONLY way a tab resolves to a workflow — + * the standalone agent, which has no ingest workflow list. False in the + * cloud, where a saved tab also resolves by name through that list. + */ + bindingIsAuthoritative: boolean +} + +/** + * Which workflow a turn is attributed to. + * + * A resolved tab names its workflow. An unresolved one is sent tab-only, which + * lets the agent mint a workflow for it and lets the tab adopt that workflow on + * the ack — EXCEPT for a saved tab in the cloud: there "unresolved" most likely + * means the workflow list has not loaded, and a tab-only context would let the + * agent mint a second workflow for a tab that already has one. Standalone has + * no list to wait for, so an unbound saved tab is exactly what a temporary one + * is: a tab the agent has not yet been given a workflow for. + */ +export function turnContextFor({ + id, + tabPath, + isTemporary, + hasOrigin, + bindingIsAuthoritative +}: TurnContextInput): WorkflowTurnContext | undefined { + if (id !== undefined) return { id, tabPath } + if (!isTemporary && hasOrigin && !bindingIsAuthoritative) return undefined + return { tabPath } +} From 32958b8754e72299b82ca880f9b79add2ebd25ea Mon Sep 17 00:00:00 2001 From: kishore Date: Fri, 11 Sep 2026 14:56:57 -0700 Subject: [PATCH 03/64] fix(agent): the standalone harness enables the panel without the PostHog flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a production bundle the panel was enabled only by the PostHog flag. A standalone panel has no cloud identity for PostHog to evaluate the flag against, so `enabled` stayed false and nothing rendered — no toggle, no dock, and the agent never saw the page. The dev server escaped this only through MODE === 'development'. The standalone harness is the opt-in (the panel is tree-shaken out of every other non-cloud build), so it now forces the panel on and settles the gate immediately, exactly as development does. --- src/extensions/core/agentPanel.test.ts | 14 ++++++++++++++ src/extensions/core/agentPanel.ts | 14 +++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index 51594da8920..7bcbb036b4a 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -127,6 +127,20 @@ describe('AgentPanel extension flag gate', () => { expect(agentStore.enabled).toBe(true) }) + it('forces the panel on for the standalone agent harness even while the flag is false', async () => { + // A standalone panel has no cloud identity for PostHog to evaluate the + // flag against, so gating it on the flag leaves it permanently off in a + // production bundle. The harness itself is the opt-in (the panel is + // tree-shaken out of every other non-cloud build). + vi.stubEnv('VITE_AGENT_STANDALONE', 'true') + mocks.flagEnabled = false + + await loadEntryAndSetup() + + expect(agentStore.enabled).toBe(true) + expect(agentStore.gateSettled).toBe(true) + }) + it('leaves the panel disabled while the flag is undefined', async () => { await loadEntryAndSetup() expect(agentStore.enabled).toBe(false) diff --git a/src/extensions/core/agentPanel.ts b/src/extensions/core/agentPanel.ts index 3bd24ab4c0d..346666bd9ba 100644 --- a/src/extensions/core/agentPanel.ts +++ b/src/extensions/core/agentPanel.ts @@ -88,9 +88,17 @@ async function setupFlagGate(): Promise { import('posthog-js') ]) const source = createPostHogFlagSource(posthog) + // Two harnesses force the panel on regardless of the flag: development, + // and the standalone (local agent) build. The standalone panel has no + // cloud identity for PostHog to evaluate the flag against, so gating it on + // the flag left it permanently off in a production bundle — the harness + // itself is the opt-in, since the panel is tree-shaken out of every other + // non-cloud build (see extensions/core/index.ts). + const forcedOn = + import.meta.env.MODE === 'development' || + import.meta.env.VITE_AGENT_STANDALONE === 'true' const sync = (): void => { - const forceInDev = import.meta.env.MODE === 'development' - agentPanelStore.enabled = forceInDev || source.isEnabled() + agentPanelStore.enabled = forcedOn || source.isEnabled() if (!agentPanelStore.enabled) { const nodeSelectionStore = useAgentNodeSelectionStore() if (nodeSelectionStore.isLoadingWorkflow) @@ -102,7 +110,7 @@ async function setupFlagGate(): Promise { settle() }) sync() - if (import.meta.env.MODE === 'development') settle() + if (forcedOn) settle() else setTimeout(settle, FLAG_SETTLE_TIMEOUT_MS) } catch (error) { settle() From 4128daae6d95f7a60c3f0ff8c47795789e0663d6 Mon Sep 17 00:00:00 2001 From: kishore Date: Fri, 11 Sep 2026 15:18:55 -0700 Subject: [PATCH 04/64] fix(agent): a turn from a tab with no workflow mints one instead of reusing the thread's A panel can only name a tab that resolved to a workflow. A fresh tab has not, so it sent no workflow_id and no current_tab, and the server fell back to the thread's previous workflow: the agent edited the tab the user was NOT looking at, the ack named that workflow, the fresh tab could not adopt it (another tab already owned it), and the user saw an empty canvas while the agent truthfully reported the node it had added elsewhere. The session now sends `current_tab_unbound: true` for a turn context that carries a tab but no workflow id, and the server mints a workflow for that turn instead of taking the thread fallback. The ack's workflow is then fresh and unclaimed, so the originating tab adopts it and the follower tracks it. --- .../composables/agent/useAgentSession.test.ts | 58 +++++++++++++++++++ .../composables/agent/useAgentSession.ts | 16 +++-- .../services/agent/agentRestClient.test.ts | 14 +++++ .../agent/services/agent/agentRestClient.ts | 7 +++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts index e3b9ae76002..0c013bce2ad 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.test.ts @@ -995,6 +995,64 @@ describe('useAgentSession (v1 composition root)', () => { }) }) + it('tells the server the current tab is unbound when the turn context has no workflow id', async () => { + const postMessage = vi.fn(async () => ({ + thread_id: 'th-1', + message_id: 'msg-1', + workflow_id: 'wf-minted' + })) as unknown as AgentRestClient['postMessage'] + const rest = fakeRest({ postMessage }) + const { source } = fakeEvents() + const session = useAgentSession({ + rest, + events: source, + workflow: { + current: () => ({ tabPath: 'tab-new' }), + adopted: vi.fn(), + prepare: vi.fn(async () => undefined) + } + }) + session.start() + + await session.sendMessage('add a node') + + expect(vi.mocked(postMessage).mock.calls[0][1]).toMatchObject({ + currentTabUnbound: true + }) + expect(vi.mocked(postMessage).mock.calls[0][1]).not.toHaveProperty( + 'workflowId' + ) + }) + + it('does not flag the current tab as unbound when the turn context names a workflow', async () => { + const postMessage = vi.fn(async () => ({ + thread_id: 'th-1', + message_id: 'msg-1', + workflow_id: 'wf-a' + })) as unknown as AgentRestClient['postMessage'] + const rest = fakeRest({ postMessage }) + const { source } = fakeEvents() + const session = useAgentSession({ + rest, + events: source, + workflow: { + current: () => ({ id: 'wf-a', tabPath: 'tab-a' }), + adopted: vi.fn(), + prepare: vi.fn(async () => undefined) + } + }) + session.start() + + await session.sendMessage('add a node') + + expect(vi.mocked(postMessage).mock.calls[0][1]).toMatchObject({ + workflowId: 'wf-a' + }) + expect(vi.mocked(postMessage).mock.calls[0][1]).not.toHaveProperty( + 'currentTabUnbound' + ) + }) + it('(h8) the draft snapshot follows the originating tab, not the tab switched to during prepare()', async () => { const postMessage = vi.fn(async () => ({ thread_id: 'th-1', diff --git a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts index 3279f75caeb..ec641b61953 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentSession.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentSession.ts @@ -274,12 +274,16 @@ export function useAgentSession(deps: AgentSessionDeps) { attachments: attachments?.map((attachment) => attachment.ref), ...(shouldSendDraft ? { draft } : {}) } - return rest.postMessage( - threadId, - wfContext?.id !== undefined - ? { ...input, workflowId: wfContext.id } - : input - ) + // A context with a tab but no workflow is a tab the agent has not been + // given a workflow for; say so, or the server reuses the thread's + // previous workflow and edits a tab the user is not looking at. + const attributed = + wfContext === undefined + ? input + : wfContext.id === undefined + ? { ...input, currentTabUnbound: true } + : { ...input, workflowId: wfContext.id } + return rest.postMessage(threadId, attributed) } try { const ack = await postTurn(conversationStore.threadId ?? 'new') diff --git a/src/workbench/extensions/agent/services/agent/agentRestClient.test.ts b/src/workbench/extensions/agent/services/agent/agentRestClient.test.ts index 187ecdcdc52..03b02992b2b 100644 --- a/src/workbench/extensions/agent/services/agent/agentRestClient.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentRestClient.test.ts @@ -183,6 +183,20 @@ describe('postMessage wire body', () => { expect(contentType(init)).toBe('application/json') }) + it('sends current_tab_unbound when the turn comes from a tab with no workflow', async () => { + respond(jsonResponse(202, turnAccepted)) + await makeClient().postMessage('t1', { + content: 'add a node', + currentTabUnbound: true + }) + + const parsed = JSON.parse(lastCall().init.body as string) as Record< + string, + unknown + > + expect(parsed).toEqual({ content: 'add a node', current_tab_unbound: true }) + }) + it('omits absent optionals rather than sending them as undefined keys', async () => { respond(jsonResponse(202, turnAccepted)) await makeClient().postMessage('t1', { content: 'just text' }) diff --git a/src/workbench/extensions/agent/services/agent/agentRestClient.ts b/src/workbench/extensions/agent/services/agent/agentRestClient.ts index d18e0710dff..cc4f56999c8 100644 --- a/src/workbench/extensions/agent/services/agent/agentRestClient.ts +++ b/src/workbench/extensions/agent/services/agent/agentRestClient.ts @@ -65,6 +65,12 @@ export interface DraftSnapshot { export interface PostMessageInput { content: string workflowId?: string + /** + * The turn comes from a tab that has no workflow yet. The server then mints + * one for it instead of falling back to the thread's previous workflow — + * which would edit a tab the user is not looking at. + */ + currentTabUnbound?: boolean selection?: Record attachments?: string[] tabs?: OpenTabsSnapshot @@ -139,6 +145,7 @@ export function createAgentRestClient() { ): Promise { const body: Record = { content: req.content } if (req.workflowId !== undefined) body.workflow_id = req.workflowId + if (req.currentTabUnbound) body.current_tab_unbound = true if (req.tabs !== undefined) { body.open_tabs = req.tabs.open_tabs if (req.tabs.current_tab !== undefined) From 13ef516151dd6c1241b2daf78d9f49d2cd5d911a Mon Sep 17 00:00:00 2001 From: kishore Date: Fri, 11 Sep 2026 15:29:41 -0700 Subject: [PATCH 05/64] fix(agent): re-drive a dropped subscribe when the standalone socket opens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A doc_subscribe sent while the transport's socket is still connecting is dropped by design, and the follower re-drives its intent on ComfyUI's own socket events (status, reconnected) because in the cloud the doc frames ride that socket. Locally they ride the agent's socket, which those events never describe, so a dropped subscribe waited for ComfyUI's next unrelated status frame — the canvas caught up seconds late. DocFrameTransport gains an optional onConnected hook. The standalone transport implements it from the chat stream's onStatus; the follower reconciles on it exactly as it does on a ComfyUI status frame. The cloud transport needs nothing. --- .../extensions/agent/crdt/docFrameClient.ts | 10 ++++++ .../crdt/standaloneDocFrameTransport.test.ts | 28 +++++++++++++++ .../agent/crdt/standaloneDocFrameTransport.ts | 13 +++++++ .../agent/crdt/useAgentCrdtFollower.test.ts | 34 +++++++++++++++++-- .../agent/crdt/useAgentCrdtFollower.ts | 8 ++++- 5 files changed, 90 insertions(+), 3 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/docFrameClient.ts b/src/workbench/extensions/agent/crdt/docFrameClient.ts index 318ccb5963b..f69862c0ced 100644 --- a/src/workbench/extensions/agent/crdt/docFrameClient.ts +++ b/src/workbench/extensions/agent/crdt/docFrameClient.ts @@ -101,6 +101,16 @@ export interface DocFrameTransport { send(frame: string): boolean addEventListener(type: string, listener: EventListener): void removeEventListener(type: string, listener: EventListener): void + /** + * Optional: notify when the transport's socket has (re)opened and can carry + * a frame. A subscribe sent while the socket was still connecting is dropped + * by design (`send` returned false), and the follower re-drives its intent + * on this signal. The cloud transport does not need it — ComfyUI's socket + * announces itself with `status` and `reconnected` events the follower + * already listens to — but a transport on a different socket must provide + * it, or a dropped subscribe waits for an unrelated trigger. + */ + onConnected?(listener: () => void): () => void } interface WireData { diff --git a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts index 0af3443474b..bce2832c555 100644 --- a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts +++ b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.test.ts @@ -11,12 +11,17 @@ import { createStandaloneDocFrameTransport } from './standaloneDocFrameTransport function fakeSource() { const listeners = new Set<(raw: unknown) => void>() + const statusListeners = new Set<(live: boolean) => void>() const send = vi.fn((_frame: string) => true) const source: StandaloneAgentEventSource = { subscribe(listener) { listeners.add(listener) return () => listeners.delete(listener) }, + onStatus(listener) { + statusListeners.add(listener) + return () => statusListeners.delete(listener) + }, send } return { @@ -25,6 +30,9 @@ function fakeSource() { receive(frame: unknown) { listeners.forEach((listener) => listener(frame)) }, + status(live: boolean) { + statusListeners.forEach((listener) => listener(live)) + }, listenerCount: () => listeners.size } } @@ -78,6 +86,26 @@ describe('createStandaloneDocFrameTransport', () => { expect(seen).not.toHaveBeenCalled() }) + it('reports when the shared socket becomes usable, so a dropped subscribe can be re-driven', () => { + // A subscribe sent while the socket is still connecting is dropped by + // design. In the cloud the follower re-drives it off ComfyUI's own socket + // events; those never fire for the agent socket, so this transport has + // to say when ITS socket opened. + const fake = fakeSource() + const transport = createStandaloneDocFrameTransport(fake.source) + const connected = vi.fn() + transport.onConnected(connected) + + fake.status(true) + expect(connected).toHaveBeenCalledTimes(1) + + fake.status(false) + expect(connected).toHaveBeenCalledTimes(1) + + fake.status(true) + expect(connected).toHaveBeenCalledTimes(2) + }) + it('stops listening once destroyed', () => { const fake = fakeSource() const transport = createStandaloneDocFrameTransport(fake.source) diff --git a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts index 2830389f8fc..054871c445f 100644 --- a/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts +++ b/src/workbench/extensions/agent/crdt/standaloneDocFrameTransport.ts @@ -34,6 +34,7 @@ interface AgentFrame { } export interface StandaloneDocFrameTransport extends DocFrameTransport { + onConnected(listener: () => void): () => void destroy(): void } @@ -65,6 +66,18 @@ export function createStandaloneDocFrameTransport( removeEventListener(type, listener) { target.removeEventListener(type, listener) }, + /** + * Fires each time the shared socket opens. The chat stream's `onStatus` + * reports every open and close; only the opens matter here, because + * that is the moment a subscribe dropped while connecting can go out. + */ + onConnected(listener) { + return ( + source.onStatus?.((live) => { + if (live) listener() + }) ?? (() => {}) + ) + }, destroy() { unsubscribe?.() unsubscribe = null diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 6e805620ef1..7f07e22db68 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -134,6 +134,7 @@ vi.mock(import('@/scripts/app'), () => ({ import { STALE_AFTER_MS, useAgentCrdtFollower } from './useAgentCrdtFollower' import type { AgentCrdtStatus } from './useAgentCrdtFollower' +import type { DocFrameTransport } from './docFrameClient' const graphMutations = {} as GraphMutations const DOC_ID_KEY = 'Comfy.Agent.CrdtDocId' @@ -166,7 +167,8 @@ function writeRawRecord(overrides: { function mountFollower( initial: string | null = null, initiallyActive = true, - getGraph: () => MaterializableGraph | null = () => null + getGraph: () => MaterializableGraph | null = () => null, + baseTransport?: DocFrameTransport ): { unmount: () => void workflowId: Ref @@ -183,7 +185,8 @@ function mountFollower( graphMutations, () => null, isTargetActive, - getGraph + getGraph, + baseTransport ) exposedStatus = () => status.value as AgentCrdtStatus return () => null @@ -212,6 +215,33 @@ describe('useAgentCrdtFollower', () => { definitionsState.readSubgraphDefinitions.mockClear() }) + it('reconciles subscription intent when the transport reports its socket connected', () => { + // A subscribe sent while the transport's socket is still connecting is + // dropped. The cloud follower re-drives it off ComfyUI's own socket + // events; a transport on a different socket (the standalone agent's) + // must be able to say "I am usable now" and get the same reconcile. + let connectedListener: (() => void) | null = null + const stopConnected = vi.fn() + const transport: DocFrameTransport = { + send: vi.fn(() => false), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + onConnected: (listener) => { + connectedListener = listener + return stopConnected + } + } + const { unmount } = mountFollower('wf-1', true, () => null, transport) + expect(connectedListener).toBeTypeOf('function') + const before = bridge().reconcile.mock.calls.length + + connectedListener!() + + expect(bridge().reconcile.mock.calls.length).toBe(before + 1) + unmount() + expect(stopConnected).toHaveBeenCalledTimes(1) + }) + it('subscribes immediately to a bound workflow and reports it in status', () => { const { unmount, status } = mountFollower('wf-1') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index bc79ba05012..7da9d9390c4 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -568,10 +568,11 @@ export function useAgentCrdtFollower( * nothing unless a refused subscribe has a scheduled retry. In that case, * the retry timer owns the next attempt and its backoff. */ - const onSocketActivity: EventListener = () => { + const reconcileIfIdle = (): void => { if (subscribeRetryTimer !== null) return bridge.reconcile() } + const onSocketActivity: EventListener = () => reconcileIfIdle() bridge.addEventListener('doc_subscribed', onSubscribed) bridge.addEventListener('doc_update', onUpdate) @@ -583,6 +584,10 @@ export function useAgentCrdtFollower( bridge.addEventListener('doc_stale', onStale) api.addEventListener('reconnected', onReconnected) api.addEventListener('status', onSocketActivity) + // A transport on a socket other than ComfyUI's (the standalone agent's) + // announces its own opens; without this a subscribe dropped while that + // socket was connecting waited for ComfyUI's next unrelated status frame. + const stopTransportConnected = baseTransport.onConnected?.(reconcileIfIdle) // FE-1902 (poc-3): distinguish the mount-time null (in-memory doc id died // with the previous mount — rebind from sessionStorage) from a later null @@ -697,6 +702,7 @@ export function useAgentCrdtFollower( clearSubscribeRetry() clearStaleProbe() api.removeEventListener('reconnected', onReconnected) + stopTransportConnected?.() api.removeEventListener('status', onSocketActivity) bridge.removeEventListener('doc_subscribed', onSubscribed) bridge.removeEventListener('doc_update', onUpdate) From 8fb28c742b8b0a2687b7962e0e61c8327ad69f44 Mon Sep 17 00:00:00 2001 From: kishore Date: Fri, 11 Sep 2026 15:49:02 -0700 Subject: [PATCH 06/64] fix(agent): resolve identity through the rest client and hold the follower until it is known Review follow-ups on the standalone panel: - Identity comes from the typed rest client (GET /agent/identity, schema checked, a non-OK or malformed answer throws and is reported) instead of a raw fetch that left the value null in silence. - The follower is held inactive in standalone until that identity resolves. It stamps every canvas op with the actor the agent reported; an op sent before the answer arrived was attributed "anonymous", which the writer refuses. - The flag-gate test awaits setup() instead of polling, and the session tests use typed postMessage mocks. --- src/extensions/core/agentPanel.test.ts | 8 +++---- .../extensions/agent/AgentPanelRoot.vue | 24 +++++++++++++------ .../composables/agent/useAgentSession.test.ts | 17 +++++++++---- .../agent/schemas/agentApiSchema.ts | 6 +++++ .../services/agent/agentRestClient.test.ts | 19 +++++++++++++++ .../agent/services/agent/agentRestClient.ts | 23 +++++++++++++++++- 6 files changed, 79 insertions(+), 18 deletions(-) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index 7bcbb036b4a..7febee9a3ab 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -65,9 +65,6 @@ vi.mock('posthog-js', () => ({ } })) -const flush = (): Promise => - new Promise((resolve) => setTimeout(resolve, 0)) - async function loadEntryAndSetup(): Promise { const { registerAgentPanelExtension } = await import('./agentPanel') registerAgentPanelExtension() @@ -75,8 +72,9 @@ async function loadEntryAndSetup(): Promise { (e) => e.name === 'Comfy.AgentPanel' ) expect(ext).toBeDefined() - ext!.setup!({} as Parameters>[0]) - for (let i = 0; i < 2000 && mocks.flagListener === null; i++) await flush() + // setup() returns the flag gate's promise; awaiting it is the completion + // signal, so no polling is needed. + await ext!.setup!({} as Parameters>[0]) expect(mocks.flagListener).toBeTypeOf('function') } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 82f89d26a7f..6524d729fea 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -1,4 +1,5 @@