From 4c32311fef0ba9d3c70da02bcc0aab041efc942f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 Oct 2026 16:07:35 -0700 Subject: [PATCH 01/52] feat(api): send the workflow id as opaque prompt metadata Core now echoes an opaque `workflow_metadata` dict onto every JSON WebSocket message sent while a prompt runs (Comfy-Org/ComfyUI#16763), so `queuePrompt` sends `{ workflow_id }` and every execution frame comes back tagged with the workflow that produced it. The field is omitted when the workflow has no id, and `extra_pnginfo.workflow.id` is still sent so older backends and `/api/jobs` are unaffected. --- src/scripts/api.queuePrompt.test.ts | 80 +++++++++++++++++++++++++++++ src/scripts/api.ts | 9 ++++ 2 files changed, 89 insertions(+) create mode 100644 src/scripts/api.queuePrompt.test.ts diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts new file mode 100644 index 00000000000..ac804d5d4d9 --- /dev/null +++ b/src/scripts/api.queuePrompt.test.ts @@ -0,0 +1,80 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ComfyApiWorkflow, + ComfyWorkflowJSON +} from '@/platform/workflow/validation/schemas/workflowSchema' +import { api } from '@/scripts/api' + +// Tests for the workflow_metadata field api.queuePrompt sends; fetchApi is stubbed. +const WORKFLOW_ID = '0199e3a3-6c01-7000-8000-c1a1f7a8d9b2' +const EMPTY_PROMPT: ComfyApiWorkflow = {} + +const promptResponse = () => + ({ + ok: true, + status: 200, + json: () => Promise.resolve({ prompt_id: 'p1', number: 1, node_errors: {} }) + }) as unknown as Response + +function workflow(id?: string): ComfyWorkflowJSON { + return { + id, + revision: 0, + last_node_id: 0, + last_link_id: 0, + nodes: [], + links: [], + groups: [], + config: {}, + extra: {}, + version: 1 + } +} + +function sentBody( + fetchApiSpy: ReturnType +): Record { + const [, init] = fetchApiSpy.mock.calls[0] as [string, RequestInit] + return JSON.parse(String(init.body)) as Record +} + +describe('api.queuePrompt workflow_metadata', () => { + let fetchApiSpy: ReturnType + + beforeEach(() => { + fetchApiSpy = vi.spyOn(api, 'fetchApi').mockResolvedValue(promptResponse()) + }) + + it('sends the workflow id as opaque metadata', async () => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(WORKFLOW_ID) + }) + + expect(sentBody(fetchApiSpy).workflow_metadata).toEqual({ + workflow_id: WORKFLOW_ID + }) + }) + + it('omits the field entirely when the workflow has no id', async () => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(undefined) + }) + + expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') + }) + + it('still sends the id inside extra_pnginfo for older backends', async () => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(WORKFLOW_ID) + }) + + const body = sentBody(fetchApiSpy) as { + extra_data: { extra_pnginfo: { workflow: { id?: string } } } + } + expect(body.extra_data.extra_pnginfo.workflow.id).toBe(WORKFLOW_ID) + }) +}) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 9946c51b24d..2ba884653cf 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -97,6 +97,14 @@ interface QueuePromptRequestBody { client_id: string prompt: ComfyApiWorkflow partial_execution_targets?: NodeExecutionId[] + /** + * Opaque key/values the server echoes on every JSON WebSocket message sent + * while this prompt runs. `workflow_id` is the only key the frontend relies + * on; the server never interprets the contents. Omitted when the workflow + * has no id, and capped by the server at 256 bytes. + */ + workflow_metadata?: Record + extra_data: { extra_pnginfo: { workflow: ComfyWorkflowJSON @@ -1342,6 +1350,7 @@ export class ComfyApi extends EventTarget { ...(options?.partialExecutionTargets && { partial_execution_targets: options.partialExecutionTargets }), + ...(workflow.id && { workflow_metadata: { workflow_id: workflow.id } }), extra_data: { auth_token_comfy_org: this.authToken, api_key_comfy_org: this.apiKey, From d5e3b3f0f60ea9ecbaf8d74a8c5e997eee61f323 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 Oct 2026 16:07:51 -0700 Subject: [PATCH 02/52] test(e2e): add a scriptable backend simulator for execution concurrency Stuck node progress, progress leaking between workflow tabs, and outputs applied to the wrong workflow only reproduce under specific frame orders, drops and overlaps, which is why they survive hand-testing. ExecutionHelper can now stamp `workflow_id` on the frames it sends via a `withWorkflowId` scope, and BackendSimulator turns a scenario into an explicit frame script with drop, duplicate, swap and seeded interleave helpers. `status` and binary frames stay unstamped, matching core. --- .../fixtures/helpers/BackendSimulator.test.ts | 200 ++++++++++++++++++ .../fixtures/helpers/BackendSimulator.ts | 166 +++++++++++++++ .../fixtures/helpers/ExecutionHelper.ts | 128 ++++++----- 3 files changed, 426 insertions(+), 68 deletions(-) create mode 100644 browser_tests/fixtures/helpers/BackendSimulator.test.ts create mode 100644 browser_tests/fixtures/helpers/BackendSimulator.ts diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts new file mode 100644 index 00000000000..bfa054cca3d --- /dev/null +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -0,0 +1,200 @@ +import type { WebSocketRoute } from '@playwright/test' +import { describe, expect, it } from 'vitest' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import { + BackendSimulator, + dropFrames, + duplicateFrame, + interleave, + swapFrames +} from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' + +interface SentFrame { + type: string + data: Record +} + +function harness() { + const sent: SentFrame[] = [] + const ws = { + send: (message: string | Buffer) => { + if (typeof message === 'string') sent.push(JSON.parse(message)) + } + } as unknown as WebSocketRoute + const execution = new ExecutionHelper({} as unknown as ComfyPage, ws) + return { sent, simulator: new BackendSimulator(execution), execution } +} + +describe('BackendSimulator frame scripting', () => { + it('stamps each prompt with its own workflow id', () => { + const { sent, simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + + simulator.play([a.start(), b.start(), a.success(), b.success()]) + + expect( + sent.map((frame) => [ + frame.type, + frame.data.prompt_id, + frame.data.workflow_id + ]) + ).toEqual([ + ['execution_start', 'job-a', 'wf-a'], + ['execution_start', 'job-b', 'wf-b'], + ['execution_success', 'job-a', 'wf-a'], + ['execution_success', 'job-b', 'wf-b'] + ]) + }) + + it('omits workflow_id for a prompt with no metadata (legacy backend)', () => { + const { sent, simulator } = harness() + const legacy = simulator.prompt('job-a') + + simulator.play([legacy.start(), legacy.progress('1', 1, 4)]) + + expect(sent.every((frame) => !('workflow_id' in frame.data))).toBe(true) + }) + + it('leaves the metadata scope clean after each frame', () => { + const { sent, simulator } = harness() + const scoped = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const legacy = simulator.prompt('job-b') + + simulator.play([scoped.start(), legacy.start()]) + + expect(sent[1].data.workflow_id).toBeUndefined() + }) + + it('never lets metadata overwrite a frame field', () => { + const { sent, simulator } = harness() + // `workflow_id` is spread first, exactly as core does, so a real field wins. + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + simulator.play([prompt.executing('7')]) + + expect(sent[0].data).toMatchObject({ + prompt_id: 'job-a', + node: '7', + workflow_id: 'wf-a' + }) + }) + + it('does not stamp status frames', () => { + const { sent, simulator, execution } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + simulator.play([prompt.start()]) + execution.status(0) + + expect(sent[1].type).toBe('status') + expect(sent[1].data.workflow_id).toBeUndefined() + }) + + it('drops a frame the backend never sent', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [ + prompt.start(), + prompt.progress('1', 1, 4), + prompt.success() + ] + + simulator.play(dropFrames(script, 'wf-a:execution_success')) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'progress' + ]) + }) + + it('duplicates a frame', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + simulator.play( + duplicateFrame( + [prompt.start(), prompt.success()], + 'wf-a:execution_success' + ) + ) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'execution_success', + 'execution_success' + ]) + }) + + it('swaps two frames to produce out-of-order arrival', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start(), prompt.executed('1', {}), prompt.success()] + + simulator.play( + swapFrames(script, 'wf-a:executed', 'wf-a:execution_success') + ) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'execution_success', + 'executed' + ]) + }) + + it('throws rather than silently skipping an unknown swap target', () => { + const { simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + expect(() => + swapFrames([prompt.start()], 'wf-a:execution_start', 'wf-a:nope') + ).toThrow(/not in the script/) + }) + + it('interleaves deterministically for a given seed', () => { + const { sent, simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + const script = interleave( + [a.start(), a.progress('1', 1, 4), a.success()], + [b.start(), b.progress('2', 1, 4), b.success()], + 7 + ) + + simulator.play(script) + const firstRun = sent.map((frame) => frame.data.workflow_id) + + const second = harness() + const a2 = second.simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b2 = second.simulator.prompt('job-b', { workflowId: 'wf-b' }) + second.simulator.play( + interleave( + [a2.start(), a2.progress('1', 1, 4), a2.success()], + [b2.start(), b2.progress('2', 1, 4), b2.success()], + 7 + ) + ) + + expect(second.sent.map((frame) => frame.data.workflow_id)).toEqual(firstRun) + expect(firstRun).toHaveLength(6) + }) + + it('keeps every frame exactly once when interleaving', () => { + const { sent, simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + + simulator.play( + interleave([a.start(), a.success()], [b.start(), b.success()], 42) + ) + + expect( + sent.filter((frame) => frame.data.workflow_id === 'wf-a') + ).toHaveLength(2) + expect( + sent.filter((frame) => frame.data.workflow_id === 'wf-b') + ).toHaveLength(2) + }) +}) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts new file mode 100644 index 00000000000..4245df8faad --- /dev/null +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -0,0 +1,166 @@ +import type { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' + +/** + * One scheduled WebSocket frame. Frames are built up front and sent by + * {@link BackendSimulator.play}, so a test states the exact arrival order of + * two prompts' messages instead of relying on timing. + */ +export interface Frame { + /** `:` — used by the fault helpers. */ + label: string + send: () => void +} + +/** + * A prompt the simulated backend is running. Every method returns a {@link Frame} + * rather than sending immediately, and each frame carries this prompt's + * `workflow_id` when one was given. + */ +export class SimulatedPrompt { + constructor( + private readonly execution: ExecutionHelper, + readonly jobId: string, + readonly workflowId?: string + ) {} + + private frame(type: string, send: () => void): Frame { + return { + label: `${this.workflowId ?? this.jobId}:${type}`, + send: () => this.execution.withWorkflowId(this.workflowId, send) + } + } + + start(): Frame { + return this.frame('execution_start', () => + this.execution.executionStart(this.jobId) + ) + } + + executing(nodeId: string | null): Frame { + return this.frame('executing', () => + this.execution.executing(this.jobId, nodeId) + ) + } + + progress(nodeId: string, value: number, max: number): Frame { + return this.frame('progress', () => + this.execution.progress(this.jobId, nodeId, value, max) + ) + } + + nodeRunning(nodeId: string, value: number, max: number): Frame { + return this.frame('progress_state', () => + this.execution.nodeRunning(this.jobId, nodeId, value, max) + ) + } + + executed( + nodeId: string, + output: Record | null | undefined + ): Frame { + return this.frame('executed', () => + this.execution.executed(this.jobId, nodeId, output) + ) + } + + success(): Frame { + return this.frame('execution_success', () => + this.execution.executionSuccess(this.jobId) + ) + } + + error(nodeId: string, message: string): Frame { + return this.frame('execution_error', () => + this.execution.executionError(this.jobId, nodeId, message) + ) + } + + interrupted(nodeId: string): Frame { + return this.frame('execution_interrupted', () => + this.execution.executionInterrupted(this.jobId, nodeId) + ) + } + + latentPreview(nodeId: string): Frame { + return this.frame('b_preview', () => + this.execution.latentPreview(this.jobId, nodeId) + ) + } +} + +/** + * Scripts the WebSocket side of execution for one or more prompts. + * + * The bugs this exists for — node progress stuck after a run, progress from one + * workflow tab landing on another, outputs applied to the wrong workflow — only + * reproduce under specific frame orders, drops and overlaps. Hand-testing cannot + * produce them reliably, so they are expressed as explicit frame scripts here. + * + * Socket drop/reconnect is not modelled: use `nextWebSocket` from the + * `webSocketFixture` to arm a waiter, then close the current route. + */ +export class BackendSimulator { + constructor(private readonly execution: ExecutionHelper) {} + + /** + * Declare a prompt the backend is running. `jobId` is what + * {@link ExecutionHelper.run} returned for the tab that submitted it. + * Omit `workflowId` to emulate a core build that does not send metadata. + */ + prompt( + jobId: string, + options: { workflowId?: string } = {} + ): SimulatedPrompt { + return new SimulatedPrompt(this.execution, jobId, options.workflowId) + } + + /** Send frames in the given order. */ + play(frames: Frame[]): void { + for (const frame of frames) frame.send() + } +} + +/** Remove every frame with this label — a backend that never sent it. */ +export function dropFrames(frames: Frame[], label: string): Frame[] { + return frames.filter((frame) => frame.label !== label) +} + +/** Send a frame twice, as a retrying or reconnecting backend does. */ +export function duplicateFrame(frames: Frame[], label: string): Frame[] { + return frames.flatMap((frame) => + frame.label === label ? [frame, frame] : [frame] + ) +} + +/** Swap two frames, producing out-of-order arrival. */ +export function swapFrames( + frames: Frame[], + first: string, + second: string +): Frame[] { + const a = frames.findIndex((frame) => frame.label === first) + const b = frames.findIndex((frame) => frame.label === second) + if (a === -1 || b === -1) { + throw new Error(`swapFrames: ${first} or ${second} is not in the script`) + } + const swapped = [...frames] + ;[swapped[a], swapped[b]] = [swapped[b], swapped[a]] + return swapped +} + +/** + * Deterministic interleave of two prompts' frames. `seed` selects the order, so + * a failure is reproducible from the seed printed in the test name. + */ +export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { + const out: Frame[] = [] + let state = seed + let i = 0 + let j = 0 + while (i < a.length || j < b.length) { + state = (state * 1103515245 + 12345) & 0x7fffffff + const takeA = j >= b.length || (i < a.length && state % 2 === 0) + out.push(takeA ? a[i++] : b[j++]) + } + return out +} diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index ec6e056bb66..145996c1592 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -45,6 +45,7 @@ export function buildKSamplerError( */ export class ExecutionHelper { private jobCounter = 0 + private workflowId: string | undefined private readonly completedJobs: RawJobListItem[] = [] private readonly page: ComfyPage['page'] private readonly command: ComfyPage['command'] @@ -69,6 +70,33 @@ export class ExecutionHelper { return this.ws } + /** + * Stamp `workflow_id` on every JSON frame sent inside `fn`, the way core's + * `send_sync` spreads a prompt's `workflow_metadata` onto outgoing messages. + * Leave `workflowId` undefined to emit frames the way a core build without + * that support does — the legacy backend profile. + */ + withWorkflowId(workflowId: string | undefined, fn: () => T): T { + const previous = this.workflowId + this.workflowId = workflowId + try { + return fn() + } finally { + this.workflowId = previous + } + } + + /** + * Metadata first so a frame's own fields always win on collision, matching + * core's `{**workflow_metadata, **data}`. + */ + private emit(type: string, data: Record): void { + const payload = this.workflowId + ? { workflow_id: this.workflowId, ...data } + : data + this.requireWs().send(JSON.stringify({ type, data: payload })) + } + /** * Intercept POST /api/prompt, execute Comfy.QueuePrompt, and return * the synthetic job ID. @@ -168,22 +196,12 @@ export class ExecutionHelper { /** Send `execution_start` WS event. */ executionStart(jobId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_start', - data: { prompt_id: jobId, timestamp: Date.now() } - }) - ) + this.emit('execution_start', { prompt_id: jobId, timestamp: Date.now() }) } /** Send `executing` WS event to signal which node is currently running. */ executing(jobId: string, nodeId: string | null): void { - this.requireWs().send( - JSON.stringify({ - type: 'executing', - data: { prompt_id: jobId, node: nodeId } - }) - ) + this.emit('executing', { prompt_id: jobId, node: nodeId }) } /** Send `executed` WS event with node output. */ @@ -192,45 +210,30 @@ export class ExecutionHelper { nodeId: string, output: Record | null | undefined ): void { - this.requireWs().send( - JSON.stringify({ - type: 'executed', - data: { - prompt_id: jobId, - node: nodeId, - display_node: nodeId, - output - } - }) - ) + this.emit('executed', { + prompt_id: jobId, + node: nodeId, + display_node: nodeId, + output + }) } /** Send `execution_success` WS event. */ executionSuccess(jobId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_success', - data: { prompt_id: jobId, timestamp: Date.now() } - }) - ) + this.emit('execution_success', { prompt_id: jobId, timestamp: Date.now() }) } /** Send `execution_error` WS event. */ executionError(jobId: string, nodeId: string, message: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_error', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - exception_message: message, - exception_type: 'RuntimeError', - traceback: [] - } - }) - ) + this.emit('execution_error', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + exception_message: message, + exception_type: 'RuntimeError', + traceback: [] + }) } /** Send `execution_error` WS event carrying cloud validation node errors. */ @@ -257,28 +260,18 @@ export class ExecutionHelper { /** Send `execution_interrupted` WS event (user-initiated stop). */ executionInterrupted(jobId: string, nodeId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_interrupted', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - executed: [] - } - }) - ) + this.emit('execution_interrupted', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + executed: [] + }) } /** Send `progress` WS event. */ progress(jobId: string, nodeId: string, value: number, max: number): void { - this.requireWs().send( - JSON.stringify({ - type: 'progress', - data: { prompt_id: jobId, node: nodeId, value, max } - }) - ) + this.emit('progress', { prompt_id: jobId, node: nodeId, value, max }) } /** @@ -301,12 +294,7 @@ export class ExecutionHelper { /** Send `progress_state` WS event with per-node execution state. */ progressState(jobId: string, nodes: Record): void { - this.requireWs().send( - JSON.stringify({ - type: 'progress_state', - data: { prompt_id: jobId, nodes } - }) - ) + this.emit('progress_state', { prompt_id: jobId, nodes }) } /** @@ -339,7 +327,11 @@ export class ExecutionHelper { this.status(0) } - /** Send `status` WS event to update queue count. */ + /** + * Send `status` WS event to update queue count. Deliberately NOT stamped with + * `workflow_id`: core exempts `status` because one server-wide socket serves + * every workflow, so a queue frame has no single owner. + */ status(queueRemaining: number): void { this.requireWs().send( JSON.stringify({ From d1b46561f6d3f0588b8bbde0bef5b2d331ab366f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 Oct 2026 18:38:11 -0700 Subject: [PATCH 03/52] test(e2e): make the execution concurrency harness actually concurrent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways the harness could pass while exercising nothing. `interleave` multiplied a 32-bit state by 1103515245 as a double. That product exceeds 2^53, so the low bits were rounded away and the `state % 2` coin came out constant: every seed emitted all of `a` before any of `b`, and a "concurrency" schedule was sequential. Use `Math.imul`, read a high bit rather than bit 0 (bit 0 of a power-of-two-modulus LCG flips every step whatever the seed, which only trades a constant coin for a fixed A/B/A/B), and guarantee overlap directly: neither prompt's last frame is emitted while the other has not started. Seven seeds now produce six distinct schedules, all of them overlapping, each preserving its own prompt's dependency order. Frames were selected by label alone, and two concurrent prompts of the same workflow share a label — the case the harness exists for. Frames now carry their `jobId` and the fault helpers take `{ label, jobId }`, with a bare label keeping the workflow-wide behaviour. A selector that matches nothing throws instead of silently applying no fault, and `swapFrames` throws on an ambiguous selector rather than quietly reordering the wrong prompt's frame. `withWorkflowId` restored the previous scope as soon as its callback returned, so an async callback lost the scope at its first `await` and every later frame was unstamped or mis-stamped. The scope now stays open until a returned promise settles; opening a second scope while one is in flight throws rather than guessing, since without ambient async context overlapping scopes cannot be attributed to their prompts. Two assertions were vacuous. The collision test drove `executing()`, which cannot supply a competing top-level `workflow_id`, so nothing collided; it now goes through the general `withMetadata` form with `prompt_id` and `node` both spoofed. The `status` test asserted an absent `workflow_id` with no scope open, which is true of every frame; `status` now routes through `emit` inside a live scope and comes out unstamped because it carries no `prompt_id` — core's own gate, so the exemption cannot drift away from the list of frame builders. Each guard was verified by mutation: reverting any one of the seven fixes turns at least one test red. Co-Authored-By: Claude Opus 5 (1M context) --- .../fixtures/helpers/BackendSimulator.test.ts | 295 +++++++++++++++--- .../fixtures/helpers/BackendSimulator.ts | 133 ++++++-- .../fixtures/helpers/ExecutionHelper.ts | 113 +++++-- 3 files changed, 460 insertions(+), 81 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index bfa054cca3d..9877f36bfbc 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -16,6 +16,9 @@ interface SentFrame { data: Record } +/** Seeds every schedule assertion runs over, so none of them can be lucky. */ +const SEEDS = [0, 1, 3, 7, 42, 99, 12345] + function harness() { const sent: SentFrame[] = [] const ws = { @@ -27,6 +30,13 @@ function harness() { return { sent, simulator: new BackendSimulator(execution), execution } } +/** The positions a workflow's frames occupy in the order they were sent. */ +function positionsOf(sent: SentFrame[], workflowId: string): number[] { + return sent.flatMap((frame, index) => + frame.data.workflow_id === workflowId ? [index] : [] + ) +} + describe('BackendSimulator frame scripting', () => { it('stamps each prompt with its own workflow id', () => { const { sent, simulator } = harness() @@ -55,6 +65,7 @@ describe('BackendSimulator frame scripting', () => { simulator.play([legacy.start(), legacy.progress('1', 1, 4)]) + expect(sent).toHaveLength(2) expect(sent.every((frame) => !('workflow_id' in frame.data))).toBe(true) }) @@ -65,32 +76,75 @@ describe('BackendSimulator frame scripting', () => { simulator.play([scoped.start(), legacy.start()]) - expect(sent[1].data.workflow_id).toBeUndefined() + expect(sent[0].data.workflow_id).toBe('wf-a') + expect(sent[1].data).not.toHaveProperty('workflow_id') }) it('never lets metadata overwrite a frame field', () => { - const { sent, simulator } = harness() - // `workflow_id` is spread first, exactly as core does, so a real field wins. - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - - simulator.play([prompt.executing('7')]) + const { sent, execution } = harness() + // Core merges `{**workflow_metadata, **data}`, so a client cannot hijack a + // frame's own routing fields by naming them in its metadata. Reversing the + // spread order in `emit` turns this red. + execution.withMetadata( + { workflow_id: 'wf-a', prompt_id: 'spoofed', node: 'spoofed' }, + () => execution.executing('job-a', '7') + ) - expect(sent[0].data).toMatchObject({ + expect(sent[0].data).toEqual({ + workflow_id: 'wf-a', prompt_id: 'job-a', - node: '7', - workflow_id: 'wf-a' + node: '7' }) }) - it('does not stamp status frames', () => { - const { sent, simulator, execution } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + it('does not stamp status frames, even inside an active scope', () => { + const { sent, execution } = harness() - simulator.play([prompt.start()]) - execution.status(0) + execution.withWorkflowId('wf-a', () => { + execution.executionStart('job-a') + execution.status(2) + }) + // The first frame proves the scope was live when `status` was sent, so the + // second assertion is about the exemption and not about an empty scope. + expect(sent[0].data.workflow_id).toBe('wf-a') expect(sent[1].type).toBe('status') - expect(sent[1].data.workflow_id).toBeUndefined() + expect(sent[1].data).not.toHaveProperty('workflow_id') + }) + + it('keeps the workflow id scope open across awaits', async () => { + const { sent, execution } = harness() + + await execution.withWorkflowId('wf-a', async () => { + execution.executionStart('job-a') + await Promise.resolve() + execution.executionSuccess('job-a') + }) + execution.executionStart('job-b') + + expect(sent.map((frame) => frame.data.workflow_id)).toEqual([ + 'wf-a', + 'wf-a', + undefined + ]) + }) + + it('refuses a second scope while an async one is still in flight', async () => { + const { execution } = harness() + let release!: () => void + const blocked = new Promise((resolve) => { + release = resolve + }) + + const scope = execution.withWorkflowId('wf-a', () => blocked) + expect(() => execution.withWorkflowId('wf-b', () => {})).toThrow( + /in flight/ + ) + + release() + await scope + + expect(() => execution.withWorkflowId('wf-b', () => {})).not.toThrow() }) it('drops a frame the backend never sent', () => { @@ -110,6 +164,31 @@ describe('BackendSimulator frame scripting', () => { ]) }) + it('drops one job of two prompts sharing a workflow id', () => { + const { sent, simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) + const script = [ + first.start(), + second.start(), + first.success(), + second.success() + ] + + simulator.play( + dropFrames(script, { + label: 'wf-a:execution_success', + jobId: 'job-a' + }) + ) + + expect(sent.map((frame) => [frame.type, frame.data.prompt_id])).toEqual([ + ['execution_start', 'job-a'], + ['execution_start', 'job-b'], + ['execution_success', 'job-b'] + ]) + }) + it('duplicates a frame', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -128,6 +207,25 @@ describe('BackendSimulator frame scripting', () => { ]) }) + it('duplicates one job of two prompts sharing a workflow id', () => { + const { sent, simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) + + simulator.play( + duplicateFrame([first.success(), second.success()], { + label: 'wf-a:execution_success', + jobId: 'job-b' + }) + ) + + expect(sent.map((frame) => frame.data.prompt_id)).toEqual([ + 'job-a', + 'job-b', + 'job-b' + ]) + }) + it('swaps two frames to produce out-of-order arrival', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -144,6 +242,34 @@ describe('BackendSimulator frame scripting', () => { ]) }) + it('swaps one job of two prompts sharing a workflow id', () => { + const { sent, simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) + const script = [ + first.start(), + second.start(), + first.success(), + second.success() + ] + + // job-a terminates before job-b is even announced. + simulator.play( + swapFrames( + script, + { label: 'wf-a:execution_start', jobId: 'job-b' }, + { label: 'wf-a:execution_success', jobId: 'job-a' } + ) + ) + + expect(sent.map((frame) => [frame.type, frame.data.prompt_id])).toEqual([ + ['execution_start', 'job-a'], + ['execution_success', 'job-a'], + ['execution_start', 'job-b'], + ['execution_success', 'job-b'] + ]) + }) + it('throws rather than silently skipping an unknown swap target', () => { const { simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -153,32 +279,123 @@ describe('BackendSimulator frame scripting', () => { ).toThrow(/not in the script/) }) - it('interleaves deterministically for a given seed', () => { - const { sent, simulator } = harness() - const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) - const script = interleave( - [a.start(), a.progress('1', 1, 4), a.success()], - [b.start(), b.progress('2', 1, 4), b.success()], - 7 - ) + it('throws rather than swapping an ambiguously selected frame', () => { + const { simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) - simulator.play(script) - const firstRun = sent.map((frame) => frame.data.workflow_id) + expect(() => + swapFrames( + [first.start(), second.start(), first.success()], + 'wf-a:execution_start', + 'wf-a:execution_success' + ) + ).toThrow(/matches 2 frames/) + }) + + it('throws rather than silently applying no fault at all', () => { + const { simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start()] - const second = harness() - const a2 = second.simulator.prompt('job-a', { workflowId: 'wf-a' }) - const b2 = second.simulator.prompt('job-b', { workflowId: 'wf-b' }) - second.simulator.play( + expect(() => dropFrames(script, 'wf-a:execution_success')).toThrow( + /not in the script/ + ) + expect(() => + duplicateFrame(script, { + label: 'wf-a:execution_start', + jobId: 'job-zzz' + }) + ).toThrow(/not in the script/) + }) +}) + +describe('BackendSimulator interleaving', () => { + function twoPrompts(seed: number) { + const h = harness() + const a = h.simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = h.simulator.prompt('job-b', { workflowId: 'wf-b' }) + h.simulator.play( interleave( - [a2.start(), a2.progress('1', 1, 4), a2.success()], - [b2.start(), b2.progress('2', 1, 4), b2.success()], - 7 + [a.start(), a.progress('1', 1, 4), a.success()], + [b.start(), b.progress('2', 1, 4), b.success()], + seed + ) + ) + return h.sent + } + + it('produces the documented overlapping order for seed 7', () => { + expect( + twoPrompts(7).map((frame) => `${frame.data.workflow_id}:${frame.type}`) + ).toEqual([ + 'wf-a:execution_start', + 'wf-a:progress', + 'wf-b:execution_start', + 'wf-a:execution_success', + 'wf-b:progress', + 'wf-b:execution_success' + ]) + }) + + it.for(SEEDS)('keeps both prompts in flight at once for seed %i', (seed) => { + const sent = twoPrompts(seed) + const a = positionsOf(sent, 'wf-a') + const b = positionsOf(sent, 'wf-b') + + // The two runs' spans intersect: neither prompt's script finishes before + // the other's starts. Without this the schedule is sequential and a + // concurrency spec can pass having never had two prompts in flight. + expect(a[0]).toBeLessThan(b[b.length - 1]) + expect(b[0]).toBeLessThan(a[a.length - 1]) + }) + + it.for(SEEDS)( + "preserves each prompt's own dependency order for seed %i", + (seed) => { + const sent = twoPrompts(seed) + const typesOf = (workflowId: string) => + sent + .filter((frame) => frame.data.workflow_id === workflowId) + .map((frame) => frame.type) + + expect(typesOf('wf-a')).toEqual([ + 'execution_start', + 'progress', + 'execution_success' + ]) + expect(typesOf('wf-b')).toEqual([ + 'execution_start', + 'progress', + 'execution_success' + ]) + } + ) + + it('interleaves deterministically for a given seed', () => { + const first = twoPrompts(7).map((frame) => frame.data.workflow_id) + const second = twoPrompts(7).map((frame) => frame.data.workflow_id) + + expect(second).toEqual(first) + expect(first).toHaveLength(6) + }) + + it('gives different seeds different schedules', () => { + const schedules = new Set( + SEEDS.map((seed) => + twoPrompts(seed) + .map((frame) => frame.data.workflow_id) + .join(',') ) ) - expect(second.sent.map((frame) => frame.data.workflow_id)).toEqual(firstRun) - expect(firstRun).toHaveLength(6) + // `seed` has to name a schedule, so near-distinct is the bar: these seven + // seeds produce six distinct orders. A coin that merely alternates — LCG + // bit 0, which flips every step whatever the seed — collapses them to two + // (`ababab` and `bababa`) and still passes a `> 1` check. The exact-32-bit + // multiply is pinned separately by the seed 7 order above; this assertion + // is about the coin, not the arithmetic. + expect(schedules.size).toBeGreaterThanOrEqual(5) }) it('keeps every frame exactly once when interleaving', () => { @@ -190,11 +407,7 @@ describe('BackendSimulator frame scripting', () => { interleave([a.start(), a.success()], [b.start(), b.success()], 42) ) - expect( - sent.filter((frame) => frame.data.workflow_id === 'wf-a') - ).toHaveLength(2) - expect( - sent.filter((frame) => frame.data.workflow_id === 'wf-b') - ).toHaveLength(2) + expect(positionsOf(sent, 'wf-a')).toHaveLength(2) + expect(positionsOf(sent, 'wf-b')).toHaveLength(2) }) }) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts index 4245df8faad..40fb54ab7cc 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -8,9 +8,62 @@ import type { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' export interface Frame { /** `:` — used by the fault helpers. */ label: string + /** + * The prompt that sent this frame. Two concurrent prompts of the *same* + * workflow share a label, so a fault that must hit one of them selects on + * this as well — see {@link FrameSelector}. + */ + jobId: string send: () => void } +/** + * Which frames a fault applies to. A bare string keeps the workflow-wide + * selection every script already uses; add `jobId` to reach one prompt when + * several share a workflow id. + */ +export type FrameSelector = string | { label: string; jobId?: string } + +function describeSelector(selector: FrameSelector): string { + return typeof selector === 'string' + ? selector + : selector.jobId === undefined + ? selector.label + : `${selector.label} (job ${selector.jobId})` +} + +function matches(frame: Frame, selector: FrameSelector): boolean { + if (typeof selector === 'string') return frame.label === selector + if (frame.label !== selector.label) return false + return selector.jobId === undefined || frame.jobId === selector.jobId +} + +function indicesOf(frames: Frame[], selector: FrameSelector): number[] { + const found: number[] = [] + frames.forEach((frame, index) => { + if (matches(frame, selector)) found.push(index) + }) + return found +} + +/** + * A fault that silently matches nothing is a test that silently stops + * exercising its fault, so every helper fails loudly instead. + */ +function requireMatches( + frames: Frame[], + selector: FrameSelector, + helper: string +): number[] { + const found = indicesOf(frames, selector) + if (found.length === 0) { + throw new Error( + `${helper}: ${describeSelector(selector)} is not in the script` + ) + } + return found +} + /** * A prompt the simulated backend is running. Every method returns a {@link Frame} * rather than sending immediately, and each frame carries this prompt's @@ -26,6 +79,7 @@ export class SimulatedPrompt { private frame(type: string, send: () => void): Frame { return { label: `${this.workflowId ?? this.jobId}:${type}`, + jobId: this.jobId, send: () => this.execution.withWorkflowId(this.workflowId, send) } } @@ -120,46 +174,89 @@ export class BackendSimulator { } } -/** Remove every frame with this label — a backend that never sent it. */ -export function dropFrames(frames: Frame[], label: string): Frame[] { - return frames.filter((frame) => frame.label !== label) +/** + * Remove every selected frame — a backend that never sent it. Pass a `jobId` to + * drop one prompt's frame while a concurrent prompt of the same workflow keeps + * its own. + */ +export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { + const drop = new Set(requireMatches(frames, selector, 'dropFrames')) + return frames.filter((_, index) => !drop.has(index)) } -/** Send a frame twice, as a retrying or reconnecting backend does. */ -export function duplicateFrame(frames: Frame[], label: string): Frame[] { - return frames.flatMap((frame) => - frame.label === label ? [frame, frame] : [frame] +/** Send selected frames twice, as a retrying or reconnecting backend does. */ +export function duplicateFrame( + frames: Frame[], + selector: FrameSelector +): Frame[] { + const duplicate = new Set(requireMatches(frames, selector, 'duplicateFrame')) + return frames.flatMap((frame, index) => + duplicate.has(index) ? [frame, frame] : [frame] ) } -/** Swap two frames, producing out-of-order arrival. */ +/** + * Swap two frames, producing out-of-order arrival. + * + * Each selector must identify exactly one frame. A swap is a pair operation, so + * quietly taking the first of several matches is how a two-prompt script ends up + * reordering the wrong prompt's frame; add a `jobId` to disambiguate. + */ export function swapFrames( frames: Frame[], - first: string, - second: string + first: FrameSelector, + second: FrameSelector ): Frame[] { - const a = frames.findIndex((frame) => frame.label === first) - const b = frames.findIndex((frame) => frame.label === second) - if (a === -1 || b === -1) { - throw new Error(`swapFrames: ${first} or ${second} is not in the script`) + const a = requireMatches(frames, first, 'swapFrames') + const b = requireMatches(frames, second, 'swapFrames') + for (const [selector, found] of [ + [first, a], + [second, b] + ] as const) { + if (found.length > 1) { + throw new Error( + `swapFrames: ${describeSelector(selector)} matches ${found.length} frames; ` + + 'add a jobId to select one' + ) + } } const swapped = [...frames] - ;[swapped[a], swapped[b]] = [swapped[b], swapped[a]] + ;[swapped[a[0]], swapped[b[0]]] = [swapped[b[0]], swapped[a[0]]] return swapped } /** * Deterministic interleave of two prompts' frames. `seed` selects the order, so * a failure is reproducible from the seed printed in the test name. + * + * Each input's own order is preserved — frames of one prompt never arrive out of + * dependency order relative to each other, only relative to the other prompt's. + * + * The result is guaranteed to **overlap**: neither prompt's script finishes + * before the other's starts, so the two runs are genuinely concurrent. Without + * that guarantee a schedule can come out sequential and a concurrency spec + * passes without ever having two prompts in flight. (A one-frame-each pair + * cannot overlap; everything larger does.) */ export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { const out: Frame[] = [] - let state = seed + let state = seed | 0 let i = 0 let j = 0 while (i < a.length || j < b.length) { - state = (state * 1103515245 + 12345) & 0x7fffffff - const takeA = j >= b.length || (i < a.length && state % 2 === 0) + // Exact 32-bit arithmetic. `state * 1103515245` exceeds 2^53 as a double, + // so a plain multiply rounds the low bits away: every seed then produced + // the same constant coin and the helper emitted all of `a` before any of + // `b`. `Math.imul` is the multiply that does not lose them. + state = (Math.imul(state, 1103515245) + 12345) | 0 + // Bit 0 of a power-of-two-modulus LCG flips on every step whatever the + // seed, which would make the schedule a fixed A/B/A/B. Take a high bit. + let takeA = j >= b.length || (i < a.length && ((state >>> 16) & 1) === 0) + // Overlap guarantee, in two symmetric rules: do not emit one prompt's last + // frame while the other has not started. + if (takeA && b.length > 0 && j === 0 && i === a.length - 1) takeA = false + else if (!takeA && a.length > 0 && i === 0 && j === b.length - 1) + takeA = true out.push(takeA ? a[i++] : b[j++]) } return out diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 145996c1592..163361f409a 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -11,6 +11,14 @@ import { createMockJob } from '@e2e/fixtures/helpers/AssetsHelper' const PROMPT_ROUTE_PATTERN = /\/api\/prompt$/ +function isPromiseLike(value: unknown): value is PromiseLike { + return ( + typeof value === 'object' && + value !== null && + typeof (value as PromiseLike).then === 'function' + ) +} + type RunOptions = { nodeErrors?: Record onPromptRequest?: (requestBody: unknown) => void | Promise @@ -45,7 +53,8 @@ export function buildKSamplerError( */ export class ExecutionHelper { private jobCounter = 0 - private workflowId: string | undefined + private metadata: Record | undefined + private asyncScopeOpen = false private readonly completedJobs: RawJobListItem[] = [] private readonly page: ComfyPage['page'] private readonly command: ComfyPage['command'] @@ -71,29 +80,88 @@ export class ExecutionHelper { } /** - * Stamp `workflow_id` on every JSON frame sent inside `fn`, the way core's - * `send_sync` spreads a prompt's `workflow_metadata` onto outgoing messages. - * Leave `workflowId` undefined to emit frames the way a core build without - * that support does — the legacy backend profile. + * Stamp `workflow_id` on every eligible JSON frame sent inside `fn`, the way + * core's `send_sync` spreads a prompt's `workflow_metadata` onto outgoing + * messages. Leave `workflowId` undefined to emit frames the way a core build + * without that support does — the legacy backend profile. */ withWorkflowId(workflowId: string | undefined, fn: () => T): T { - const previous = this.workflowId - this.workflowId = workflowId + return this.withMetadata( + workflowId === undefined ? undefined : { workflow_id: workflowId }, + fn + ) + } + + /** + * The general form of {@link withWorkflowId}: core accepts an arbitrary + * `workflow_metadata` dict, so a scenario can set keys that collide with a + * frame's own fields and assert the frame still wins. + * + * The scope survives `await`s inside `fn`. A callback that returns a promise + * keeps the scope open until that promise settles, so frames emitted after an + * `await` are still stamped — returning the promise used to restore the + * previous scope immediately, silently unstamping (or mis-stamping) every + * frame after the first suspension point. + * + * Opening a second scope while an async one is still in flight throws rather + * than guessing: without ambient async context two overlapping scopes cannot + * be attributed to their prompts, and the wrong `workflow_id` on a frame is + * precisely the bug this harness exists to catch. Await the first scope, or + * script the prompts as separate synchronous frames via `BackendSimulator`. + */ + withMetadata( + metadata: Record | undefined, + fn: () => T + ): T { + if (this.asyncScopeOpen) { + throw new Error( + 'ExecutionHelper: an async metadata scope is still in flight. ' + + 'Await it before opening another one.' + ) + } + + const previous = this.metadata + this.metadata = metadata + const restore = () => { + this.metadata = previous + } + + let result: T try { - return fn() - } finally { - this.workflowId = previous + result = fn() + } catch (error) { + restore() + throw error + } + + if (!isPromiseLike(result)) { + restore() + return result } + + this.asyncScopeOpen = true + // Cast: `T` is promise-like here, and the returned promise settles with the + // same value or rejection, only after the scope has been restored. + return Promise.resolve(result).finally(() => { + this.asyncScopeOpen = false + restore() + }) as T } /** * Metadata first so a frame's own fields always win on collision, matching * core's `{**workflow_metadata, **data}`. + * + * Only frames carrying a `prompt_id` are stamped, which is core's own gate + * (`send_sync` checks `"prompt_id" in data`) rather than an event-name + * carve-out. `status` is exempt because it has no `prompt_id`, so the + * exemption cannot drift out of sync with the list of frame builders. */ private emit(type: string, data: Record): void { - const payload = this.workflowId - ? { workflow_id: this.workflowId, ...data } - : data + const payload = + this.metadata && 'prompt_id' in data + ? { ...this.metadata, ...data } + : data this.requireWs().send(JSON.stringify({ type, data: payload })) } @@ -328,16 +396,17 @@ export class ExecutionHelper { } /** - * Send `status` WS event to update queue count. Deliberately NOT stamped with - * `workflow_id`: core exempts `status` because one server-wide socket serves - * every workflow, so a queue frame has no single owner. + * Send `status` WS event to update queue count. + * + * Goes through {@link emit} like every other frame and comes out unstamped + * because it carries no `prompt_id` — core exempts it for exactly that + * reason: one server-wide socket serves every workflow, so a queue frame has + * no single owner. Routing it through `emit` is what makes the exemption + * testable; sending it directly would make any assertion about it vacuous. */ status(queueRemaining: number): void { - this.requireWs().send( - JSON.stringify({ - type: 'status', - data: { status: { exec_info: { queue_remaining: queueRemaining } } } - }) - ) + this.emit('status', { + status: { exec_info: { queue_remaining: queueRemaining } } + }) } } From 8272a42977942c8215bef2bd81c249ac85a94d95 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 Oct 2026 18:59:48 -0700 Subject: [PATCH 04/52] test(e2e): one WebSocket frame per scheduled Frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects found reviewing this branch at 87de4bd174, each with a falsifier that is red without the fix. 1. `SimulatedPrompt.nodeRunning` returned one `Frame` that sent two WebSocket frames. `Frame` documents itself as one frame, and every fault helper depends on that: no Frame carried the `progress` label, so `dropFrames(script, 'wf:progress')` threw `not in the script` while a `progress` frame was in fact on the wire, a drop or duplicate of the pair necessarily hit both halves, and `interleave` scheduled four units that emitted six. It now returns the two frames separately. A backend that reports the state change but never the step count is one of the shapes behind progress stuck after a run, which this simulator exists to script, so the halves have to be separable. `ExecutionHelper.nodeRunning` is unchanged for its existing callers; the state payload is factored out as `runningNodeState` so the two paths cannot drift. 2. `swapFrames(script, x, x)` silently returned the script unchanged — both selectors resolve, each unambiguously, and no fault is applied. That is the same silent no-fault `requireMatches` was added to rule out, so it throws too. 3. The `workflow_metadata` doc comment overstated what the server does. Verified against ComfyUI#16763 at 52463316ca: only JSON frames whose `data` object carries a `prompt_id` are stamped, so `status`, `logs` and binary preview frames arrive unstamped and a consumer still needs the prompt_id-to-workflow mapping; and the 256 limit is 256 characters of the server's `json.dumps` of this object, not 256 bytes of content, so punctuation counts and non-ASCII escapes to \uXXXX at six characters each. Also records that #16763 is not yet merged, and that a server without it ignores the field rather than rejecting it. Also takes the open review suggestion to build the fetch stub with `Response.json` instead of a double assertion. A contract test now pins the invariant behind (1) for every frame builder, including ones added later: exactly one raw send per scheduled Frame, and a label naming the event the frame sends. --- .../fixtures/helpers/BackendSimulator.test.ts | 80 +++++++++++++++++++ .../fixtures/helpers/BackendSimulator.ts | 43 ++++++++-- .../fixtures/helpers/ExecutionHelper.ts | 37 ++++++--- src/scripts/api.queuePrompt.test.ts | 13 +-- src/scripts/api.ts | 23 +++++- 5 files changed, 169 insertions(+), 27 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 9877f36bfbc..81fc503b52f 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -2,6 +2,7 @@ import type { WebSocketRoute } from '@playwright/test' import { describe, expect, it } from 'vitest' import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { Frame } from '@e2e/fixtures/helpers/BackendSimulator' import { BackendSimulator, dropFrames, @@ -293,6 +294,18 @@ describe('BackendSimulator frame scripting', () => { ).toThrow(/matches 2 frames/) }) + it('throws rather than swapping a frame with itself', () => { + const { simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start(), prompt.success()] + + // Both selectors resolve, each unambiguously, and the swap is still a + // no-op — the same silent no-fault `requireMatches` rules out. + expect(() => + swapFrames(script, 'wf-a:execution_start', 'wf-a:execution_start') + ).toThrow(/same frame/) + }) + it('throws rather than silently applying no fault at all', () => { const { simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -310,6 +323,73 @@ describe('BackendSimulator frame scripting', () => { }) }) +describe('BackendSimulator frame granularity', () => { + /** Every frame builder on `SimulatedPrompt`, scripted once. */ + function everyFrame(simulator: BackendSimulator): Frame[] { + const p = simulator.prompt('job-a', { workflowId: 'wf-a' }) + return [ + p.start(), + p.executing('1'), + p.progress('1', 1, 4), + ...p.nodeRunning('1', 1, 4), + p.executed('1', {}), + p.success(), + p.error('1', 'boom'), + p.interrupted('1'), + p.latentPreview('1') + ] + } + + it('sends exactly one WebSocket frame per scheduled Frame', () => { + // The contract `Frame` documents, and the one a fault helper depends on: a + // Frame that sent two messages would leave the second unreachable to every + // selector and would make `interleave` schedule fewer units than it emits. + // Counts raw sends, so the binary `latentPreview` is included. + let sends = 0 + const ws = { + send: () => { + sends++ + } + } as unknown as WebSocketRoute + const execution = new ExecutionHelper({} as unknown as ComfyPage, ws) + const simulator = new BackendSimulator(execution) + const script = everyFrame(simulator) + + simulator.play(script) + + expect(sends).toBe(script.length) + }) + + it('gives a frame builder a label matching the event it sends', () => { + const { sent, simulator } = harness() + const script = everyFrame(simulator) + + simulator.play(script) + + // `latentPreview` is binary and never reaches the JSON stub, so compare the + // JSON frames against the labels of the Frames that produce them. + expect(sent.map((frame) => frame.type)).toEqual( + script.slice(0, -1).map((frame) => frame.label.split(':')[1]) + ) + }) + + it('scripts the two halves of a running node independently', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start(), ...prompt.nodeRunning('1', 1, 4)] + + // A backend that reported the state change but never the step count is one + // of the shapes behind progress stuck after a run; it needs the `progress` + // half to be individually droppable. + simulator.play(dropFrames(script, 'wf-a:progress')) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'progress_state' + ]) + }) +}) + describe('BackendSimulator interleaving', () => { function twoPrompts(seed: number) { const h = harness() diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts index 40fb54ab7cc..365fbe22db6 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -102,10 +102,30 @@ export class SimulatedPrompt { ) } - nodeRunning(nodeId: string, value: number, max: number): Frame { - return this.frame('progress_state', () => - this.execution.nodeRunning(this.jobId, nodeId, value, max) - ) + /** + * The two frames a running node produces — `progress_state`, then `progress`. + * + * Returned as two {@link Frame}s rather than one because a Frame is one + * WebSocket frame. A single Frame that sent both would make the `progress` + * half unreachable to every fault helper (no Frame carries its label, so + * `dropFrames(script, 'wf:progress')` would throw `not in the script` while a + * `progress` frame was in fact on the wire), would force a drop or duplicate + * of one half to hit both, and would make {@link interleave} schedule fewer + * units than it emits. Spread it into a script: + * `[p.start(), ...p.nodeRunning('1', 1, 4)]`. + */ + nodeRunning(nodeId: string, value: number, max: number): [Frame, Frame] { + return [ + this.frame('progress_state', () => + this.execution.progressState( + this.jobId, + this.execution.runningNodeState(this.jobId, nodeId, value, max) + ) + ), + this.frame('progress', () => + this.execution.progress(this.jobId, nodeId, value, max) + ) + ] } executed( @@ -198,9 +218,12 @@ export function duplicateFrame( /** * Swap two frames, producing out-of-order arrival. * - * Each selector must identify exactly one frame. A swap is a pair operation, so - * quietly taking the first of several matches is how a two-prompt script ends up - * reordering the wrong prompt's frame; add a `jobId` to disambiguate. + * Each selector must identify exactly one frame, and the two must not be the + * same frame. A swap is a pair operation, so quietly taking the first of several + * matches is how a two-prompt script ends up reordering the wrong prompt's + * frame; add a `jobId` to disambiguate. Swapping a frame with itself is the same + * silent no-fault that {@link requireMatches} exists to rule out, so it throws + * rather than returning the script unchanged. */ export function swapFrames( frames: Frame[], @@ -220,6 +243,12 @@ export function swapFrames( ) } } + if (a[0] === b[0]) { + throw new Error( + `swapFrames: ${describeSelector(first)} and ${describeSelector(second)} ` + + 'select the same frame; swapping it with itself applies no fault' + ) + } const swapped = [...frames] ;[swapped[a[0]], swapped[b[0]]] = [swapped[b[0]], swapped[a[0]]] return swapped diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 163361f409a..39919a7c4f9 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -342,21 +342,38 @@ export class ExecutionHelper { this.emit('progress', { prompt_id: jobId, node: nodeId, value, max }) } + /** + * The `progress_state` payload for a single node in the `running` state. + * + * Exposed separately from {@link nodeRunning} so a caller that needs the two + * events as individually schedulable frames can send them itself without + * rebuilding this payload — see `SimulatedPrompt.nodeRunning`. + */ + runningNodeState( + jobId: string, + nodeId: string, + value: number, + max: number + ): Record { + return { + [nodeId]: { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state: 'running', + value, + max + } + } + } + /** * Put a single node into the `running` state at the given step progress, * emitting both the `progress_state` and `progress` events the backend sends. */ nodeRunning(jobId: string, nodeId: string, value: number, max: number): void { - const state: NodeProgressState = { - node_id: nodeId, - display_node_id: nodeId, - real_node_id: nodeId, - prompt_id: jobId, - state: 'running', - value, - max - } - this.progressState(jobId, { [nodeId]: state }) + this.progressState(jobId, this.runningNodeState(jobId, nodeId, value, max)) this.progress(jobId, nodeId, value, max) } diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts index ac804d5d4d9..181fd1e64d3 100644 --- a/src/scripts/api.queuePrompt.test.ts +++ b/src/scripts/api.queuePrompt.test.ts @@ -10,12 +10,13 @@ import { api } from '@/scripts/api' const WORKFLOW_ID = '0199e3a3-6c01-7000-8000-c1a1f7a8d9b2' const EMPTY_PROMPT: ComfyApiWorkflow = {} -const promptResponse = () => - ({ - ok: true, - status: 200, - json: () => Promise.resolve({ prompt_id: 'p1', number: 1, node_errors: {} }) - }) as unknown as Response +// A real `Response`, so the stub cannot drift out of shape with the contract +// `queuePrompt` reads. A fresh one per test: a Response body reads once. +const promptResponse = (): Response => + Response.json( + { prompt_id: 'p1', number: 1, node_errors: {} }, + { status: 200 } + ) function workflow(id?: string): ComfyWorkflowJSON { return { diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 2ba884653cf..e8ff8007693 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -98,10 +98,25 @@ interface QueuePromptRequestBody { prompt: ComfyApiWorkflow partial_execution_targets?: NodeExecutionId[] /** - * Opaque key/values the server echoes on every JSON WebSocket message sent - * while this prompt runs. `workflow_id` is the only key the frontend relies - * on; the server never interprets the contents. Omitted when the workflow - * has no id, and capped by the server at 256 bytes. + * Opaque key/values the server echoes back on the WebSocket while this + * prompt runs. `workflow_id` is the only key the frontend relies on; the + * server never interprets the contents. Omitted when the workflow has no id. + * + * Three limits of the echo, so a reader does not assume more than the server + * gives (verified against ComfyUI#16763 at `52463316ca`, which adds this + * field and is **not yet merged** — a server without it ignores the field + * rather than rejecting the prompt): + * + * - Only JSON frames whose `data` object carries a `prompt_id` are stamped. + * `status` and `logs` have none, and binary preview frames are not objects, + * so none of those arrive stamped. A consumer still needs the + * `prompt_id`-to-workflow mapping for those. + * - A frame's own fields win on collision (`{**workflow_metadata, **data}`), + * so these values cannot override a frame's routing fields. + * - The cap is 256 characters of the server's JSON serialisation of this + * object, not 256 bytes of content: the `{"...": "..."}` punctuation counts, + * and non-ASCII characters are escaped to `\\uXXXX` before measuring, so + * they cost six characters each. */ workflow_metadata?: Record From bbd8f62f3c02bab8fddbfdd4588871393f262f2e Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 3 Oct 2026 21:23:35 -0700 Subject: [PATCH 05/52] fix(e2e): close the metadata-scope nesting hole and the id sentinel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the eight findings the Cursor panel raised against this exact head. Each one reproduced before being fixed; each fix is red without it. The substantive one is that `withMetadata` raised its in-flight flag *after* `fn()` returned, so a scope opened inside `fn` slipped past the guard entirely. That was worse than ambiguous: the inner scope's `finally` restored its own saved value, so an outer scope that had already closed came back to life and stamped every later frame for the rest of the test — a frame carrying a `workflow_id` from a dead scope, which is the exact bug the harness exists to catch. The flag is now raised before `fn()`, so exactly one scope can be open and nesting throws wherever it is opened from. A depth counter would instead permit nesting, reintroducing the attribution ambiguity the throw exists to prevent. In production, the `workflow_metadata` spread guarded on bare truthiness, so the all-zero uuid sentinel shipped as a real routing key. `LGraph._id` defaults to it and `LGraph.serialize()` returns `id` unfiltered, while the rest of the codebase treats it as "no id yet" (`ensureNonZeroUuid`, `adoptRootGraphId`), so every workflow still carrying it would have shared one routing key. Shape is already constrained upstream — loaded ids are `z.string().uuid()` and the only other producer is `createUuidv4()` — so the sentinel is the one value to exclude, and no second re-validation is added. `FrameSelector` gains a 1-based `occurrence`, because `label` + `jobId` cannot tell apart two frames of the same event from one prompt — which any script using both `progress()` and `nodeRunning()` produces, including this file's own `everyFrame()`. "The backend dropped the second of three progress frames" was inexpressible: `dropFrames` silently hit both and `swapFrames` advised adding a `jobId` that cannot disambiguate one job from itself. Also: `interleave` throws on an empty side rather than returning the other script verbatim with no overlap at all (the same silent degradation to no-fault that `requireMatches` refuses); `duplicateFrame` is renamed `duplicateFrames`, plural like `dropFrames`, since it duplicates every match; the binary preview frame is labelled `b_preview_with_metadata`, the event `api.ts` actually dispatches for it, rather than the legacy `b_preview` alias, and the label contract is now asserted for it instead of carved out. The scope being ambient rather than lexical while an async callback is pending is documented rather than changed — it is the intended semantics, modelling core holding a prompt's metadata for the run. All eleven pre-existing concurrency-proof mutations still go red, so none of the guarantees this harness pins were weakened. --- .../fixtures/helpers/BackendSimulator.test.ts | 163 +++++++++++++++++- .../fixtures/helpers/BackendSimulator.ts | 87 ++++++++-- .../fixtures/helpers/ExecutionHelper.ts | 40 +++-- src/scripts/api.queuePrompt.test.ts | 15 ++ src/scripts/api.ts | 14 +- 5 files changed, 282 insertions(+), 37 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 81fc503b52f..787956ad6a8 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -6,7 +6,7 @@ import type { Frame } from '@e2e/fixtures/helpers/BackendSimulator' import { BackendSimulator, dropFrames, - duplicateFrame, + duplicateFrames, interleave, swapFrames } from '@e2e/fixtures/helpers/BackendSimulator' @@ -148,6 +148,43 @@ describe('BackendSimulator frame scripting', () => { expect(() => execution.withWorkflowId('wf-b', () => {})).not.toThrow() }) + it('refuses a scope nested inside another scope', () => { + const { execution } = harness() + + // Nesting slipped past the in-flight guard entirely, because the flag was + // only raised *after* `fn()` returned. Two live scopes cannot be attributed + // to their prompts, so the second one throws wherever it is opened from. + expect(() => + execution.withWorkflowId('wf-a', () => + execution.withWorkflowId('wf-b', () => {}) + ) + ).toThrow(/already in flight/) + }) + + it('does not resurrect a closed scope when a nested async one settles', async () => { + const { sent, execution } = harness() + let release!: () => void + const blocked = new Promise((resolve) => { + release = resolve + }) + + // The shape the nesting hole produced: the inner scope's `finally` restored + // its own saved value, bringing the already-closed outer scope back to life + // and stamping every later frame `wf-a` for the rest of the test. Before the + // guard this emitted `[undefined, 'wf-a']`. + expect(() => + execution.withWorkflowId('wf-a', () => { + void execution.withWorkflowId('wf-b', () => blocked) + }) + ).toThrow(/already in flight/) + + release() + await blocked + execution.executionStart('job-after') + + expect(sent.map((frame) => frame.data.workflow_id)).toEqual([undefined]) + }) + it('drops a frame the backend never sent', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -195,7 +232,7 @@ describe('BackendSimulator frame scripting', () => { const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) simulator.play( - duplicateFrame( + duplicateFrames( [prompt.start(), prompt.success()], 'wf-a:execution_success' ) @@ -214,7 +251,7 @@ describe('BackendSimulator frame scripting', () => { const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) simulator.play( - duplicateFrame([first.success(), second.success()], { + duplicateFrames([first.success(), second.success()], { label: 'wf-a:execution_success', jobId: 'job-b' }) @@ -271,6 +308,96 @@ describe('BackendSimulator frame scripting', () => { ]) }) + it('drops one of a prompt’s two frames of the same event', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + // One prompt, two `progress` frames — what any script using both + // `progress()` and `nodeRunning()` produces. `label` + `jobId` cannot tell + // them apart, so "the backend dropped the second progress frame" (the shape + // behind progress stuck after a run) needs the occurrence selector. + const script = [ + prompt.start(), + prompt.progress('1', 1, 4), + ...prompt.nodeRunning('1', 2, 4), + prompt.success() + ] + + simulator.play( + dropFrames(script, { label: 'wf-a:progress', occurrence: 2 }) + ) + + expect(sent.map((frame) => [frame.type, frame.data.value])).toEqual([ + ['execution_start', undefined], + ['progress', 1], + ['progress_state', undefined], + ['execution_success', undefined] + ]) + }) + + it('duplicates one of a prompt’s two frames of the same event', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.progress('1', 1, 4), prompt.progress('1', 2, 4)] + + simulator.play( + duplicateFrames(script, { label: 'wf-a:progress', occurrence: 1 }) + ) + + expect(sent.map((frame) => frame.data.value)).toEqual([1, 1, 2]) + }) + + it('swaps one of a prompt’s two frames of the same event', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [ + prompt.progress('1', 1, 4), + prompt.progress('1', 2, 4), + prompt.success() + ] + + simulator.play( + swapFrames( + script, + { label: 'wf-a:progress', occurrence: 2 }, + 'wf-a:execution_success' + ) + ) + + expect(sent.map((frame) => [frame.type, frame.data.value])).toEqual([ + ['progress', 1], + ['execution_success', undefined], + ['progress', 2] + ]) + }) + + it('points an ambiguous swap at the selector that can resolve it', () => { + const { simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [ + prompt.progress('1', 1, 4), + ...prompt.nodeRunning('1', 2, 4) + ] + + // The old message only offered `jobId`, which cannot disambiguate two + // frames of the *same* job — advice a test author could not act on. + expect(() => + swapFrames(script, 'wf-a:progress', 'wf-a:progress_state') + ).toThrow(/occurrence/) + }) + + it('throws rather than silently applying no fault for a missing occurrence', () => { + const { simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.progress('1', 1, 4)] + + expect(() => + dropFrames(script, { label: 'wf-a:progress', occurrence: 2 }) + ).toThrow(/not in the script/) + expect(() => + dropFrames(script, { label: 'wf-a:progress', occurrence: 0 }) + ).toThrow(/positive integer/) + }) + it('throws rather than silently skipping an unknown swap target', () => { const { simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -315,7 +442,7 @@ describe('BackendSimulator frame scripting', () => { /not in the script/ ) expect(() => - duplicateFrame(script, { + duplicateFrames(script, { label: 'wf-a:execution_start', jobId: 'job-zzz' }) @@ -373,6 +500,21 @@ describe('BackendSimulator frame granularity', () => { ) }) + it('labels the binary frame with the event it dispatches', () => { + const { simulator } = harness() + const script = everyFrame(simulator) + + // The carve-out above excludes exactly one builder from the label contract, + // so this is the only thing that holds it to it. `latentPreview` sends a + // type-4 message, which `api.ts` dispatches as `b_preview_with_metadata` + // (plus a legacy `b_preview` alias); labelling the frame with the alias made + // the primary event name throw `not in the script`. + expect(script.at(-1)?.label).toBe('wf-a:b_preview_with_metadata') + expect(() => + dropFrames(script, 'wf-a:b_preview_with_metadata') + ).not.toThrow() + }) + it('scripts the two halves of a running node independently', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -478,6 +620,19 @@ describe('BackendSimulator interleaving', () => { expect(schedules.size).toBeGreaterThanOrEqual(5) }) + it('throws rather than returning a schedule that cannot overlap', () => { + const { simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [a.start(), a.success()] + + // Both forcing rules are gated on the other side being non-empty, so an + // empty side used to return the other script verbatim: a schedule with no + // overlap at all, passing every concurrency assertion vacuously because + // there is only one prompt to assert about. + expect(() => interleave(script, [], 7)).toThrow(/both sides/) + expect(() => interleave([], script, 7)).toThrow(/both sides/) + }) + it('keeps every frame exactly once when interleaving', () => { const { sent, simulator } = harness() const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts index 365fbe22db6..a49e73b528d 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -20,19 +20,34 @@ export interface Frame { /** * Which frames a fault applies to. A bare string keeps the workflow-wide * selection every script already uses; add `jobId` to reach one prompt when - * several share a workflow id. + * several share a workflow id, and `occurrence` to reach one frame when a + * single prompt sends the same event more than once. + * + * `occurrence` is 1-based and counts within whatever `label` (+ `jobId`) already + * matched, so `{ label: 'wf-a:progress', occurrence: 2 }` is "the second + * progress frame" — the shape behind *progress stuck after a run*, which needs + * one of several identical events to go missing. Without it `label` + `jobId` + * cannot tell two `progress` frames of one prompt apart, and one prompt sends + * two of them as soon as a script uses both `progress()` and `nodeRunning()`. */ -export type FrameSelector = string | { label: string; jobId?: string } +export type FrameSelector = + | string + | { label: string; jobId?: string; occurrence?: number } function describeSelector(selector: FrameSelector): string { - return typeof selector === 'string' - ? selector - : selector.jobId === undefined - ? selector.label - : `${selector.label} (job ${selector.jobId})` + if (typeof selector === 'string') return selector + const qualifiers = [ + selector.jobId === undefined ? undefined : `job ${selector.jobId}`, + selector.occurrence === undefined + ? undefined + : `occurrence ${selector.occurrence}` + ].filter((part) => part !== undefined) + return qualifiers.length === 0 + ? selector.label + : `${selector.label} (${qualifiers.join(', ')})` } -function matches(frame: Frame, selector: FrameSelector): boolean { +function matchesLabelAndJob(frame: Frame, selector: FrameSelector): boolean { if (typeof selector === 'string') return frame.label === selector if (frame.label !== selector.label) return false return selector.jobId === undefined || frame.jobId === selector.jobId @@ -41,9 +56,21 @@ function matches(frame: Frame, selector: FrameSelector): boolean { function indicesOf(frames: Frame[], selector: FrameSelector): number[] { const found: number[] = [] frames.forEach((frame, index) => { - if (matches(frame, selector)) found.push(index) + if (matchesLabelAndJob(frame, selector)) found.push(index) }) - return found + if (typeof selector === 'string' || selector.occurrence === undefined) { + return found + } + if (selector.occurrence < 1 || !Number.isInteger(selector.occurrence)) { + throw new Error( + `occurrence must be a positive integer, got ${selector.occurrence}` + ) + } + // `.at` rather than an index, so an occurrence past the end is `undefined` + // and falls through to `requireMatches` instead of being a hole in the array. + // The positive-integer check above is what keeps this from wrapping. + const picked = found.at(selector.occurrence - 1) + return picked === undefined ? [] : [picked] } /** @@ -155,8 +182,14 @@ export class SimulatedPrompt { ) } + /** + * The binary preview frame. Labelled `b_preview_with_metadata`, which is the + * event `api.ts` dispatches for the type-4 message this sends; the `b_preview` + * it also dispatches is a back-compat alias, and labelling the frame with the + * alias made the primary event name unselectable. + */ latentPreview(nodeId: string): Frame { - return this.frame('b_preview', () => + return this.frame('b_preview_with_metadata', () => this.execution.latentPreview(this.jobId, nodeId) ) } @@ -204,12 +237,18 @@ export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { return frames.filter((_, index) => !drop.has(index)) } -/** Send selected frames twice, as a retrying or reconnecting backend does. */ -export function duplicateFrame( +/** + * Send selected frames twice, as a retrying or reconnecting backend does. + * + * Plural like {@link dropFrames}: every match is duplicated, so a bare label + * matching two prompts injects a wider fault than a test naming one frame + * probably means. Narrow it with `jobId`/`occurrence` when that matters. + */ +export function duplicateFrames( frames: Frame[], selector: FrameSelector ): Frame[] { - const duplicate = new Set(requireMatches(frames, selector, 'duplicateFrame')) + const duplicate = new Set(requireMatches(frames, selector, 'duplicateFrames')) return frames.flatMap((frame, index) => duplicate.has(index) ? [frame, frame] : [frame] ) @@ -221,9 +260,9 @@ export function duplicateFrame( * Each selector must identify exactly one frame, and the two must not be the * same frame. A swap is a pair operation, so quietly taking the first of several * matches is how a two-prompt script ends up reordering the wrong prompt's - * frame; add a `jobId` to disambiguate. Swapping a frame with itself is the same - * silent no-fault that {@link requireMatches} exists to rule out, so it throws - * rather than returning the script unchanged. + * frame; add a `jobId` or an `occurrence` to disambiguate. Swapping a frame with + * itself is the same silent no-fault that {@link requireMatches} exists to rule + * out, so it throws rather than returning the script unchanged. */ export function swapFrames( frames: Frame[], @@ -239,7 +278,8 @@ export function swapFrames( if (found.length > 1) { throw new Error( `swapFrames: ${describeSelector(selector)} matches ${found.length} frames; ` + - 'add a jobId to select one' + 'add a jobId (different prompts) or an occurrence (same prompt, ' + + 'repeated event) to select one' ) } } @@ -266,8 +306,19 @@ export function swapFrames( * that guarantee a schedule can come out sequential and a concurrency spec * passes without ever having two prompts in flight. (A one-frame-each pair * cannot overlap; everything larger does.) + * + * An empty input throws. Both forcing rules below are gated on the other side + * being non-empty, so an empty side would return the other script verbatim with + * no overlap at all — the same silent degradation to no-fault that + * {@link requireMatches} refuses, and with the same consequence: a concurrency + * spec passing having never been concurrent. */ export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { + if (a.length === 0 || b.length === 0) { + throw new Error( + `interleave: needs frames on both sides to overlap, got ${a.length} and ${b.length}` + ) + } const out: Frame[] = [] let state = seed | 0 let i = 0 diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 39919a7c4f9..1f7bfa9f4e0 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -54,7 +54,7 @@ export function buildKSamplerError( export class ExecutionHelper { private jobCounter = 0 private metadata: Record | undefined - private asyncScopeOpen = false + private scopeOpen = false private readonly completedJobs: RawJobListItem[] = [] private readonly page: ComfyPage['page'] private readonly command: ComfyPage['command'] @@ -103,26 +103,42 @@ export class ExecutionHelper { * previous scope immediately, silently unstamping (or mis-stamping) every * frame after the first suspension point. * - * Opening a second scope while an async one is still in flight throws rather - * than guessing: without ambient async context two overlapping scopes cannot - * be attributed to their prompts, and the wrong `workflow_id` on a frame is - * precisely the bug this harness exists to catch. Await the first scope, or - * script the prompts as separate synchronous frames via `BackendSimulator`. + * The scope is ambient for as long as it is open, not lexical to `fn`: a frame + * emitted from anywhere while an async scope is pending is stamped with it. + * That is the point — it models core holding a prompt's `workflow_metadata` + * for the duration of the run — but it means an un-awaited scope keeps + * stamping, so always await the returned promise. + * + * **Exactly one scope may be open at a time, and a second one throws** — + * whether it overlaps an async scope or is nested inside `fn`. Without + * ambient async context two live scopes cannot be attributed to their + * prompts, and the wrong `workflow_id` on a frame is precisely the bug this + * harness exists to catch. Nesting used to slip past this guard and was + * worse than ambiguous: the inner scope's `finally` restored *its* saved + * value, so an outer scope that had already closed came back to life and + * stamped every later frame for the rest of the test. Await the first scope, + * or script the prompts as separate synchronous frames via + * `BackendSimulator`. */ withMetadata( metadata: Record | undefined, fn: () => T ): T { - if (this.asyncScopeOpen) { + if (this.scopeOpen) { throw new Error( - 'ExecutionHelper: an async metadata scope is still in flight. ' + - 'Await it before opening another one.' + 'ExecutionHelper: a metadata scope is already in flight. ' + + 'Await it before opening another one, and do not nest them.' ) } const previous = this.metadata this.metadata = metadata + // Set before `fn` runs, so a scope opened synchronously inside it is + // rejected too. Setting it afterwards only caught the overlapping-async + // case and left nesting — the strictly worse one — unguarded. + this.scopeOpen = true const restore = () => { + this.scopeOpen = false this.metadata = previous } @@ -139,13 +155,9 @@ export class ExecutionHelper { return result } - this.asyncScopeOpen = true // Cast: `T` is promise-like here, and the returned promise settles with the // same value or rejection, only after the scope has been restored. - return Promise.resolve(result).finally(() => { - this.asyncScopeOpen = false - restore() - }) as T + return Promise.resolve(result).finally(restore) as T } /** diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts index 181fd1e64d3..e6231f56bfe 100644 --- a/src/scripts/api.queuePrompt.test.ts +++ b/src/scripts/api.queuePrompt.test.ts @@ -5,6 +5,7 @@ import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema' import { api } from '@/scripts/api' +import { zeroUuid } from '@/utils/uuid' // Tests for the workflow_metadata field api.queuePrompt sends; fetchApi is stubbed. const WORKFLOW_ID = '0199e3a3-6c01-7000-8000-c1a1f7a8d9b2' @@ -67,6 +68,20 @@ describe('api.queuePrompt workflow_metadata', () => { expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') }) + it('omits the field for the all-zero id sentinel', async () => { + // `LGraph._id` defaults to `zeroUuid` and `LGraph.serialize()` returns it + // unfiltered, so a bare truthiness check shipped the sentinel as a real + // routing key — and every workflow still carrying it would have shared one, + // which is how a WS frame gets attributed to the wrong workflow. The rest of + // the codebase already treats it as "no id yet" (`ensureNonZeroUuid`). + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(zeroUuid) + }) + + expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') + }) + it('still sends the id inside extra_pnginfo for older backends', async () => { await api.queuePrompt(0, { output: EMPTY_PROMPT, diff --git a/src/scripts/api.ts b/src/scripts/api.ts index e8ff8007693..2e7777f63db 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -29,6 +29,7 @@ import { } from '@/platform/auth/unified/remintRetry' import { getDevOverride } from '@/utils/devFeatureFlagOverride' import { getSessionOverride } from '@/utils/sessionFeatureFlagOverride' +import { zeroUuid } from '@/utils/uuid' import type { ModelFile, ModelFolderInfo @@ -1365,7 +1366,18 @@ export class ComfyApi extends EventTarget { ...(options?.partialExecutionTargets && { partial_execution_targets: options.partialExecutionTargets }), - ...(workflow.id && { workflow_metadata: { workflow_id: workflow.id } }), + // `LGraph.serialize()` returns `id` unfiltered and `LGraph._id` defaults + // to the all-zero sentinel, which the codebase treats as "no id yet" and + // replaces on load (`adoptRootGraphId`) and on clear. A bare truthiness + // check ships that sentinel as a real routing key, so every workflow + // still carrying it would share one and WS frames could be attributed to + // the wrong workflow. Shape is already constrained upstream — the loaded + // schema is `z.string().uuid()` and the only other producer is + // `createUuidv4()` — so the sentinel is the one value to exclude here. + ...(workflow.id && + workflow.id !== zeroUuid && { + workflow_metadata: { workflow_id: workflow.id } + }), extra_data: { auth_token_comfy_org: this.authToken, api_key_comfy_org: this.apiKey, From 33b29f1865065b36f8e6d76e6d179d352cdc796d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 5 Oct 2026 20:00:02 -0700 Subject: [PATCH 06/52] fix: scope execution state by workflow and recover dropped terminal frames (#20050) --- .../assets/execution/workflow_with_id.json | 227 ++++ .../assets/execution/workflow_with_id_b.json | 227 ++++ .../workflow_with_id_pasted_copy.json | 227 ++++ .../fixtures/helpers/BackendSimulator.test.ts | 3 +- .../fixtures/helpers/ExecutionHelper.ts | 15 +- .../fixtures/helpers/WorkflowHelper.ts | 33 +- .../tests/twoTabExecutionLeak.spec.ts | 339 +++++ .../tests/workflowScopedExecution.spec.ts | 366 ++++++ .../remote/comfyui/execution/types.ts | 13 + .../remote/comfyui/useQueuePolling.test.ts | 8 +- .../remote/comfyui/useQueuePolling.ts | 4 +- src/scripts/api.ts | 27 +- src/scripts/app.ts | 10 + .../executionStore.terminalRecovery.test.ts | 232 ++++ src/stores/executionStore.ts | 442 ++++++- .../executionStore.workflowGating.test.ts | 1156 +++++++++++++++++ src/stores/queueStore.ts | 17 +- 17 files changed, 3261 insertions(+), 85 deletions(-) create mode 100644 browser_tests/assets/execution/workflow_with_id.json create mode 100644 browser_tests/assets/execution/workflow_with_id_b.json create mode 100644 browser_tests/assets/execution/workflow_with_id_pasted_copy.json create mode 100644 browser_tests/tests/twoTabExecutionLeak.spec.ts create mode 100644 browser_tests/tests/workflowScopedExecution.spec.ts create mode 100644 src/stores/executionStore.terminalRecovery.test.ts create mode 100644 src/stores/executionStore.workflowGating.test.ts diff --git a/browser_tests/assets/execution/workflow_with_id.json b/browser_tests/assets/execution/workflow_with_id.json new file mode 100644 index 00000000000..669efb97138 --- /dev/null +++ b/browser_tests/assets/execution/workflow_with_id.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 9, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [413, 389], + "size": [425.27801513671875, 180.6060791015625], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [6], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": ["text, watermark"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [415, 186], + "size": [422.84503173828125, 164.31304931640625], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [4], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [473, 609], + "size": [315, 106], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + { + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [7], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1209, 188], + "size": [210, 46], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [9], + "slot_index": 0 + } + ], + "properties": {} + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1451, 189], + "size": [210, 26], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {} + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [26, 474], + "size": [315, 98], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [1], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [3, 5], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [8], + "slot_index": 2 + } + ], + "properties": { + "models": [ + { + "name": "v1-5-pruned-emaonly-fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + "directory": "checkpoints" + } + ] + }, + "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] + } + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 5, 0, 3, 3, "LATENT"], + [3, 4, 1, 6, 0, "CLIP"], + [4, 6, 0, 3, 1, "CONDITIONING"], + [5, 4, 1, 7, 0, "CLIP"], + [6, 7, 0, 3, 2, "CONDITIONING"], + [7, 3, 0, 8, 0, "LATENT"], + [8, 4, 2, 8, 1, "VAE"], + [9, 8, 0, 9, 0, "IMAGE"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "offset": [0, 0], + "scale": 1 + } + }, + "version": 0.4, + "id": "0199e3a3-6c01-7000-8000-aaaaaaaaaaaa" +} diff --git a/browser_tests/assets/execution/workflow_with_id_b.json b/browser_tests/assets/execution/workflow_with_id_b.json new file mode 100644 index 00000000000..a6804a22c21 --- /dev/null +++ b/browser_tests/assets/execution/workflow_with_id_b.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 9, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [413, 389], + "size": [425.27801513671875, 180.6060791015625], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [6], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": ["text, watermark"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [415, 186], + "size": [422.84503173828125, 164.31304931640625], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [4], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [473, 609], + "size": [315, 106], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + { + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [7], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1209, 188], + "size": [210, 46], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [9], + "slot_index": 0 + } + ], + "properties": {} + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1451, 189], + "size": [210, 26], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {} + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [26, 474], + "size": [315, 98], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [1], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [3, 5], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [8], + "slot_index": 2 + } + ], + "properties": { + "models": [ + { + "name": "v1-5-pruned-emaonly-fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + "directory": "checkpoints" + } + ] + }, + "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] + } + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 5, 0, 3, 3, "LATENT"], + [3, 4, 1, 6, 0, "CLIP"], + [4, 6, 0, 3, 1, "CONDITIONING"], + [5, 4, 1, 7, 0, "CLIP"], + [6, 7, 0, 3, 2, "CONDITIONING"], + [7, 3, 0, 8, 0, "LATENT"], + [8, 4, 2, 8, 1, "VAE"], + [9, 8, 0, 9, 0, "IMAGE"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "offset": [0, 0], + "scale": 1 + } + }, + "version": 0.4, + "id": "0199e3a3-6c01-7000-8000-bbbbbbbbbbbb" +} diff --git a/browser_tests/assets/execution/workflow_with_id_pasted_copy.json b/browser_tests/assets/execution/workflow_with_id_pasted_copy.json new file mode 100644 index 00000000000..669efb97138 --- /dev/null +++ b/browser_tests/assets/execution/workflow_with_id_pasted_copy.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 9, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [413, 389], + "size": [425.27801513671875, 180.6060791015625], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [6], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": ["text, watermark"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [415, 186], + "size": [422.84503173828125, 164.31304931640625], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [4], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [473, 609], + "size": [315, 106], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + { + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [7], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1209, 188], + "size": [210, 46], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [9], + "slot_index": 0 + } + ], + "properties": {} + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1451, 189], + "size": [210, 26], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {} + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [26, 474], + "size": [315, 98], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [1], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [3, 5], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [8], + "slot_index": 2 + } + ], + "properties": { + "models": [ + { + "name": "v1-5-pruned-emaonly-fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + "directory": "checkpoints" + } + ] + }, + "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] + } + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 5, 0, 3, 3, "LATENT"], + [3, 4, 1, 6, 0, "CLIP"], + [4, 6, 0, 3, 1, "CONDITIONING"], + [5, 4, 1, 7, 0, "CLIP"], + [6, 7, 0, 3, 2, "CONDITIONING"], + [7, 3, 0, 8, 0, "LATENT"], + [8, 4, 2, 8, 1, "VAE"], + [9, 8, 0, 9, 0, "IMAGE"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "offset": [0, 0], + "scale": 1 + } + }, + "version": 0.4, + "id": "0199e3a3-6c01-7000-8000-aaaaaaaaaaaa" +} diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 787956ad6a8..cd48e47fcdf 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -94,7 +94,8 @@ describe('BackendSimulator frame scripting', () => { expect(sent[0].data).toEqual({ workflow_id: 'wf-a', prompt_id: 'job-a', - node: '7' + node: '7', + display_node: '7' }) }) diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 1f7bfa9f4e0..6cc9497c928 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -279,9 +279,20 @@ export class ExecutionHelper { this.emit('execution_start', { prompt_id: jobId, timestamp: Date.now() }) } - /** Send `executing` WS event to signal which node is currently running. */ + /** + * Send `executing` WS event to signal which node is currently running. + * + * Shape matches the server: a node-start frame carries `display_node` + * (execution.py), the terminal `node: null` frame does not (main.py). The + * app reads `display_node || node`, so omitting it would silently exercise + * the fallback instead of the normal path. + */ executing(jobId: string, nodeId: string | null): void { - this.emit('executing', { prompt_id: jobId, node: nodeId }) + this.emit('executing', { + prompt_id: jobId, + node: nodeId, + ...(nodeId !== null && { display_node: nodeId }) + }) } /** Send `executed` WS event with node output. */ diff --git a/browser_tests/fixtures/helpers/WorkflowHelper.ts b/browser_tests/fixtures/helpers/WorkflowHelper.ts index 546aa068a97..bc2aafd8f13 100644 --- a/browser_tests/fixtures/helpers/WorkflowHelper.ts +++ b/browser_tests/fixtures/helpers/WorkflowHelper.ts @@ -222,23 +222,26 @@ export class WorkflowHelper { }) } + /** + * `workflowStore.openWorkflow` registers the tab but never loads the graph, + * so going through the store leaves the canvas on whatever it already had. + * Clicking the sidebar entry is the path a user takes and the only one that + * puts the file's nodes on the canvas. + */ async openPersistedWorkflow(workflowName: string): Promise { - await this.comfyPage.page.evaluate(async (name) => { - const store = (window.app!.extensionManager as WorkspaceStore).workflow - await store.syncWorkflows() - const workflow = - store.getWorkflowByPath(`workflows/${name}.json`) ?? - store.persistedWorkflows.find( - (candidate) => - candidate.filename === name || - candidate.path.endsWith(`${name}.json`) - ) - if (!workflow) { - throw new Error(`Persisted workflow not found: ${name}`) - } - await store.openWorkflow(workflow) - }, workflowName) + const sidebar = this.comfyPage.menu.workflowsTab + await sidebar.open() + await sidebar.getPersistedItem(workflowName).click() + await sidebar.close() await this.waitForWorkflowIdle() + await this.comfyPage.page.waitForFunction( + (name) => + ( + window.app!.extensionManager as WorkspaceStore + ).workflow.activeWorkflow?.path.endsWith(`${name}.json`) ?? false, + workflowName, + { timeout: 5000 } + ) await this.comfyPage.vueNodes.waitForNodes() } diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts new file mode 100644 index 00000000000..7b9fc2009cf --- /dev/null +++ b/browser_tests/tests/twoTabExecutionLeak.spec.ts @@ -0,0 +1,339 @@ +import { mergeTests } from '@playwright/test' + +import { + comfyExpect as expect, + comfyPageFixture +} from '@e2e/fixtures/ComfyPage' +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import { BackendSimulator } from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' +import { webSocketFixture } from '@e2e/fixtures/ws' +import { toNodeId } from '@/types/nodeId' +import type { WorkspaceStore } from '@e2e/types/globals' + +/** + * The reported bug, with two real workflow tabs rather than simulated ones. + * + * `workflowScopedExecution.spec.ts` injects frames tagged with a foreign + * workflow id, which exercises the gate but takes the frontend's own tab + * bookkeeping on trust. These tests open two persisted workflows, queue from + * one, switch to the other, and assert the second tab is untouched — the + * actual sequence users reported. + */ + +const test = mergeTests(comfyPageFixture, webSocketFixture) + +const KSAMPLER_NODE = '3' +const SAVE_IMAGE_NODE = '9' +const WORKFLOW_A = 'exec_leak_a' +const WORKFLOW_B = 'exec_leak_b' +const WORKFLOW_A_COPY = 'pasted_twin' + +function imageOutput(filename: string) { + return { images: [{ filename, subfolder: '', type: 'output' }] } +} + +async function activeWorkflowId( + comfyPage: ComfyPage +): Promise { + return comfyPage.page.evaluate(() => { + const workflow = (window.app!.extensionManager as WorkspaceStore).workflow + .activeWorkflow + return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined + }) +} + +async function canvasNodeRender(comfyPage: ComfyPage, nodeId: string) { + return comfyPage.page.evaluate((id) => { + const node = window.app!.graph.getNodeById(id) + if (!node) return null + const stroke = node.strokeStyles['running']?.call(node) + return { progress: node.progress ?? null, outlined: stroke !== undefined } + }, toNodeId(nodeId)) +} + +test.describe('cross-tab execution leak', { tag: '@ui' }, () => { + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.workflow.setupWorkflowsDirectory({ + [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', + [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json' + }) + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) + }) + + test('a run in one tab leaves the other tab idle', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + + const workflowAId = await activeWorkflowId(comfyPage) + expect(workflowAId, 'workflow A must carry an id').toBeDefined() + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + const workflowBId = await activeWorkflowId(comfyPage) + expect(workflowBId, 'the second tab must be a different workflow').not.toBe( + workflowAId + ) + + // Tab A's run continues while the user looks at tab B. + simulator.play([ + ...running.nodeRunning(KSAMPLER_NODE, 3, 4), + running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), + running.success() + ]) + + // Per-tab state, read from the topbar badges: A owns the run, B never + // shows one. Asserted rather than screenshotted — this is a state bug. + const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) + const tabB = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_B) + await expect(tabB.getByRole('img', { name: 'Running' })).toHaveCount(0) + await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() + await expect(tabB.getByRole('img', { name: 'Completed' })).toHaveCount(0) + }) + + test('switching back shows the finished run, not a stuck one', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowAId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) + + // Terminal frames arrive while tab A is in the background. That is the + // stuck-progress report: the run must reach its end, not stay Running. + simulator.play([ + running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), + running.success() + ]) + await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() + await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) + + await comfyPage.workflow.switchToTab(WORKFLOW_A) + + // An active tab deliberately carries no status badge (WorkflowTab.vue:235 + // — the user is already looking at it), so the absence of Completed here + // proves nothing. What must hold is that nothing anywhere still reads as + // running once the run has ended. + await expect( + comfyPage.menu.topbar.workflowTabs.getByRole('img', { name: 'Running' }) + ).toHaveCount(0) + }) + + test('the background tab records its own run while the other is visible', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowAId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + simulator.play([running.success()]) + + // The tab badge is per-workflow state, so it must still reach its terminal + // mark while another tab is in front — the regression CI caught earlier. + const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) + await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() + await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) + }) + + test('the node the user is looking at does not take over a foreign run', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowAId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + await comfyPage.nextFrame() + + // Both workflows own a KSampler numbered 3, which is what makes this + // reachable: node progress is keyed by a locator resolved against the + // graph that happens to be in front, so A's entry lands on B's node. + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: 0.25, + outlined: true + }) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + await comfyPage.nextFrame() + + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + // A keeps running out of sight; none of it may surface on B. + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.nextFrame() + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await comfyPage.nextFrame() + + // Returning to A shows where its run actually got to, not where it was + // when the user left. + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: 0.75, + outlined: true + }) + }) + + test('stays clean across repeated switches during one run', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowAId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + await comfyPage.nextFrame() + + // Round 2 of the GPU QA: the first arrival in B was clean and later ones + // were not, so one switch proves nothing here. + for (const [index, step] of [3, 5, 7, 9].entries()) { + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await comfyPage.nextFrame() + simulator.play([...running.nodeRunning(KSAMPLER_NODE, step, 10)]) + await comfyPage.nextFrame() + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: step / 10, + outlined: true + }) + + await comfyPage.workflow.switchToTab(WORKFLOW_B) + await comfyPage.nextFrame() + expect( + await canvasNodeRender(comfyPage, KSAMPLER_NODE), + `arrival ${index + 1} in B must not inherit A` + ).toEqual({ progress: null, outlined: false }) + } + }) + + test('stays clean while the visible workflow is queued behind the running one', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowAId = await activeWorkflowId(comfyPage) + + const jobA = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobA, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + await comfyPage.nextFrame() + + // B is queued behind A and has not started. QA saw A's progress land on + // B's KSampler the moment B was queued. + await exec.run() + await comfyPage.nextFrame() + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.nextFrame() + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) + + // QA on the core PR thread: copy a workflow, paste it into a new tab, run + // both. `ensureWorkflowId` keeps the id already in the pasted json, so the + // two tabs share one workflow id while being different open workflows, and + // the node ids are identical too. Outputs landed on the wrong tab. + test('a pasted copy sharing the workflow id does not receive the original output', async ({ + comfyPage, + getWebSocket + }) => { + await comfyPage.workflow.setupWorkflowsDirectory({ + [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', + [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id_pasted_copy.json' + }) + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const sharedId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const running = simulator.prompt(jobId, { workflowId: sharedId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A_COPY) + await comfyPage.nextFrame() + expect( + await activeWorkflowId(comfyPage), + 'the copy must carry the same id, which is what makes this reachable' + ).toBe(sharedId) + + simulator.play([ + ...running.nodeRunning(KSAMPLER_NODE, 2, 4), + running.executed(SAVE_IMAGE_NODE, imageOutput('from-the-original.png')), + running.success() + ]) + await comfyPage.nextFrame() + + const leaked = await comfyPage.page.evaluate(() => + JSON.stringify(window.app!.nodeOutputs) + ) + expect( + leaked, + 'the original output must not land on the copy' + ).not.toContain('from-the-original.png') + expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) +}) diff --git a/browser_tests/tests/workflowScopedExecution.spec.ts b/browser_tests/tests/workflowScopedExecution.spec.ts new file mode 100644 index 00000000000..33a75398e60 --- /dev/null +++ b/browser_tests/tests/workflowScopedExecution.spec.ts @@ -0,0 +1,366 @@ +import { mergeTests } from '@playwright/test' + +import { + comfyExpect as expect, + comfyPageFixture +} from '@e2e/fixtures/ComfyPage' +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import { + BackendSimulator, + duplicateFrames, + interleave, + swapFrames +} from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' +import { webSocketFixture } from '@e2e/fixtures/ws' +import type { WorkspaceStore } from '@e2e/types/globals' + +/** + * Execution WebSocket frames must only move the visible workflow's UI. + * + * Every scenario here is a frame script rather than a timing race: the backend + * is simulated by {@link BackendSimulator}, so "a frame from the other tab + * arrives between this tab's progress and its terminal frame" is an ordered + * list, not a sleep. See `BackendSimulator.test.ts` for the harness's own + * tests. + * + * Every scenario loads a workflow that carries an `id`. Ownership gating has + * nothing to resolve for an unsaved workflow — no id on the message's side to + * compare against, and no queue-time mapping — so it stays deliberately + * permissive there, and a foreign-frame assertion would pass for the wrong + * reason. `assets/default.json` has no id, which is why these use + * `assets/execution/workflow_with_id.json`. + */ + +const test = mergeTests(comfyPageFixture, webSocketFixture) + +const KSAMPLER_NODE = '3' +const SAVE_IMAGE_NODE = '9' +const FOREIGN_WORKFLOW_ID = '0199e3a3-6c01-7000-8000-ffffffffffff' +/** Carries `id`, unlike `assets/default.json`. See the describe block comment. */ +const WORKFLOW_WITH_ID = 'execution/workflow_with_id' + +function imageOutput(filename: string) { + return { + images: [{ filename, subfolder: '', type: 'output' }] + } +} + +/** The id the gate compares against; undefined for an unsaved workflow. */ +async function activeWorkflowId( + comfyPage: ComfyPage +): Promise { + return comfyPage.page.evaluate(() => { + const workflow = (window.app!.extensionManager as WorkspaceStore).workflow + .activeWorkflow + return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined + }) +} + +test.describe('workflow-scoped execution', { tag: '@ui' }, () => { + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.workflow.loadWorkflow(WORKFLOW_WITH_ID) + await comfyPage.appMode.enterAppModeWithInputs([[KSAMPLER_NODE, 'seed']]) + await expect(comfyPage.appMode.linearWidgets).toBeVisible() + }) + + test('renders the visible workflow own frames', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const jobId = await exec.run() + await comfyPage.nextFrame() + + const mine = simulator.prompt(jobId, { + workflowId: await activeWorkflowId(comfyPage) + }) + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a frame from another workflow does not start the visible tab', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() + + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + simulator.play([ + foreign.start(), + ...foreign.nodeRunning(KSAMPLER_NODE, 1, 4), + foreign.executing(KSAMPLER_NODE) + ]) + + await expect(comfyPage.appMode.outputHistory.inProgressItems).toHaveCount(0) + await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) + }) + + test('an output from another workflow does not land on the visible tab', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() + + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([ + mine.start(), + foreign.start(), + foreign.executed(SAVE_IMAGE_NODE, imageOutput('theirs.png')), + foreign.success(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('interleaved prompts keep the visible tab on its own run', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() + + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + // Seed 11 fixes the order; it is printed here so a failure is reproducible. + simulator.play( + interleave( + [ + mine.start(), + ...mine.nodeRunning(KSAMPLER_NODE, 1, 4), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ], + [ + foreign.start(), + ...foreign.nodeRunning(KSAMPLER_NODE, 3, 4), + foreign.executed(SAVE_IMAGE_NODE, imageOutput('theirs.png')), + foreign.success() + ], + 11 + ) + ) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('an output that arrives after the terminal frame is ignored', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const jobId = await exec.run() + await comfyPage.nextFrame() + + const workflowId = await activeWorkflowId(comfyPage) + const mine = simulator.prompt(jobId, { workflowId }) + const script = [ + mine.start(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ] + const label = workflowId ?? jobId + + simulator.play( + swapFrames(script, `${label}:executed`, `${label}:execution_success`) + ) + + // A single WebSocket delivers in send order, so this ordering only happens + // on a reconnect replay. The terminal frame has already torn down the run, + // so the late output is dropped rather than applied to a finished job — + // outputs for a completed job come back through history instead. Pinned + // here so a future change to the teardown is a deliberate decision. + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(0) + await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) + }) + + test('a repeated terminal frame does not duplicate the output', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const jobId = await exec.run() + await comfyPage.nextFrame() + + const workflowId = await activeWorkflowId(comfyPage) + const mine = simulator.prompt(jobId, { workflowId }) + const script = [ + mine.start(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ] + + simulator.play( + duplicateFrames(script, `${workflowId ?? jobId}:execution_success`) + ) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a backend that sends no workflow id behaves as before', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const jobId = await exec.run() + await comfyPage.nextFrame() + + // No workflowId: every frame is emitted the way a core build without + // `workflow_metadata` emits it. This is the control for the 90% of users + // whose server has not been updated. + const legacy = simulator.prompt(jobId) + simulator.play([legacy.start(), ...legacy.nodeRunning(KSAMPLER_NODE, 1, 4)]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + legacy.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + legacy.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) + }) + + test('another workflow finishing does not end the visible run', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() + + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + // The other tab's run ends. Its terminal `executing: null` must not tear + // down this tab's run — the frame carries only a node id on the public + // event, so this is the path that needed the raw message to gate. + simulator.play([ + foreign.start(), + foreign.executing(null), + foreign.success() + ]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a reconnect mid-run does not duplicate the output', async ({ + comfyPage, + getWebSocket, + nextWebSocket + }) => { + const ws = await getWebSocket() + const exec = new ExecutionHelper(comfyPage, ws) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + // Arm the waiter before dropping the socket, then replay the run's frames + // on the reconnected socket the way a catch-up would. + const reconnected = nextWebSocket() + await ws.close() + const nextRoute = await reconnected + const afterReconnect = new BackendSimulator( + new ExecutionHelper(comfyPage, nextRoute) + ).prompt(jobId, { workflowId }) + + afterReconnect.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')).send() + afterReconnect.success().send() + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a binary preview for another workflow does not reach the visible tab', async ({ + comfyPage, + getWebSocket + }) => { + const exec = new ExecutionHelper(comfyPage, await getWebSocket()) + const simulator = new BackendSimulator(exec) + const workflowId = await activeWorkflowId(comfyPage) + + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + + // Binary frames carry a JSON header with prompt_id but core does not stamp + // workflow_id on them, so this leans on the queue-time mapping. Pinned + // here because the binary contract is a separate follow-up. + simulator.play([foreign.latentPreview(KSAMPLER_NODE)]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(0) + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + }) +}) diff --git a/src/platform/remote/comfyui/execution/types.ts b/src/platform/remote/comfyui/execution/types.ts index 65255056d3b..223d41eb480 100644 --- a/src/platform/remote/comfyui/execution/types.ts +++ b/src/platform/remote/comfyui/execution/types.ts @@ -5,6 +5,13 @@ import type { TaskStatus } from '@/platform/tasks/services/taskService' import { resultItemType } from '@/schemas/resultItemTypeSchema' export type JobId = string +/** + * The id of the workflow a prompt was queued from. Present on execution + * messages only when the server knows it — core omits the field entirely for a + * workflow with no id, and a server without `workflow_metadata` support never + * sends it, so every consumer must treat it as optional. + */ +type WorkflowId = string export const zResultItem = z.object({ filename: z.string().optional(), @@ -42,6 +49,7 @@ export interface ProgressWsMessage { value: number max: number prompt_id: JobId + workflow_id?: WorkflowId node: NodeId } export interface NodeProgressState { @@ -50,18 +58,21 @@ export interface NodeProgressState { state: 'pending' | 'running' | 'finished' | 'error' node_id: NodeId prompt_id: JobId + workflow_id?: WorkflowId display_node_id?: NodeId parent_node_id?: NodeId real_node_id?: NodeId } export interface ProgressStateWsMessage { prompt_id: JobId + workflow_id?: WorkflowId nodes: Record } export interface ExecutingWsMessage { node: NodeId | null display_node?: NodeId prompt_id: JobId + workflow_id?: WorkflowId } export interface ExecutedWsMessage extends ExecutingWsMessage { node: NodeId @@ -71,6 +82,7 @@ export interface ExecutedWsMessage extends ExecutingWsMessage { } interface ExecutionWsMessageBase { prompt_id: JobId + workflow_id?: WorkflowId timestamp: number } export type ExecutionStartWsMessage = ExecutionWsMessageBase @@ -97,6 +109,7 @@ export interface ProgressTextWsMessage { nodeId: NodeId text: string prompt_id?: string + workflow_id?: WorkflowId } export interface NotificationWsMessage { value: string diff --git a/src/platform/remote/comfyui/useQueuePolling.test.ts b/src/platform/remote/comfyui/useQueuePolling.test.ts index 6f2635ce501..108a1daaf89 100644 --- a/src/platform/remote/comfyui/useQueuePolling.test.ts +++ b/src/platform/remote/comfyui/useQueuePolling.test.ts @@ -39,13 +39,15 @@ describe('useQueuePolling', () => { expect(store.update).toHaveBeenCalledOnce() }) - it('does not poll when activeJobsCount > 1', async () => { + // Polling with several jobs in flight is what recovers per-job progress when + // a terminal WebSocket frame is dropped, so it must not be limited to one. + it('polls when activeJobsCount > 1', async () => { mountUseQueuePolling() Object.assign(store, { activeJobsCount: 2 }) - await vi.advanceTimersByTimeAsync(16_000) + await vi.advanceTimersByTimeAsync(8_000) - expect(store.update).not.toHaveBeenCalled() + expect(store.update).toHaveBeenCalledOnce() }) it('stops polling when activeJobsCount drops to 0', async () => { diff --git a/src/platform/remote/comfyui/useQueuePolling.ts b/src/platform/remote/comfyui/useQueuePolling.ts index c66c14a25a6..8625ba622c8 100644 --- a/src/platform/remote/comfyui/useQueuePolling.ts +++ b/src/platform/remote/comfyui/useQueuePolling.ts @@ -13,7 +13,7 @@ export function useQueuePolling() { const { start, stop } = useTimeoutFn( () => { - if (queueStore.activeJobsCount !== 1 || queueStore.isLoading) return + if (queueStore.activeJobsCount < 1 || queueStore.isLoading) return delay.value = Math.min(delay.value * BACKOFF_MULTIPLIER, MAX_INTERVAL_MS) void queueStore.update() }, @@ -22,7 +22,7 @@ export function useQueuePolling() { ) function scheduleNextPoll() { - if (queueStore.activeJobsCount === 1 && !queueStore.isLoading) start() + if (queueStore.activeJobsCount >= 1 && !queueStore.isLoading) start() else stop() } diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 2e7777f63db..29084b5aca8 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -486,6 +486,13 @@ export class ComfyApi extends EventTarget { * The current client id from websocket status updates. */ clientId?: string + /** + * The last `executing` message as the server sent it, including `prompt_id` + * and `workflow_id`. The public `executing` event carries only the node id + * for backwards compatibility with extensions, so a consumer that needs to + * know which run a frame belongs to reads it here. + */ + lastExecutingMessage: ExecutingWsMessage | null = null /** * The current user id. */ @@ -1178,10 +1185,22 @@ export class ComfyApi extends EventTarget { this.dispatchCustomEvent('status', msg.data.status ?? null) break case 'executing': - this.dispatchCustomEvent( - 'executing', - msg.data.display_node || msg.data.node - ) + // The public `executing` event keeps its bare NodeId detail — + // extensions depend on that shape — so the ids travel here + // instead, for consumers that must know which run the frame + // belongs to. Dispatch is synchronous, so clearing it straight + // after keeps the window to this frame's own handlers: an + // `executing` dispatched later by an extension must not be read + // as belonging to whichever run happened to arrive last. + this.lastExecutingMessage = msg.data + try { + this.dispatchCustomEvent( + 'executing', + msg.data.display_node || msg.data.node + ) + } finally { + this.lastExecutingMessage = null + } break case 'execution_start': case 'execution_error': diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 7b7d220dbcc..8e61556a96f 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -903,6 +903,16 @@ export class ComfyApp { detail.display_node || detail.node ) if (!executionId) return + // Outputs are keyed by a locator resolved against the visible graph, so + // a job finishing in another tab would otherwise write its result onto + // the same-numbered node of the tab in front. + if ( + !useExecutionStore().frameBelongsToVisibleWorkflow( + detail.prompt_id, + detail.workflow_id + ) + ) + return nodeOutputStore.setNodeOutputsByExecutionId(executionId, detail.output, { merge: detail.merge diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts new file mode 100644 index 00000000000..0f8999f4600 --- /dev/null +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -0,0 +1,232 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' +import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' +import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' +import { useExecutionStore } from '@/stores/executionStore' + +/** + * Recovery from a dropped terminal WebSocket frame. + * + * `execution_success` / `execution_error` / `execution_interrupted` are + * broadcast once and never retried, so a single dropped frame leaves node + * progress pinned forever — the user sees a node stuck mid-run with no way to + * clear it. Queue polling is the only authority that can notice: the backend + * reports the job in history while the frontend still holds progress for it. + */ + +const { mockShowTextPreview } = await vi.hoisted(async () => ({ + mockShowTextPreview: vi.fn() +})) + +vi.mock(import('@/composables/useAppMode')) +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) + +vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ + useNodeProgressText: () => ({ + showTextPreview: mockShowTextPreview, + removeTextPreview: vi.fn() + }) +})) + +type EventHandler = (event: CustomEvent) => void +const apiEventHandlers = new Map() + +vi.mock(import('@/scripts/api'), () => ({ + api: { + addEventListener: vi.fn((event: string, handler: EventHandler) => { + apiEventHandlers.set(event, handler) + }), + removeEventListener: vi.fn((event: string) => { + apiEventHandlers.delete(event) + }), + clientId: 'test-client', + apiURL: vi.fn((path: string) => `/api${path}`) + } +})) + +vi.mock(import('@/scripts/app'), () => { + const rootGraph = { getNodeById: vi.fn(), nodes: [] } + return { + app: { + rootGraph, + rootGraphOrUndefined: rootGraph, + revokePreviews: vi.fn(), + nodePreviewImages: {} + } + } +}) + +const WORKFLOW_A_ID = 'workflow-a-id' +const WORKFLOW_B_ID = 'workflow-b-id' + +function workflow(id: string, path: string): LoadedComfyWorkflow { + return fromPartial({ + activeState: { id }, + initialState: { id }, + path + }) +} + +const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') +const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') + +function fire(event: string, detail: Record) { + const handler = apiEventHandlers.get(event) + if (!handler) throw new Error(`${event} handler not bound`) + handler(new CustomEvent(event, { detail })) + if (event === 'progress_state' || event === 'progress') { + vi.advanceTimersToNextFrame() + } +} + +function runningNode(jobId: string, nodeId: string): NodeProgressState { + return { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state: 'running', + value: 3, + max: 10 + } +} + +describe('executionStore terminal-job recovery', () => { + let store: ReturnType + + beforeEach(() => { + apiEventHandlers.clear() + useWorkflowStore().activeWorkflow = workflowA + Object.assign(useWorkflowStore(), { openWorkflows: [workflowA, workflowB] }) + vi.mocked(useWorkflowStore().isOpen).mockImplementation((wf) => + useWorkflowStore().openWorkflows.some((open) => open.path === wf.path) + ) + store = useExecutionStore() + store.bindExecutionEvents() + }) + + /** A job of the active workflow, mid-run, whose terminal frame never came. */ + function stuckActiveJob(jobId = 'job-a') { + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fire('executing', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + node: '1', + display_node: '1' + }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + return jobId + } + + it('clears progress for a job the backend reports as finished', () => { + const jobId = stuckActiveJob() + expect(store.nodeProgressStates['1']?.state).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob[jobId]).toBeUndefined() + expect(store.activeJobId).toBeNull() + }) + + it('leaves a job the backend still reports as running alone', () => { + const jobId = stuckActiveJob() + + store.reconcileTerminalJobs(new Set([jobId]), new Set([jobId])) + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe(jobId) + }) + + it('ignores a job that is neither running nor in history yet', () => { + const jobId = stuckActiveJob() + + // Submitted but not yet visible in either list: evicting here would blank + // a run that is about to report progress. + store.reconcileTerminalJobs(new Set(), new Set()) + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe(jobId) + }) + + it('does not blank a live run when evicting an older job', () => { + const stale = 'job-stale' + store.registerJobWorkflowIdMapping(stale, WORKFLOW_A_ID) + fire('progress_state', { + prompt_id: stale, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(stale, '1') } + }) + const live = stuckActiveJob('job-live') + + store.reconcileTerminalJobs(new Set([live]), new Set([stale])) + + expect(store.nodeProgressStatesByJob[stale]).toBeUndefined() + expect(store.nodeProgressStates['1']?.prompt_id).toBe(live) + expect(store.activeJobId).toBe(live) + }) + + it('clears a stuck job belonging to another workflow', () => { + // The other tab's job is tracked per-job even though it never wrote the + // shared mirror, so it needs evicting too or that tab stays stuck. + store.registerJobWorkflowIdMapping('job-b', WORKFLOW_B_ID) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': runningNode('job-b', '1') } + }) + expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() + + store.reconcileTerminalJobs(new Set(), new Set(['job-b'])) + + expect(store.nodeProgressStatesByJob['job-b']).toBeUndefined() + }) + + it('clears an initializing job that reached history without a frame', () => { + fire('notification', { id: 'job-init', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-init')).toBe(true) + + store.reconcileTerminalJobs(new Set(), new Set(['job-init'])) + + expect(store.isJobInitializing('job-init')).toBe(false) + }) + + it('is idempotent', () => { + const jobId = stuckActiveJob() + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + expect(() => + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + ).not.toThrow() + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + + it('still recovers when the terminal frame was dropped for a legacy backend', () => { + // No workflow_id anywhere: recovery must not depend on the new field. + const jobId = 'job-legacy' + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + fire('execution_start', { prompt_id: jobId, timestamp: 1 }) + fire('progress_state', { + prompt_id: jobId, + nodes: { '1': runningNode(jobId, '1') } + }) + expect(store.nodeProgressStates['1']?.state).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) +}) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 8554849756f..6e31030698c 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -11,7 +11,10 @@ import type { WorkflowExecutionFailureReason, WorkflowExecutionIntent } from '@/platform/telemetry/types' -import type { ComfyWorkflow } from '@/platform/workflow/management/stores/workflowStore' +import type { + ComfyWorkflow, + LoadedComfyWorkflow +} from '@/platform/workflow/management/stores/workflowStore' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import type { ComfyApiWorkflow, @@ -38,6 +41,7 @@ import { useNodeOutputStore } from '@/stores/nodeOutputStore' import { useJobPreviewStore } from '@/stores/jobPreviewStore' import { useExecutionErrorStore } from '@/stores/executionErrorStore' import { tryNormalizeNodeExecutionId } from '@/types/nodeIdentification' +import type { SerializedNodeId } from '@/types/nodeId' import { parseNodeId } from '@/types/nodeId' import type { NodeLocatorId } from '@/types/nodeIdentification' import type { AppMode } from '@/utils/appMode' @@ -372,6 +376,8 @@ export const useExecutionStore = defineStore('execution', () => { const states = nodeProgressStates.value // Apparently doing this inside `Object.entries` causes issues for (const state of Object.values(states)) { + if (!messageMatchesActiveWorkflow(state.prompt_id, state.workflow_id)) + continue const parts = String(state.display_node_id).split(':') for (let i = 0; i < parts.length; i++) { const executionId = parts.slice(0, i + 1).join(':') @@ -480,35 +486,39 @@ export const useExecutionStore = defineStore('execution', () => { } function handleExecutionStart(e: CustomEvent) { - executionIdToLocatorCache.clear() - activeJobId.value = e.detail.prompt_id - queuedJobs.value[activeJobId.value] ??= { nodes: {} } - clearInitializationByJobId(activeJobId.value) + const jobId = e.detail.prompt_id + queuedJobs.value[jobId] ??= { nodes: {} } + clearInitializationByJobId(jobId) // Ensure path mapping exists — execution_start can arrive via WebSocket // before the HTTP response from queuePrompt triggers storeJob. - if (!jobIdToSessionWorkflowPath.value.has(activeJobId.value)) { - const workflow = queuedJobs.value[activeJobId.value]?.workflow + if (!jobIdToSessionWorkflowPath.value.has(jobId)) { + const workflow = queuedJobs.value[jobId]?.workflow if (workflow) { - ensureSessionWorkflowPath( - activeJobId.value, - workflow.path, - workflow.instanceId - ) + ensureSessionWorkflowPath(jobId, workflow.path, workflow.instanceId) } } - executionErrorStore.clearExecutionStartErrors( - runErrorKeyForJob(activeJobId.value) - ) - queuedJobs.value[activeJobId.value].executionStartedAt ??= performance.now() - setWorkflowStatus(activeJobId.value, { + queuedJobs.value[jobId].executionStartedAt ??= performance.now() + setWorkflowStatus(jobId, { status: 'running', - executionStartedAt: queuedJobs.value[activeJobId.value].executionStartedAt + executionStartedAt: queuedJobs.value[jobId].executionStartedAt }) + + // Only adopt as the global active job and clear shared UI state when the + // starting job belongs to the active workflow. Otherwise a job started + // from another tab would steal activeJobId and clobber the active tab's + // execution UI. + if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) return + + executionIdToLocatorCache.clear() + executionErrorStore.clearExecutionStartErrors(runErrorKeyForJob(jobId)) + activeJobId.value = jobId } function handleExecutionCached(e: CustomEvent) { if (!activeJob.value) return + if (!messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + return for (const n of e.detail.nodes) { activeJob.value.nodes[n] = true } @@ -528,17 +538,31 @@ export const useExecutionStore = defineStore('execution', () => { const workflow = jobIdToWorkflow.get(jobId) if (workflow) clearWorkflowStatus(workflow) if (activeJobId.value) clearInitializationByJobId(activeJobId.value) + if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { + // The job is finished either way, so its own records have to be released + // or they leak for the lifetime of the session. Only the shared UI state + // below belongs to the visible tab. Text previews stay gated: they are + // keyed by node, and the visible graph may hold the same node ids. + releaseFinishedJobRecords(jobId) + return + } resetExecutionState(jobId) } function handleExecuted(e: CustomEvent) { if (!activeJob.value) return + if (!messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + return activeJob.value.nodes[e.detail.node] = true } function handleExecutionSuccess(e: CustomEvent) { const jobId = e.detail.prompt_id pendingExecutionErrorsByJobId.delete(jobId) + clearInitializationByJobId(jobId) + // Per-workflow status is keyed by this job's own workflow, so it is set + // for every job — a background tab must still show Completed. Only the + // shared execution state below is gated. setWorkflowStatus(jobId, { status: 'completed', endTime: performance.now() @@ -558,10 +582,27 @@ export const useExecutionStore = defineStore('execution', () => { }) } } + if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { + // Finished either way, so release this job's own records or they leak for + // the session. Only the shared UI state below belongs to the visible tab, + // and text previews stay gated because they are keyed by node and the + // visible graph may hold the same node ids. + releaseFinishedJobRecords(jobId) + return + } resetExecutionState(jobId) } - function handleExecuting(e: CustomEvent): void { + function handleExecuting(e: CustomEvent): void { + // The event detail is just the node id, for extension compatibility, so + // the ids come from the raw message. Without this, the final + // `executing: null` of a job started in another workflow tab would clear + // the visible tab's active job and its node progress. + const raw = api.lastExecutingMessage + if (raw && !messageMatchesActiveWorkflow(raw.prompt_id, raw.workflow_id)) { + return + } + progressCoalescer.cancel() if (e.detail == null) progressStateCoalescer.cancel() @@ -570,7 +611,6 @@ export const useExecutionStore = defineStore('execution', () => { if (!activeJob.value) return - // Update the executing nodes list if (e.detail == null) { activeJobId.value = null } @@ -613,51 +653,156 @@ export const useExecutionStore = defineStore('execution', () => { progressStateCoalescer.push(e.detail) } + /** + * Revoke previews for nodes that just started executing. + * + * Uses the *actual* node id rather than the display node id intentionally, + * so the preview is not cleared every time a new node inside an expanded + * graph starts. + */ + function revokeStartedNodePreviews( + nodes: Record, + previousForJob: Record + ) { + const { revokePreviewsByExecutionId } = useNodeOutputStore() + for (const nodeId in nodes) { + if (nodes[nodeId].state !== 'running') continue + if (previousForJob[nodeId]?.state === 'running') continue + const executionId = tryNormalizeNodeExecutionId(nodeId) + if (executionId) revokePreviewsByExecutionId(executionId) + } + } + + /** Mirror the executing node's progress for backwards compatibility. */ + function mirrorExecutingNodeProgress( + nodes: Record + ) { + const executingId = executingNodeId.value + if (!executingId || !Object.hasOwn(nodes, executingId)) return + const nodeState = nodes[executingId] + _executingNodeProgress.value = { + value: nodeState.value, + max: nodeState.max, + prompt_id: nodeState.prompt_id, + node: nodeState.display_node_id || nodeState.node_id + } + } + function applyProgressState(detail: ProgressStateWsMessage) { - const { nodes, prompt_id: jobId } = detail + const { nodes, prompt_id: jobId, workflow_id: messageWorkflowId } = detail + const isActiveWorkflowMessage = messageMatchesActiveWorkflow( + jobId, + messageWorkflowId + ) - // Revoke previews for nodes that are starting to execute - const previousForJob = - jobId in nodeProgressStatesByJob.value - ? nodeProgressStatesByJob.value[jobId] - : {} - for (const nodeId in nodes) { - const nodeState = nodes[nodeId] - if (nodeState.state === 'running' && !(nodeId in previousForJob)) { - // This node just started executing, revoke its previews - // Note that we're doing the *actual* node id instead of the display node id - // here intentionally. That way, we don't clear the preview every time a new node - // within an expanded graph starts executing. - const { revokePreviewsByExecutionId } = useNodeOutputStore() - const executionId = tryNormalizeNodeExecutionId(nodeId) - if (executionId) revokePreviewsByExecutionId(executionId) - } + if (isActiveWorkflowMessage) { + revokeStartedNodePreviews( + nodes, + nodeProgressStatesByJob.value[jobId] ?? {} + ) } - // Update the progress states for all nodes nodeProgressStatesByJob.value = { ...nodeProgressStatesByJob.value, [jobId]: nodes } evictOldProgressJobs() + + // Per-job state is recorded for every workflow; only the active workflow + // writes the shared mirror the canvas reads. + if (!isActiveWorkflowMessage) return nodeProgressStates.value = nodes + mirrorExecutingNodeProgress(nodes) + } - // If we have progress for the currently executing node, update it for backwards compatibility - if (executingNodeId.value) { - const nodeState = Object.hasOwn(nodes, executingNodeId.value) - ? nodes[executingNodeId.value] - : undefined - if (!nodeState) return - _executingNodeProgress.value = { - value: nodeState.value, - max: nodeState.max, - prompt_id: nodeState.prompt_id, - node: nodeState.display_node_id || nodeState.node_id - } + /** + * Determines whether a WebSocket execution message belongs to the + * currently active workflow tab. Used to gate writes to the global + * "current execution" mirror so a job initiated from another open + * workflow cannot leak its progress into the active one. + * + * Resolution order: + * 1. `workflow_id` carried on the WS message (when backend supports it). + * 2. {@link jobIdToWorkflowId} mapping populated when the job was queued + * from this tab. + * 3. {@link jobIdToSessionWorkflowPath} mapping (path-based fallback). + * + * When the workflow cannot be resolved at all (e.g. job queued in a + * different browser session), the message is treated as belonging to + * the active workflow to preserve current behaviour for the existing + * single-tab common case. + */ + /** + * Graph id of the active workflow, or null when it has none. + * + * `LoadedComfyWorkflow` types both states as present, but the running store + * does not always honour that, and this is read from a watcher and from a + * computed now — so a missing state has to return null rather than throw and + * take unrelated features down with it. + */ + function activeWorkflowGraphId(): string | null { + const active: Partial | null = + workflowStore.activeWorkflow + if (!active) return null + return active.activeState?.id ?? active.initialState?.id ?? null + } + + /** + * A workflow id names a workflow, not an open tab, and several open tabs can + * carry the same one: `ensureWorkflowId` keeps whatever id the json already + * has, so copying a workflow into a new tab duplicates it, and loading one + * can leave both a temporary and a persisted entry holding it. So the id is + * the weakest signal here, not the strongest, and anything unique per tab has + * to be consulted first. QA hit this with three unsaved copies of the default + * graph, where every tab rendered every other tab's outputs. + */ + function messageMatchesActiveWorkflow( + jobId: JobId, + messageWorkflowId: string | undefined + ): boolean { + const activeWorkflow = workflowStore.activeWorkflow + if (!activeWorkflow) return true + + const mappedInstance = jobIdToWorkflowInstanceId.get(jobId) + if (mappedInstance && activeWorkflow.instanceId) { + return mappedInstance === activeWorkflow.instanceId + } + + const mappedPath = jobIdToSessionWorkflowPath.value.get(jobId) + if (mappedPath && activeWorkflow.path) { + return mappedPath === activeWorkflow.path + } + + const activeId = activeWorkflowGraphId() + if (activeId) { + const ownerId = messageWorkflowId || jobIdToWorkflowId.value.get(jobId) + if (ownerId) return ownerId === activeId } + + return true + } + + /** + * Returns true when workflow ownership for {@link jobId} can be resolved + * — either by an explicit `workflow_id` on the incoming message or by a + * mapping registered when the job was queued. When this returns false + * the caller should fall back to whatever legacy guard applied before + * workflow gating was introduced. + */ + function canResolveWorkflowOwnership( + jobId: JobId, + messageWorkflowId: string | undefined + ): boolean { + return ( + Boolean(messageWorkflowId) || + jobIdToWorkflowId.value.has(jobId) || + jobIdToSessionWorkflowPath.value.has(jobId) + ) } const progressCoalescer = createRafCoalescer((detail) => { + const { prompt_id: jobId, workflow_id: messageWorkflowId } = detail + if (!messageMatchesActiveWorkflow(jobId, messageWorkflowId)) return _executingNodeProgress.value = detail }, 'raf:progress') @@ -781,6 +926,11 @@ export const useExecutionStore = defineStore('execution', () => { executionErrorStore.recordExecutionError(detail, runErrorKey) executionErrorStore.showExecutionError(detail, runErrorKey) clearInitializationByJobId(detail.prompt_id) + // Only the active workflow's error resets the shared execution state; the + // dialog is already scoped by the run error key inside showExecutionError. + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + return + resetExecutionState(detail.prompt_id) } @@ -809,6 +959,9 @@ export const useExecutionStore = defineStore('execution', () => { return false clearInitializationByJobId(detail.prompt_id) + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + return true + resetExecutionState(detail.prompt_id) executionErrorStore.recordPromptError( { @@ -831,6 +984,9 @@ export const useExecutionStore = defineStore('execution', () => { if (!result) return false clearInitializationByJobId(detail.prompt_id) + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + return true + resetExecutionState(detail.prompt_id) if (result.kind === 'nodeErrors') { @@ -878,6 +1034,144 @@ export const useExecutionStore = defineStore('execution', () => { initializingJobIds.value = next } + /** + * Returns the prompt_id the global {@link nodeProgressStates} mirror belongs + * to, or null when it is empty. The mirror is replaced wholesale on every + * `progress_state` frame, so all of its entries share one prompt_id. + */ + function mirrorOwnerJobId(): JobId | null { + const entries = Object.values(nodeProgressStates.value) + if (entries.length === 0) return null + return entries[0].prompt_id + } + + /** + * Evict per-job execution artifacts for a job that has reached a terminal + * state, without disturbing state owned by a different running job. + * + * Unlike {@link resetExecutionState} this is safe for any jobId, including + * one that is not {@link activeJobId}. It is the recovery path for a dropped + * terminal WebSocket frame, which would otherwise leave node progress pinned + * forever: the backend broadcasts `execution_success` once and never retries. + * Idempotent. + */ + function evictTerminalJob(jobId: JobId) { + if (!jobId) return + + if (jobId in nodeProgressStatesByJob.value) { + const map = { ...nodeProgressStatesByJob.value } + delete map[jobId] + nodeProgressStatesByJob.value = map + } + + if (jobId in queuedJobs.value) { + const next = { ...queuedJobs.value } + delete next[jobId] + queuedJobs.value = next + } + + useJobPreviewStore().clearPreview(jobId) + clearInitializationByJobId(jobId) + clearTextPreviewsForJob(jobId) + + const isActive = activeJobId.value === jobId + // Only clear the shared mirror when it still belongs to the evicted job, + // otherwise evicting an old job would blank a live run's progress. + if (isActive || mirrorOwnerJobId() === jobId) { + nodeProgressStates.value = {} + executionIdToLocatorCache.clear() + } + + if (_executingNodeProgress.value?.prompt_id === jobId) { + _executingNodeProgress.value = null + } + + if (isActive) { + activeJobId.value = null + executionErrorStore.clearPromptError(runErrorKeyForJob(jobId)) + } + } + + /** + * Re-point the shared mirror at the now-active workflow's own job. + * + * Gating stops a background workflow's frames from *writing* the mirror, but + * whatever was written while that workflow was in front stays there — and + * `nodeLocationProgressStates` resolves node ids against the *currently* + * active graph, so two workflows that share a node id will show the stale + * entry on the newly visible node. Replay the active workflow's own per-job + * progress instead, or clear the mirror when it has no running job. + */ + function reconcileMirrorForActiveWorkflow() { + const activeWorkflow = workflowStore.activeWorkflow + if (!activeWorkflow) return + + const activeId = activeWorkflowGraphId() + const activePath = activeWorkflow.path + + const jobIds = Object.keys(nodeProgressStatesByJob.value) + let matchedJobId: JobId | null = null + for (let i = jobIds.length - 1; i >= 0; i--) { + const jobId = jobIds[i] + const idMatch = + activeId !== null && jobIdToWorkflowId.value.get(jobId) === activeId + const pathMatch = + jobIdToSessionWorkflowPath.value.get(jobId) === activePath + if (idMatch || pathMatch) { + matchedJobId = jobId + break + } + } + + if (matchedJobId) { + nodeProgressStates.value = + nodeProgressStatesByJob.value[matchedJobId] ?? {} + executionIdToLocatorCache.clear() + if (_executingNodeProgress.value?.prompt_id !== matchedJobId) { + _executingNodeProgress.value = null + } + return + } + + if (Object.keys(nodeProgressStates.value).length > 0) { + nodeProgressStates.value = {} + executionIdToLocatorCache.clear() + } + _executingNodeProgress.value = null + } + + watch( + () => workflowStore.activeWorkflow, + () => { + reconcileMirrorForActiveWorkflow() + } + ) + + /** + * Reconcile tracked per-job state against the backend's authoritative job + * sets. A job the backend reports as terminal but which still holds progress + * state lost its terminal frame, so evict it. + * + * @param activeJobIds jobs the backend reports as Running or Pending + * @param terminalJobIds jobs the backend reports in history + */ + function reconcileTerminalJobs( + activeJobIds: Set, + terminalJobIds: Set + ) { + const tracked = new Set([ + ...Object.keys(nodeProgressStatesByJob.value), + ...initializingJobIds.value + ]) + if (activeJobId.value) tracked.add(activeJobId.value) + + for (const jobId of tracked) { + if (activeJobIds.has(jobId)) continue + if (!terminalJobIds.has(jobId)) continue + evictTerminalJob(jobId) + } + } + function reconcileInitializingJobs(activeJobIds: Set) { const orphaned = [...initializingJobIds.value].filter( (id) => !activeJobIds.has(id) @@ -924,6 +1218,17 @@ export const useExecutionStore = defineStore('execution', () => { /** * Reset execution-related state after a run completes or is stopped. */ + function releaseFinishedJobRecords(jobId: JobId) { + if (jobId in queuedJobs.value) delete queuedJobs.value[jobId] + if (jobId in nodeProgressStatesByJob.value) { + const map = { ...nodeProgressStatesByJob.value } + delete map[jobId] + nodeProgressStatesByJob.value = map + } + jobIdToWorkflow.delete(jobId) + useJobPreviewStore().clearPreview(jobId) + } + function resetExecutionState(jobIdParam?: JobId | null) { cancelPendingProgressUpdates() @@ -952,15 +1257,35 @@ export const useExecutionStore = defineStore('execution', () => { : nodeIdStr } + /** + * Whether a text preview belongs on the visible canvas. + * + * Prefers the workflow-ownership gate and falls back to the legacy + * active-prompt guard only when ownership is unresolvable: activeJobId can + * point at another workflow's job, which would otherwise drop frames for the + * workflow the user is looking at. + * + * Exported because node outputs are applied outside this store, in ComfyApp's + * own `executed` listener, which resolves the execution id against the + * visible graph. Without this gate a finished job writes its output onto the + * same-numbered node of whatever tab is in front. + */ + function frameBelongsToVisibleWorkflow( + promptId: string | undefined, + workflowId: string | undefined + ): boolean { + if (!promptId) return true + if (canResolveWorkflowOwnership(promptId, workflowId)) { + return messageMatchesActiveWorkflow(promptId, workflowId) + } + return !activeJobId.value || promptId === activeJobId.value + } + function handleProgressText(e: CustomEvent) { - const { nodeId, text, prompt_id } = e.detail + const { nodeId, text, prompt_id, workflow_id } = e.detail if (!text || !nodeId) return + if (!frameBelongsToVisibleWorkflow(prompt_id, workflow_id)) return - // Filter: only accept progress for the active prompt - if (prompt_id && activeJobId.value && prompt_id !== activeJobId.value) - return - - // Handle execution node IDs for subgraphs const currentId = getNodeIdIfExecuting(nodeId) if (!currentId) return const parsedCurrentId = parseNodeId(currentId) @@ -1157,6 +1482,8 @@ export const useExecutionStore = defineStore('execution', () => { clearInitializationByJobId, clearInitializationByJobIds, reconcileInitializingJobs, + reconcileTerminalJobs, + reconcileMirrorForActiveWorkflow, clearActiveJobIfStale, bindExecutionEvents, unbindExecutionEvents, @@ -1172,6 +1499,7 @@ export const useExecutionStore = defineStore('execution', () => { ensureSessionWorkflowPath, getWorkflowStatus, clearWorkflowStatus, - rewriteSessionWorkflowPaths + rewriteSessionWorkflowPaths, + frameBelongsToVisibleWorkflow } }) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts new file mode 100644 index 00000000000..02602dd5280 --- /dev/null +++ b/src/stores/executionStore.workflowGating.test.ts @@ -0,0 +1,1156 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { nextTick } from 'vue' + +import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' +import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' +import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' +import { api } from '@/scripts/api' +import { useExecutionStore } from '@/stores/executionStore' +import { useNodeOutputStore } from '@/stores/nodeOutputStore' + +/** + * Workflow-ownership gating for execution WebSocket messages. + * + * Core stamps `workflow_id` on every JSON frame of a running prompt, so a frame + * that belongs to another open workflow tab must not write the shared "current + * execution" state. Three resolution paths have to keep working, because a + * frontend talks to backends of three different vintages: + * + * - **modern**: the frame carries `workflow_id` + * - **legacy**: no `workflow_id` anywhere — ownership resolves from the + * mapping registered when the job was queued, or from the session path + * - **unresolvable**: neither is available, and behaviour must stay exactly as + * it was before gating existed (permissive) + */ + +const { mockShowTextPreview } = await vi.hoisted(async () => ({ + mockShowTextPreview: vi.fn() +})) + +vi.mock(import('@/composables/useAppMode')) +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) + +vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ + useNodeProgressText: () => ({ + showTextPreview: mockShowTextPreview, + removeTextPreview: vi.fn() + }) +})) + +type EventHandler = (event: CustomEvent) => void +const apiEventHandlers = new Map() + +vi.mock(import('@/scripts/api'), () => ({ + api: { + addEventListener: vi.fn((event: string, handler: EventHandler) => { + apiEventHandlers.set(event, handler) + }), + removeEventListener: vi.fn((event: string) => { + apiEventHandlers.delete(event) + }), + clientId: 'test-client', + apiURL: vi.fn((path: string) => `/api${path}`), + lastExecutingMessage: null + } +})) + +vi.mock(import('@/scripts/app'), () => { + const rootGraph = { getNodeById: vi.fn(), nodes: [] } + return { + app: { + rootGraph, + rootGraphOrUndefined: rootGraph, + revokePreviews: vi.fn(), + nodePreviewImages: {} + } + } +}) + +const WORKFLOW_A_ID = 'workflow-a-id' +const WORKFLOW_B_ID = 'workflow-b-id' + +function workflow(id: string, path: string): LoadedComfyWorkflow { + return fromPartial({ + activeState: { id }, + initialState: { id }, + path, + // Every open tab carries its own instanceId in the running app, and it is + // the only identifier that is unique per tab rather than per workflow. + instanceId: `instance-${path}` + }) +} + +const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') +const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') + +/** + * Copy a workflow, open a new tab and paste it: `ensureWorkflowId` keeps the + * id already in the pasted json, so both tabs carry the same workflow id while + * being different open workflows. Reported by QA on the core PR thread, with + * three unsaved copies of the default graph. + */ +const workflowACopy = workflow(WORKFLOW_A_ID, 'workflows/a copy.json') + +const RAF_COALESCED = new Set(['progress_state', 'progress']) + +/** + * `executing` is dispatched with a bare node id for extension compatibility, + * so the store reads the ids off the raw message api recorded. Mirror that. + */ +function fireExecuting( + node: string | null, + ids: { prompt_id: string; workflow_id?: string } +) { + api.lastExecutingMessage = { node, ...ids } + const handler = apiEventHandlers.get('executing') + if (!handler) throw new Error('executing handler not bound') + handler(new CustomEvent('executing', { detail: node })) +} + +function fire(event: string, detail: Record) { + const handler = apiEventHandlers.get(event) + if (!handler) throw new Error(`${event} handler not bound`) + handler(new CustomEvent(event, { detail })) + // progress frames are RAF-coalesced in the store; flush so the assertion + // sees the applied state rather than the pending batch. + if (RAF_COALESCED.has(event)) vi.advanceTimersToNextFrame() +} + +function nodeState( + jobId: string, + nodeId: string, + state: NodeProgressState['state'], + value = 0 +): NodeProgressState { + return { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state, + value, + max: 10 + } +} + +describe('executionStore workflow gating', () => { + let store: ReturnType + let revokePreviews: ReturnType + + beforeEach(() => { + apiEventHandlers.clear() + api.lastExecutingMessage = null + mockShowTextPreview.mockClear() + revokePreviews = vi + .spyOn(useNodeOutputStore(), 'revokePreviewsByExecutionId') + .mockImplementation(() => {}) + useWorkflowStore().activeWorkflow = null + Object.assign(useWorkflowStore(), { openWorkflows: [workflowA, workflowB] }) + vi.mocked(useWorkflowStore().isOpen).mockImplementation((wf) => + useWorkflowStore().openWorkflows.some((open) => open.path === wf.path) + ) + store = useExecutionStore() + store.bindExecutionEvents() + }) + + /** Queue a job from `wf` so the id mapping exists, as storeJob does. */ + function queueLegacyJobFrom(jobId: string, wf: LoadedComfyWorkflow) { + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + startTime: 42, + submissionAcceptedAt: 62, + workflow: wf, + mode: 'graph' + }) + } + + function queueJobFrom(jobId: string, wf: LoadedComfyWorkflow) { + const graphId = wf.activeState.id ?? wf.initialState.id + if (!graphId) throw new Error('workflow graph id missing') + store.registerJobWorkflowIdMapping(jobId, graphId) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + startTime: 42, + submissionAcceptedAt: 62, + workflow: wf, + mode: 'graph' + }) + } + + describe('execution_start', () => { + it('adopts activeJobId for a frame from the active workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('does not steal activeJobId for a frame from another workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('still records the other workflow as running', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + + expect(store.getWorkflowStatus(workflowB)).toBe('running') + expect(store.activeJobId).toBeNull() + }) + + it('adopts the job when there is no active workflow at all', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-a') + }) + }) + + describe('node completion frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('marks nodes for a frame from the active workflow', () => { + fire('executed', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + node: '1', + display_node: '1', + output: {} + }) + + expect(store.activeJob?.nodes['1']).toBe(true) + }) + + it('executed from another workflow does not mark active job nodes', () => { + fire('executed', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + node: '1', + display_node: '1', + output: {} + }) + + expect(store.activeJob?.nodes['1']).toBeUndefined() + }) + + it('execution_cached from another workflow does not mark active job nodes', () => { + fire('execution_cached', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: ['1'], + timestamp: 2 + }) + + expect(store.activeJob?.nodes['1']).toBeUndefined() + }) + + it('execution_cached from the active workflow does mark them', () => { + fire('execution_cached', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: ['1'], + timestamp: 2 + }) + + expect(store.activeJob?.nodes['1']).toBe(true) + }) + }) + + describe('terminal frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('execution_success from another workflow does not clear activeJobId', () => { + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + // Regression: gating the whole handler hid the completed badge on a + // background tab, which workflowTabStatus.spec.ts caught on CI. Per-workflow + // status is keyed by the job's own workflow and is never shared state. + it('still marks another workflow completed on its own tab', () => { + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + expect(store.getWorkflowStatus(workflowB)).toBe('running') + + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.getWorkflowStatus(workflowB)).toBe('completed') + expect(store.activeJobId).toBe('job-a') + }) + + it('execution_success from the active workflow clears it', () => { + fire('execution_success', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBeNull() + }) + + it('clears the initializing flag for the completed job either way', () => { + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) + + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.isJobInitializing('job-b')).toBe(false) + }) + + it('execution_interrupted from another workflow does not clear activeJobId', () => { + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + executed: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('execution_error from another workflow leaves active state alone but clears its initializing flag', () => { + queueJobFrom('job-b', workflowB) + + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [] + }) + + expect(store.activeJobId).toBe('job-a') + expect(store.isJobInitializing('job-b')).toBe(false) + }) + }) + + describe('progress_state', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('updates the global mirror when the frame matches the active workflow', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(3) + }) + + it('skips the global mirror when the frame is from another workflow', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + + it('records per-job progress regardless of which workflow sent it', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(7) + }) + + it('updates the mirror when there is no active workflow', () => { + useWorkflowStore().activeWorkflow = null + + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(7) + }) + + it('updates _executingNodeProgress on a workflow_id match', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fireExecuting('1', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 4) } + }) + + expect(store._executingNodeProgress?.value).toBe(4) + }) + + it('skips _executingNodeProgress on a workflow_id mismatch', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fireExecuting('1', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 9) } + }) + + expect(store._executingNodeProgress?.value).not.toBe(9) + }) + }) + + describe('preview revocation', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('revokes previews for a frame from the active workflow', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running') } + }) + + expect(revokePreviews).toHaveBeenCalled() + }) + + it('does not revoke previews for a frame from another workflow', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running') } + }) + + expect(revokePreviews).not.toHaveBeenCalled() + }) + + it('revokes when a node transitions pending -> running', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'pending') } + }) + expect(revokePreviews).not.toHaveBeenCalled() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running') } + }) + + expect(revokePreviews).toHaveBeenCalled() + }) + + it('does not revoke twice while a node stays running', () => { + const running = { '1': nodeState('job-a', '1', 'running') } + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: running + }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: running + }) + + expect(revokePreviews).toHaveBeenCalledTimes(1) + }) + }) + + describe('legacy backends that never send workflow_id', () => { + it('falls back to the job -> workflow id mapping', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + fire('execution_start', { prompt_id: 'job-a', timestamp: 1 }) + + fire('progress_state', { + prompt_id: 'job-b', + nodes: { '1': nodeState('job-b', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(5) + }) + + it('uses the mapping to accept a frame for the active workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(5) + }) + + it('stays permissive when ownership cannot be resolved at all', () => { + useWorkflowStore().activeWorkflow = workflowA + + // No workflow_id on the frame and no mapping for this job: the job could + // have been queued by another browser session, and single-tab behaviour + // must not regress. + fire('progress_state', { + prompt_id: 'job-from-elsewhere', + nodes: { '1': nodeState('job-from-elsewhere', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(5) + }) + + it('does not steal activeJobId from a mapped foreign job', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + fire('execution_start', { prompt_id: 'job-b', timestamp: 1 }) + + expect(store.activeJobId).toBeNull() + }) + }) + + describe('progress_text', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('drops text whose workflow_id does not match the active workflow', () => { + queueJobFrom('job-b', workflowB) + + fire('progress_text', { + nodeId: '1', + text: 'hello', + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID + }) + + expect(mockShowTextPreview).not.toHaveBeenCalled() + }) + + it('drops text with no id information when a different job is active', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + fire('progress_text', { + nodeId: '1', + text: 'hello', + prompt_id: 'job-unknown' + }) + + expect(mockShowTextPreview).not.toHaveBeenCalled() + }) + }) + + describe('metadata hygiene', () => { + it('ignores an empty-string workflow_id rather than matching on it', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + // Core omits the field entirely when unknown, never sends "" — but a + // third-party server might, and an empty id must not read as "mine". + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: '', + nodes: { '1': nodeState('job-b', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + + it('gates on workflow_id even when the job mapping disagrees', () => { + useWorkflowStore().activeWorkflow = workflowA + // Mapping says workflow A, the frame says workflow B. The frame wins: + // the server is the authority on which workflow produced the prompt. + store.registerJobWorkflowIdMapping('job-x', WORKFLOW_A_ID) + + fire('progress_state', { + prompt_id: 'job-x', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-x', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + }) + + describe('executing frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + // The event detail is a bare node id for extension compatibility, so this + // path can only gate via the raw message api records. Before that, the + // other tab's final `executing: null` cleared the visible tab's run. + it('does not let another workflow clear the visible active job', () => { + fireExecuting(null, { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('clears the active job on the visible workflow own terminal executing', () => { + fireExecuting(null, { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + expect(store.activeJobId).toBeNull() + }) + + it('does not clear node progress for another workflow executing frame', () => { + fireExecuting('1', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 5) } + }) + expect(store._executingNodeProgress?.value).toBe(5) + + fireExecuting('9', { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID }) + + expect(store._executingNodeProgress?.value).toBe(5) + }) + + // api clears the raw message straight after dispatch, so an `executing` + // dispatched later by an extension is not read as belonging to whichever + // run arrived last. Without that, a foreign frame would silently gate out + // every extension-driven executing event that followed it. + it('is not gated by a stale raw message from an earlier frame', () => { + fireExecuting(null, { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID + }) + expect(store.activeJobId).toBe('job-a') + + api.lastExecutingMessage = null + const handler = apiEventHandlers.get('executing')! + handler(new CustomEvent('executing', { detail: null })) + + expect(store.activeJobId).toBeNull() + }) + + it('still works when the raw message was never recorded', () => { + // Defensive: an `executing` dispatched by an extension or a test rather + // than by the socket has no raw message, and must behave as before. + const handler = apiEventHandlers.get('executing')! + handler(new CustomEvent('executing', { detail: null })) + + expect(store.activeJobId).toBeNull() + }) + }) + + describe('progress frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('applies a progress frame from the active workflow', () => { + fire('progress', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + node: '1', + value: 4, + max: 10 + }) + + expect(store._executingNodeProgress?.value).toBe(4) + }) + + it('drops a progress frame from another workflow', () => { + fire('progress', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + node: '1', + value: 7, + max: 10 + }) + + expect(store._executingNodeProgress).toBeNull() + }) + }) + + describe('mixed backend, upgraded mid-session', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('accepts its own frames before and after the field appears', () => { + queueJobFrom('job-a', workflowA) + + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 2) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(2) + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(6) + }) + + it('rejects a foreign job both before and after the field appears', () => { + queueJobFrom('job-b', workflowB) + + fire('progress_state', { + prompt_id: 'job-b', + nodes: { '1': nodeState('job-b', '1', 'running', 2) } + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(6) + }) + }) + + describe('switching tabs mid-run', () => { + // The QA run on a real generation found this: gating stops a background + // workflow writing the mirror, but whatever it wrote while it was in front + // stays there, and `nodeLocationProgressStates` resolves node ids against + // the *currently* active graph. Two workflows sharing a node id therefore + // show the stale entry on the newly visible node. + it('clears the mirror when switching to a workflow with no running job', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(3) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store._executingNodeProgress).toBeNull() + }) + + it('replays the active workflow own progress when switching back', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + expect(store.nodeProgressStates['1']).toBeUndefined() + + // A's run advanced while it was in the background; per-job state kept it. + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 8) } + }) + + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + + expect(store.nodeProgressStates['1']?.value).toBe(8) + }) + + it('keeps recording per-job progress for the backgrounded workflow', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 8) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-a']?.['1']?.value).toBe(8) + }) + + it('shows the newly active workflow own job rather than the one it replaced', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 2) } + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(2) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']?.value).toBe(7) + expect(store.nodeProgressStates['1']?.prompt_id).toBe('job-b') + }) + // A legacy server sends no workflow_id at all, so ownership resolves + // through the session path the client recorded at queue time. The QA pass + // against core f1072eb0 saw foreign progress here; it must clear on switch + // on that backend too, not only on one that stamps the frames. + it('clears the mirror on a backend that sends no workflow id', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueLegacyJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(3) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + // Round 2 of the GPU QA: the first switch is clean, a later one is not. + it('stays clean across repeated switches during one run', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + for (const [i, step] of [3, 5, 7, 9].entries()) { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', step) } + }) + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + expect( + store.nodeProgressStates['1'], + `switch ${i + 1} to B must not inherit A` + ).toBeUndefined() + + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + expect(store.nodeProgressStates['1']?.value).toBe(step) + } + }) + + it('stays clean while the active workflow is queued behind the running one', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + useWorkflowStore().activeWorkflow = workflowB + queueJobFrom('job-b', workflowB) + await nextTick() + expect(store.nodeProgressStates['1']).toBeUndefined() + + // A keeps running; B's own job has not started yet. + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + // The canvas stamps progress onto the new graph's nodes when the graph + // changes (nodeProgressCanvasSync.replaceGraph), reading whatever the + // mirror holds at that instant. Clearing the mirror on a watcher is a race + // against that, and it is the race the GPU QA lost. So the projection has + // to be correct even when the mirror still holds the foreign entry. + it('does not project a foreign entry that is still in the mirror', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(Object.keys(store.nodeLocationProgressStates)).toHaveLength(1) + + // Switch without letting the watcher run, which is what the canvas sees + // when it observes the graph change first. + useWorkflowStore().activeWorkflow = workflowB + + expect(store.nodeProgressStates['1']?.value).toBe(3) + expect(store.nodeLocationProgressStates).toEqual({}) + }) + + it('projects the active workflow own entry', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + const projected = Object.values(store.nodeLocationProgressStates) + expect(projected).toHaveLength(1) + expect(projected[0]?.value).toBe(3) + }) + }) + + describe('two open workflows that share an id', () => { + beforeEach(() => { + Object.assign(useWorkflowStore(), { + openWorkflows: [workflowA, workflowACopy] + }) + }) + + // Node outputs are written by ComfyApp's own `executed` listener, not by + // this store, and it resolves the node against the visible graph. The gate + // it consults is this predicate. + it('rejects a foreign job output frame for the copy', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-copy', workflowACopy) + + useWorkflowStore().activeWorkflow = workflowACopy + + expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( + false + ) + expect( + store.frameBelongsToVisibleWorkflow('job-copy', WORKFLOW_A_ID) + ).toBe(true) + }) + + it('does not project a foreign job progress onto the copy', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-copy', workflowACopy) + + useWorkflowStore().activeWorkflow = workflowACopy + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeLocationProgressStates).toEqual({}) + }) + + it('still renders the copy own job', () => { + useWorkflowStore().activeWorkflow = workflowACopy + queueJobFrom('job-copy', workflowACopy) + fire('progress_state', { + prompt_id: 'job-copy', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-copy', '1', 'running', 4) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(4) + }) + }) + + /** + * The resolution order is load-bearing and was wrong once: keying on the + * graph id first let every tab sharing an id claim every job. These pin the + * order itself, so reordering the legs fails here rather than in QA. + */ + describe('ownership resolution precedence', () => { + beforeEach(() => { + Object.assign(useWorkflowStore(), { + openWorkflows: [workflowA, workflowACopy] + }) + }) + + it('prefers the tab instance over a matching graph id', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + // The copy's graph id is identical, so the id leg would say yes. + useWorkflowStore().activeWorkflow = workflowACopy + expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( + false + ) + }) + + it('falls back to the session path when no instance is known', () => { + useWorkflowStore().activeWorkflow = workflowA + // No storeJob, so no instance mapping: register only the path. + store.ensureSessionWorkflowPath('job-pathonly', workflowACopy.path) + + expect( + store.frameBelongsToVisibleWorkflow('job-pathonly', WORKFLOW_A_ID), + 'the path says the copy owns it, even though the graph id matches A' + ).toBe(false) + useWorkflowStore().activeWorkflow = workflowACopy + expect( + store.frameBelongsToVisibleWorkflow('job-pathonly', WORKFLOW_A_ID) + ).toBe(true) + }) + + it('falls back to the graph id when neither instance nor path is known', () => { + useWorkflowStore().activeWorkflow = workflowA + store.registerJobWorkflowIdMapping('job-elsewhere', WORKFLOW_B_ID) + + expect( + store.frameBelongsToVisibleWorkflow('job-elsewhere', WORKFLOW_B_ID) + ).toBe(false) + expect( + store.frameBelongsToVisibleWorkflow('job-elsewhere', WORKFLOW_A_ID) + ).toBe(true) + }) + + it('stays permissive when ownership cannot be resolved at all', () => { + useWorkflowStore().activeWorkflow = workflowA + + // A job from another browser session: no mapping, no frame id. Keeping + // this permissive is what preserves single-tab behaviour. + expect( + store.frameBelongsToVisibleWorkflow('job-unknown', undefined) + ).toBe(true) + }) + + it('keeps a running job with its tab across a rename', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + // renameWorkflow mutates the same object and rewrites the mapping by + // instanceId, so the run must survive being given a new name mid-flight. + store.rewriteSessionWorkflowPaths( + workflowA.instanceId, + 'workflows/renamed.json' + ) + expect(store.jobIdToSessionWorkflowPath.get('job-a')).toBe( + 'workflows/renamed.json' + ) + expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( + true + ) + }) + + it('does not give a running job to the copy that Save As creates', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + // Save As mints a new workflow with its own instanceId and a new graph + // id, and switches to it. The original tab keeps the run. + const savedAs = workflow( + 'a-brand-new-graph-id', + 'workflows/saved as.json' + ) + Object.assign(useWorkflowStore(), { + openWorkflows: [workflowA, savedAs] + }) + useWorkflowStore().activeWorkflow = savedAs + + expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( + false + ) + useWorkflowStore().activeWorkflow = workflowA + expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( + true + ) + }) + }) +}) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index de3c7754f5b..b2f8c8d7e90 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -330,6 +330,10 @@ export const useQueueStore = defineStore('queue', () => { api.getHistory(maxHistoryItems.value) ]) + // Shared by the queue and history branches below. + + let activeJobIds: Set | null = null + if (queueResult.status === 'fulfilled') { const queue = queueResult.value // API returns pre-sorted data (sort_by=create_time&order=desc) @@ -346,7 +350,7 @@ export const useQueueStore = defineStore('queue', () => { } }) - const activeJobIds = new Set([ + activeJobIds = new Set([ ...queue.Running.map((j) => j.id), ...queue.Pending.map((j) => j.id) ]) @@ -359,6 +363,17 @@ export const useQueueStore = defineStore('queue', () => { const history = historyResult.value const currentHistory = toValue(historyTasks) + // A job the backend has moved to history while we still hold progress + // state for it lost its terminal WebSocket frame. Evicting it here is + // the only path that unsticks node progress in that case. + if (history.length > 0 && activeJobIds) { + const terminalJobIds = new Set(history.map((j) => j.id)) + useExecutionStore().reconcileTerminalJobs( + activeJobIds, + terminalJobIds + ) + } + // Sort by create_time descending and limit to maxItems const sortedHistory = [...history] .sort((a, b) => b.create_time - a.create_time) From 1f1f1fa7344cec29202ff3c3a922ab2269b44c1f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 6 Oct 2026 10:26:38 -0700 Subject: [PATCH 07/52] fix(execution): match restored mirror jobs by ownership, and finish eviction Addresses review on #20045. reconcileMirrorForActiveWorkflow matched on the graph id, which several open tabs can share, so it could re-point the mirror at another tab's job. It now defers to the same resolver the handlers use and requires ownership to be resolvable rather than adopting a job it cannot place. evictTerminalJob cleared text previews after deleting the queuedJobs entry that clearTextPreviewsForJob reads, so it removed nothing and the progress_text widget stayed on the node. It also never released the workflow status, leaving a tab marked running forever once a terminal frame was dropped. Both fixed, with tests that fail without them. Also corrects the resolution order in the JSDoc, which still described the old order and sat above the wrong function; dispatches `executing` with the production event shape in the recovery suite; and unrolls the repeated-switch loop per the test guidelines. --- .../tests/twoTabExecutionLeak.spec.ts | 21 ++--- .../executionStore.terminalRecovery.test.ts | 88 ++++++++++++++++--- src/stores/executionStore.ts | 60 +++++++------ .../executionStore.workflowGating.test.ts | 29 ++++-- 4 files changed, 142 insertions(+), 56 deletions(-) diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts index 7b9fc2009cf..189c2063c6e 100644 --- a/browser_tests/tests/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/twoTabExecutionLeak.spec.ts @@ -183,10 +183,9 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { // Both workflows own a KSampler numbered 3, which is what makes this // reachable: node progress is keyed by a locator resolved against the // graph that happens to be in front, so A's entry lands on B's node. - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: 0.25, - outlined: true - }) + await expect + .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) + .toEqual({ progress: 0.25, outlined: true }) await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) await comfyPage.nextFrame() @@ -209,10 +208,9 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { // Returning to A shows where its run actually got to, not where it was // when the user left. - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: 0.75, - outlined: true - }) + await expect + .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) }) test('stays clean across repeated switches during one run', async ({ @@ -238,10 +236,9 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { await comfyPage.nextFrame() simulator.play([...running.nodeRunning(KSAMPLER_NODE, step, 10)]) await comfyPage.nextFrame() - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: step / 10, - outlined: true - }) + await expect + .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) + .toEqual({ progress: step / 10, outlined: true }) await comfyPage.workflow.switchToTab(WORKFLOW_B) await comfyPage.nextFrame() diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index 0f8999f4600..92b60f919aa 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' +import { api } from '@/scripts/api' import { useExecutionStore } from '@/stores/executionStore' /** @@ -16,9 +17,12 @@ import { useExecutionStore } from '@/stores/executionStore' * reports the job in history while the frontend still holds progress for it. */ -const { mockShowTextPreview } = await vi.hoisted(async () => ({ - mockShowTextPreview: vi.fn() -})) +const { mockShowTextPreview, mockRemoveTextPreview } = await vi.hoisted( + async () => ({ + mockShowTextPreview: vi.fn(), + mockRemoveTextPreview: vi.fn() + }) +) vi.mock(import('@/composables/useAppMode')) vi.mock(import('@/platform/telemetry')) @@ -27,7 +31,7 @@ vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ useNodeProgressText: () => ({ showTextPreview: mockShowTextPreview, - removeTextPreview: vi.fn() + removeTextPreview: mockRemoveTextPreview }) })) @@ -43,7 +47,8 @@ vi.mock(import('@/scripts/api'), () => ({ apiEventHandlers.delete(event) }), clientId: 'test-client', - apiURL: vi.fn((path: string) => `/api${path}`) + apiURL: vi.fn((path: string) => `/api${path}`), + lastExecutingMessage: null } })) @@ -73,6 +78,21 @@ function workflow(id: string, path: string): LoadedComfyWorkflow { const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') +/** + * `executing` is dispatched with a bare node id for extension compatibility, so + * the store reads the ids off the raw message api recorded. Mirror that rather + * than passing an object as the detail, which would pass for the wrong reason. + */ +function fireExecuting( + node: string | null, + ids: { prompt_id: string; workflow_id?: string } +) { + api.lastExecutingMessage = { node, ...ids } + const handler = apiEventHandlers.get('executing') + if (!handler) throw new Error('executing handler not bound') + handler(new CustomEvent('executing', { detail: node })) +} + function fire(event: string, detail: Record) { const handler = apiEventHandlers.get(event) if (!handler) throw new Error(`${event} handler not bound`) @@ -116,12 +136,7 @@ describe('executionStore terminal-job recovery', () => { workflow_id: WORKFLOW_A_ID, timestamp: 1 }) - fire('executing', { - prompt_id: jobId, - workflow_id: WORKFLOW_A_ID, - node: '1', - display_node: '1' - }) + fireExecuting('1', { prompt_id: jobId, workflow_id: WORKFLOW_A_ID }) fire('progress_state', { prompt_id: jobId, workflow_id: WORKFLOW_A_ID, @@ -229,4 +244,55 @@ describe('executionStore terminal-job recovery', () => { expect(store.nodeProgressStates['1']).toBeUndefined() }) + /** As above, but queued through storeJob, which is what associates the job + * with its workflow. The status and text-preview paths both read that. */ + function stuckQueuedJob(jobId = 'job-queued') { + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowA, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + return jobId + } + + it('releases the workflow status when a terminal frame was dropped', () => { + const jobId = stuckQueuedJob() + expect(store.getWorkflowStatus(workflowA)).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + // handleExecutionStart set this and nothing else resets it, so without the + // release the tab keeps claiming it is running for the rest of the session. + expect(store.getWorkflowStatus(workflowA)).toBeUndefined() + }) + + it('removes the text preview before the job record it reads is deleted', async () => { + mockRemoveTextPreview.mockClear() + vi.mocked(useWorkflowStore().executionIdToCurrentId).mockReturnValue('1') + const { useCanvasStore } = + await import('@/renderer/core/canvas/canvasStore') + useCanvasStore().canvas = fromPartial({ + graph: { getNodeById: () => fromPartial({}) } + }) + const jobId = stuckQueuedJob() + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + // clearTextPreviewsForJob reads the job's node list out of queuedJobs and + // returns early once it is gone, so order is the whole point here. + expect(mockRemoveTextPreview).toHaveBeenCalled() + }) }) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 6e31030698c..afcf67979fd 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -715,23 +715,6 @@ export const useExecutionStore = defineStore('execution', () => { mirrorExecutingNodeProgress(nodes) } - /** - * Determines whether a WebSocket execution message belongs to the - * currently active workflow tab. Used to gate writes to the global - * "current execution" mirror so a job initiated from another open - * workflow cannot leak its progress into the active one. - * - * Resolution order: - * 1. `workflow_id` carried on the WS message (when backend supports it). - * 2. {@link jobIdToWorkflowId} mapping populated when the job was queued - * from this tab. - * 3. {@link jobIdToSessionWorkflowPath} mapping (path-based fallback). - * - * When the workflow cannot be resolved at all (e.g. job queued in a - * different browser session), the message is treated as belonging to - * the active workflow to preserve current behaviour for the existing - * single-tab common case. - */ /** * Graph id of the active workflow, or null when it has none. * @@ -748,6 +731,19 @@ export const useExecutionStore = defineStore('execution', () => { } /** + * Determines whether a WebSocket execution message belongs to the currently + * active workflow tab, so a job started in another open workflow cannot leak + * its progress into the one on screen. + * + * Resolution order, strongest first: + * 1. {@link jobIdToWorkflowInstanceId}, the id of the open tab that queued it. + * 2. {@link jobIdToSessionWorkflowPath}, unique per open workflow. + * 3. the graph id, from the message or {@link jobIdToWorkflowId}. + * + * When ownership cannot be resolved at all, for instance a job queued in a + * different browser session, the message is treated as belonging to the + * active workflow, which preserves single-tab behaviour. + * * A workflow id names a workflow, not an open tab, and several open tabs can * carry the same one: `ensureWorkflowId` keeps whatever id the json already * has, so copying a workflow into a new tab duplicates it, and loading one @@ -1064,6 +1060,18 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStatesByJob.value = map } + // Before the queuedJobs entry goes: clearTextPreviewsForJob reads the job's + // node list out of it and returns early if it is already gone, so clearing + // after the delete removed nothing and left the progress_text widget on the + // node, which is the thing this eviction path exists to fix. + clearTextPreviewsForJob(jobId) + + // A dropped terminal frame left the workflow marked running forever, since + // handleExecutionStart set that and nothing else resets it. + const evictedWorkflow = jobIdToWorkflow.get(jobId) + if (evictedWorkflow) clearWorkflowStatus(evictedWorkflow) + jobIdToWorkflow.delete(jobId) + if (jobId in queuedJobs.value) { const next = { ...queuedJobs.value } delete next[jobId] @@ -1072,7 +1080,6 @@ export const useExecutionStore = defineStore('execution', () => { useJobPreviewStore().clearPreview(jobId) clearInitializationByJobId(jobId) - clearTextPreviewsForJob(jobId) const isActive = activeJobId.value === jobId // Only clear the shared mirror when it still belongs to the evicted job, @@ -1106,18 +1113,19 @@ export const useExecutionStore = defineStore('execution', () => { const activeWorkflow = workflowStore.activeWorkflow if (!activeWorkflow) return - const activeId = activeWorkflowGraphId() - const activePath = activeWorkflow.path - + // Ownership has exactly one definition, and it is not the graph id: several + // open tabs can carry the same one. Matching here on the graph id is the bug + // this feature already shipped once, so defer to the same resolver the + // handlers use, and require ownership to be resolvable rather than adopting + // a job we cannot place. const jobIds = Object.keys(nodeProgressStatesByJob.value) let matchedJobId: JobId | null = null for (let i = jobIds.length - 1; i >= 0; i--) { const jobId = jobIds[i] - const idMatch = - activeId !== null && jobIdToWorkflowId.value.get(jobId) === activeId - const pathMatch = - jobIdToSessionWorkflowPath.value.get(jobId) === activePath - if (idMatch || pathMatch) { + if ( + canResolveWorkflowOwnership(jobId, undefined) && + messageMatchesActiveWorkflow(jobId, undefined) + ) { matchedJobId = jobId break } diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 02602dd5280..61be6095a9d 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -917,7 +917,10 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) - for (const [i, step] of [3, 5, 7, 9].entries()) { + // Unrolled on purpose: each cycle depends on the state the previous one + // left, and the first arrival in B was clean while later ones were not, + // so the cycle number is part of what is being asserted. + const cycle = async (step: number) => { fire('progress_state', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -925,15 +928,27 @@ describe('executionStore workflow gating', () => { }) useWorkflowStore().activeWorkflow = workflowB await nextTick() - expect( - store.nodeProgressStates['1'], - `switch ${i + 1} to B must not inherit A` - ).toBeUndefined() - + const inB = store.nodeProgressStates['1'] useWorkflowStore().activeWorkflow = workflowA await nextTick() - expect(store.nodeProgressStates['1']?.value).toBe(step) + return { inB, inA: store.nodeProgressStates['1']?.value } } + + const first = await cycle(3) + expect(first.inB, 'first arrival in B').toBeUndefined() + expect(first.inA).toBe(3) + + const second = await cycle(5) + expect(second.inB, 'second arrival in B').toBeUndefined() + expect(second.inA).toBe(5) + + const third = await cycle(7) + expect(third.inB, 'third arrival in B').toBeUndefined() + expect(third.inA).toBe(7) + + const fourth = await cycle(9) + expect(fourth.inB, 'fourth arrival in B').toBeUndefined() + expect(fourth.inA).toBe(9) }) it('stays clean while the active workflow is queued behind the running one', async () => { From 0d072257ca5c558ffb10fb5a6955c87d8bfb5891 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 10:23:39 -0700 Subject: [PATCH 08/52] fix(execution): adopt a background job on return, and release it on error Addresses the P1 and the three P2 leaks from review on #20045. handleExecutionStart deliberately does not adopt activeJobId for a job whose tab is in the background, but nothing adopted it on the way back either, so a tab could show node progress while isIdle stayed true: the title, favicon and overlay read idle and handleExecuted dropped the completion frames at the activeJob guard. Reachable by the second prompt of a batch, or anything queued behind another job while the user is on a different tab, and a regression against main where every start set activeJobId. reconcileMirrorForActiveWorkflow now adopts the job it restores the mirror from. The three error paths, execution_error, service-level and cloud validation, returned at the ownership gate without releasing the finished job's own records, while success and interrupt released them. All three now release. Both covered by tests that fail without the fix. --- .../executionStore.terminalRecovery.test.ts | 41 +++++++++++++ src/stores/executionStore.ts | 23 ++++++- .../executionStore.workflowGating.test.ts | 60 +++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index 92b60f919aa..0a73449fcb6 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -295,4 +295,45 @@ describe('executionStore terminal-job recovery', () => { // returns early once it is gone, so order is the whole point here. expect(mockRemoveTextPreview).toHaveBeenCalled() }) + + it('releases a background job records when it ends in an error', () => { + const jobId = 'job-bg-error' + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + expect(store.nodeProgressStatesByJob[jobId]).toBeDefined() + + // A is in front, so this error belongs to a background tab. It is still a + // finished job, so its own records have to go, like the success path. + fire('execution_error', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + node_id: '1', + node_type: 'TestNode', + exception_type: 'RuntimeError', + exception_message: 'boom', + traceback: [], + executed: [], + current_inputs: {}, + current_outputs: {} + }) + + expect(store.nodeProgressStatesByJob[jobId]).toBeUndefined() + expect(store.queuedJobs[jobId]).toBeUndefined() + }) }) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index afcf67979fd..8fb0d0bd036 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -924,8 +924,12 @@ export const useExecutionStore = defineStore('execution', () => { clearInitializationByJobId(detail.prompt_id) // Only the active workflow's error resets the shared execution state; the // dialog is already scoped by the run error key inside showExecutionError. - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + // A background job is still finished, so release its own records either way + // or they survive the session, same as the success and interrupt paths. + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) return + } resetExecutionState(detail.prompt_id) } @@ -955,8 +959,10 @@ export const useExecutionStore = defineStore('execution', () => { return false clearInitializationByJobId(detail.prompt_id) - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) return true + } resetExecutionState(detail.prompt_id) executionErrorStore.recordPromptError( @@ -980,8 +986,10 @@ export const useExecutionStore = defineStore('execution', () => { if (!result) return false clearInitializationByJobId(detail.prompt_id) - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) return true + } resetExecutionState(detail.prompt_id) @@ -1135,6 +1143,15 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStates.value = nodeProgressStatesByJob.value[matchedJobId] ?? {} executionIdToLocatorCache.clear() + // handleExecutionStart deliberately does not adopt a job whose tab is in + // the background, so nothing had adopted it by the time the user came + // back: the tab showed node progress while isIdle stayed true, which left + // the title and overlay reading idle and made handleExecuted drop the + // completion frames. Per-job progress still existing means the job has + // not reached a terminal frame, so it is the one this tab should show. + if (activeJobId.value !== matchedJobId) { + activeJobId.value = matchedJobId + } if (_executingNodeProgress.value?.prompt_id !== matchedJobId) { _executingNodeProgress.value = null } diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 61be6095a9d..0c89f3983ef 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1168,4 +1168,64 @@ describe('executionStore workflow gating', () => { ) }) }) + + /** + * Reported by review on #20045. handleExecutionStart deliberately does not + * adopt activeJobId for a job whose tab is in the background, so that a + * background run cannot steal the visible tab's execution UI. But nothing + * adopted it on the way back either, so a tab could show node progress while + * isIdle stayed true: the title, favicon and overlay read idle, and + * handleExecuted dropped completion frames at `if (!activeJob.value) return`. + * Reachable by the second prompt of a batch, or anything queued behind + * another job while the user is on a different tab. + */ + describe('returning to a tab whose job started in the background', () => { + it('adopts the job so the tab is not both running and idle', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(store.activeJobId, 'B must not adopt A job').toBeNull() + + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe('job-a') + expect(store.isIdle).toBe(false) + }) + + it('does not adopt a job belonging to a different tab', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 3) } + }) + + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + + expect(store.activeJobId).toBeNull() + expect(store.isIdle).toBe(true) + }) + }) }) From ee5590fd415f99f256b641d60431d66836d1ca8d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 10:27:46 -0700 Subject: [PATCH 09/52] fix(execution): clear a stale activeJobId, and align the two ownership lists Addresses two more P2s from review on #20045. When the active job finished while its workflow was in the background, the gated terminal branches returned early and the gated `executing: null` was dropped, so activeJobId kept naming the finished job: the actionbar, favicon and title read non-idle with nothing running until the next queue poll evicted it. releaseFinishedJobRecords now clears it. canResolveWorkflowOwnership listed the workflow-id and path maps but not the instance map, which is the resolver's first and strongest leg, so a frame could be resolvable by one rule and unresolvable by the other. Latent rather than live, since the instance and path are written together today, but the two lists have to agree. --- src/stores/executionStore.ts | 10 +++++ .../executionStore.workflowGating.test.ts | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 8fb0d0bd036..42a8d72f0af 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -789,8 +789,13 @@ export const useExecutionStore = defineStore('execution', () => { jobId: JobId, messageWorkflowId: string | undefined ): boolean { + // Must list every map messageMatchesActiveWorkflow consults, or the two + // disagree about the same frame: the instance is that resolver's first and + // strongest leg, and leaving it out here made a frame resolvable by one + // rule and unresolvable by the other. return ( Boolean(messageWorkflowId) || + jobIdToWorkflowInstanceId.has(jobId) || jobIdToWorkflowId.value.has(jobId) || jobIdToSessionWorkflowPath.value.has(jobId) ) @@ -1252,6 +1257,11 @@ export const useExecutionStore = defineStore('execution', () => { } jobIdToWorkflow.delete(jobId) useJobPreviewStore().clearPreview(jobId) + // This runs for a job whose tab is not in front, so activeJobId naming it + // is stale: the gated terminal branches return early and the gated + // `executing: null` is dropped, which left the actionbar, favicon and title + // non-idle with nothing running until the next queue poll evicted it. + if (activeJobId.value === jobId) activeJobId.value = null } function resetExecutionState(jobIdParam?: JobId | null) { diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 0c89f3983ef..175936ea4b1 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1228,4 +1228,44 @@ describe('executionStore workflow gating', () => { expect(store.isIdle).toBe(true) }) }) + + describe('the two ownership rules agree', () => { + // Documents the invariant rather than catching a live bug: today + // ensureSessionWorkflowPath writes the instance and the path together, so + // the two lists cannot disagree in practice. It will fail if a future + // caller sets one without the other. + it('treats a job known by its tab instance as resolvable', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + useWorkflowStore().activeWorkflow = workflowB + + // The instance is the resolver's first leg, so a frame it can place must + // not be reported as unresolvable by the other rule, or one gate accepts + // the frame while the other rejects it. + expect(store.frameBelongsToVisibleWorkflow('job-a', undefined)).toBe( + false + ) + }) + + it('does not leave activeJobId naming a finished background job', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + expect(store.activeJobId).toBe('job-a') + + useWorkflowStore().activeWorkflow = workflowB + fire('execution_success', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBeNull() + expect(store.isIdle).toBe(true) + }) + }) }) From bb3e809eb246cc78fa5b0c6eda8450689a48b58c Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 16:21:01 -0700 Subject: [PATCH 10/52] test: delete three tests that could not fail, and pin the output gate Review on #20045 showed these by mutation: the assertions survived deleting or inverting the thing they claimed to cover. Removed: the two-tab badge test, where the tab under assertion is the active one and the active tab never renders a status badge, so the checks on B passed for any implementation and the one falsifiable assertion is repeated by the next test; the in-progress-items test, where the rendered list filters on the session path so a job never queued from that page can never appear and both checks pass before the frames arrive; and the rename test, where the ownership assertion returns on the instance leg before reading the path, with the map rewrite already covered in executionStore.test.ts. Added the test the output gate never had. ComfyApp's `executed` listener is what writes node outputs, and deleting its ownership gate previously left every test green, so the cross-tab output leak could come back unnoticed. It spies on the writer rather than reading nodeOutputs, because the fixture graph has no nodes and the locator would not resolve either way. Deleting the gate fails it. --- .../tests/twoTabExecutionLeak.spec.ts | 40 --------------- .../tests/workflowScopedExecution.spec.ts | 22 -------- src/scripts/app.test.ts | 51 +++++++++++++++++++ .../executionStore.workflowGating.test.ts | 18 ------- 4 files changed, 51 insertions(+), 80 deletions(-) diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts index 189c2063c6e..e273833d697 100644 --- a/browser_tests/tests/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/twoTabExecutionLeak.spec.ts @@ -61,46 +61,6 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) }) - test('a run in one tab leaves the other tab idle', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - - const workflowAId = await activeWorkflowId(comfyPage) - expect(workflowAId, 'workflow A must carry an id').toBeDefined() - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: workflowAId }) - simulator.play([ - running.start(), - ...running.nodeRunning(KSAMPLER_NODE, 1, 4) - ]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - const workflowBId = await activeWorkflowId(comfyPage) - expect(workflowBId, 'the second tab must be a different workflow').not.toBe( - workflowAId - ) - - // Tab A's run continues while the user looks at tab B. - simulator.play([ - ...running.nodeRunning(KSAMPLER_NODE, 3, 4), - running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), - running.success() - ]) - - // Per-tab state, read from the topbar badges: A owns the run, B never - // shows one. Asserted rather than screenshotted — this is a state bug. - const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) - const tabB = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_B) - await expect(tabB.getByRole('img', { name: 'Running' })).toHaveCount(0) - await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() - await expect(tabB.getByRole('img', { name: 'Completed' })).toHaveCount(0) - }) - test('switching back shows the finished run, not a stuck one', async ({ comfyPage, getWebSocket diff --git a/browser_tests/tests/workflowScopedExecution.spec.ts b/browser_tests/tests/workflowScopedExecution.spec.ts index 33a75398e60..43866811a94 100644 --- a/browser_tests/tests/workflowScopedExecution.spec.ts +++ b/browser_tests/tests/workflowScopedExecution.spec.ts @@ -90,28 +90,6 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) }) - test('a frame from another workflow does not start the visible tab', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() - - const foreign = simulator.prompt('foreign-job', { - workflowId: FOREIGN_WORKFLOW_ID - }) - simulator.play([ - foreign.start(), - ...foreign.nodeRunning(KSAMPLER_NODE, 1, 4), - foreign.executing(KSAMPLER_NODE) - ]) - - await expect(comfyPage.appMode.outputHistory.inProgressItems).toHaveCount(0) - await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) - }) - test('an output from another workflow does not land on the visible tab', async ({ comfyPage, getWebSocket diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 3f82f7588b2..46ac69aad66 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3060,6 +3060,57 @@ describe('ComfyApp', () => { expect(executionErrorStore.isErrorOverlayOpen).toBe(false) }) + it('does not apply a background run output to the visible workflow', async () => { + // The gate in the `executed` listener. Outputs are keyed by a locator + // resolved against the visible graph, so without it a job finishing in + // another tab writes its result onto the same-numbered node in front. + // Spying on the writer rather than reading nodeOutputs, because the + // fixture graph has no nodes and the locator would never resolve. + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + + const executionStore = useExecutionStore() + executionStore.storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: {}, + workflow: workflowB, + mode: 'graph' + }) + + const setOutputs = vi.spyOn( + useNodeOutputStore(), + 'setNodeOutputsByExecutionId' + ) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const executedHandler = addEventListener.mock.calls.find( + ([event]) => event === 'executed' + )?.[1] as EventListener | undefined + expect(executedHandler).toBeTypeOf('function') + + executedHandler?.( + new CustomEvent('executed', { + detail: { + prompt_id: 'job-b', + workflow_id: workflowBId, + node: '1', + display_node: '1', + output: { images: [{ filename: 'from-b.png', type: 'output' }] } + } + }) + ) + + expect(setOutputs).not.toHaveBeenCalled() + }) + it('restores the failed run state when returning to a workflow tab', async () => { const workflowService = await useRealWorkflowService() const graph = new LGraph() diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 175936ea4b1..153b3f2355b 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1126,24 +1126,6 @@ describe('executionStore workflow gating', () => { ).toBe(true) }) - it('keeps a running job with its tab across a rename', () => { - useWorkflowStore().activeWorkflow = workflowA - queueJobFrom('job-a', workflowA) - - // renameWorkflow mutates the same object and rewrites the mapping by - // instanceId, so the run must survive being given a new name mid-flight. - store.rewriteSessionWorkflowPaths( - workflowA.instanceId, - 'workflows/renamed.json' - ) - expect(store.jobIdToSessionWorkflowPath.get('job-a')).toBe( - 'workflows/renamed.json' - ) - expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( - true - ) - }) - it('does not give a running job to the copy that Save As creates', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) From 5de50c3bd82e19b1178b1ae92d9bd7358894fddd Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 16:58:30 -0700 Subject: [PATCH 11/52] refactor(e2e): drop the unused frame-selector form, and fix two doc claims Review on #20045. FrameSelector's object form, with jobId and occurrence, is used by nothing outside its own self-tests: every spec selects by label. Collapsed to a label, which removes the occurrence validation, the two-branch matcher and eight self-tests that existed only to exercise the form. dropFrames stays, since three self-tests use it to show that a Frame is independently addressable, which is the harness's core contract. Replaced `as unknown as` in the simulator self-tests with fromPartial, which AGENTS.md, typescript.md and browser_tests/README.md all ask for and which PerformanceHelper.test.ts in the same folder already uses. eslint ignores this file, so nothing else was catching it. The queuePrompt doc comment cited a pre-force-push commit that is not on master and called ComfyUI#16763 unmerged; it merged on 2026-10-06. --- .../fixtures/helpers/BackendSimulator.test.ts | 186 +----------------- .../fixtures/helpers/BackendSimulator.ts | 64 ++---- src/scripts/api.ts | 5 +- 3 files changed, 29 insertions(+), 226 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index cd48e47fcdf..8f1141cecd8 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -1,4 +1,5 @@ import type { WebSocketRoute } from '@playwright/test' +import { fromPartial } from '@total-typescript/shoehorn' import { describe, expect, it } from 'vitest' import type { ComfyPage } from '@e2e/fixtures/ComfyPage' @@ -22,12 +23,12 @@ const SEEDS = [0, 1, 3, 7, 42, 99, 12345] function harness() { const sent: SentFrame[] = [] - const ws = { + const ws = fromPartial({ send: (message: string | Buffer) => { if (typeof message === 'string') sent.push(JSON.parse(message)) } - } as unknown as WebSocketRoute - const execution = new ExecutionHelper({} as unknown as ComfyPage, ws) + }) + const execution = new ExecutionHelper(fromPartial({}), ws) return { sent, simulator: new BackendSimulator(execution), execution } } @@ -186,48 +187,6 @@ describe('BackendSimulator frame scripting', () => { expect(sent.map((frame) => frame.data.workflow_id)).toEqual([undefined]) }) - it('drops a frame the backend never sent', () => { - const { sent, simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [ - prompt.start(), - prompt.progress('1', 1, 4), - prompt.success() - ] - - simulator.play(dropFrames(script, 'wf-a:execution_success')) - - expect(sent.map((frame) => frame.type)).toEqual([ - 'execution_start', - 'progress' - ]) - }) - - it('drops one job of two prompts sharing a workflow id', () => { - const { sent, simulator } = harness() - const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) - const script = [ - first.start(), - second.start(), - first.success(), - second.success() - ] - - simulator.play( - dropFrames(script, { - label: 'wf-a:execution_success', - jobId: 'job-a' - }) - ) - - expect(sent.map((frame) => [frame.type, frame.data.prompt_id])).toEqual([ - ['execution_start', 'job-a'], - ['execution_start', 'job-b'], - ['execution_success', 'job-b'] - ]) - }) - it('duplicates a frame', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -246,25 +205,6 @@ describe('BackendSimulator frame scripting', () => { ]) }) - it('duplicates one job of two prompts sharing a workflow id', () => { - const { sent, simulator } = harness() - const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) - - simulator.play( - duplicateFrames([first.success(), second.success()], { - label: 'wf-a:execution_success', - jobId: 'job-b' - }) - ) - - expect(sent.map((frame) => frame.data.prompt_id)).toEqual([ - 'job-a', - 'job-b', - 'job-b' - ]) - }) - it('swaps two frames to produce out-of-order arrival', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -281,96 +221,6 @@ describe('BackendSimulator frame scripting', () => { ]) }) - it('swaps one job of two prompts sharing a workflow id', () => { - const { sent, simulator } = harness() - const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) - const script = [ - first.start(), - second.start(), - first.success(), - second.success() - ] - - // job-a terminates before job-b is even announced. - simulator.play( - swapFrames( - script, - { label: 'wf-a:execution_start', jobId: 'job-b' }, - { label: 'wf-a:execution_success', jobId: 'job-a' } - ) - ) - - expect(sent.map((frame) => [frame.type, frame.data.prompt_id])).toEqual([ - ['execution_start', 'job-a'], - ['execution_success', 'job-a'], - ['execution_start', 'job-b'], - ['execution_success', 'job-b'] - ]) - }) - - it('drops one of a prompt’s two frames of the same event', () => { - const { sent, simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - // One prompt, two `progress` frames — what any script using both - // `progress()` and `nodeRunning()` produces. `label` + `jobId` cannot tell - // them apart, so "the backend dropped the second progress frame" (the shape - // behind progress stuck after a run) needs the occurrence selector. - const script = [ - prompt.start(), - prompt.progress('1', 1, 4), - ...prompt.nodeRunning('1', 2, 4), - prompt.success() - ] - - simulator.play( - dropFrames(script, { label: 'wf-a:progress', occurrence: 2 }) - ) - - expect(sent.map((frame) => [frame.type, frame.data.value])).toEqual([ - ['execution_start', undefined], - ['progress', 1], - ['progress_state', undefined], - ['execution_success', undefined] - ]) - }) - - it('duplicates one of a prompt’s two frames of the same event', () => { - const { sent, simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [prompt.progress('1', 1, 4), prompt.progress('1', 2, 4)] - - simulator.play( - duplicateFrames(script, { label: 'wf-a:progress', occurrence: 1 }) - ) - - expect(sent.map((frame) => frame.data.value)).toEqual([1, 1, 2]) - }) - - it('swaps one of a prompt’s two frames of the same event', () => { - const { sent, simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [ - prompt.progress('1', 1, 4), - prompt.progress('1', 2, 4), - prompt.success() - ] - - simulator.play( - swapFrames( - script, - { label: 'wf-a:progress', occurrence: 2 }, - 'wf-a:execution_success' - ) - ) - - expect(sent.map((frame) => [frame.type, frame.data.value])).toEqual([ - ['progress', 1], - ['execution_success', undefined], - ['progress', 2] - ]) - }) - it('points an ambiguous swap at the selector that can resolve it', () => { const { simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -386,19 +236,6 @@ describe('BackendSimulator frame scripting', () => { ).toThrow(/occurrence/) }) - it('throws rather than silently applying no fault for a missing occurrence', () => { - const { simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [prompt.progress('1', 1, 4)] - - expect(() => - dropFrames(script, { label: 'wf-a:progress', occurrence: 2 }) - ).toThrow(/not in the script/) - expect(() => - dropFrames(script, { label: 'wf-a:progress', occurrence: 0 }) - ).toThrow(/positive integer/) - }) - it('throws rather than silently skipping an unknown swap target', () => { const { simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) @@ -442,12 +279,9 @@ describe('BackendSimulator frame scripting', () => { expect(() => dropFrames(script, 'wf-a:execution_success')).toThrow( /not in the script/ ) - expect(() => - duplicateFrames(script, { - label: 'wf-a:execution_start', - jobId: 'job-zzz' - }) - ).toThrow(/not in the script/) + expect(() => duplicateFrames(script, 'wf-a:progress')).toThrow( + /not in the script/ + ) }) }) @@ -474,12 +308,12 @@ describe('BackendSimulator frame granularity', () => { // selector and would make `interleave` schedule fewer units than it emits. // Counts raw sends, so the binary `latentPreview` is included. let sends = 0 - const ws = { + const ws = fromPartial({ send: () => { sends++ } - } as unknown as WebSocketRoute - const execution = new ExecutionHelper({} as unknown as ComfyPage, ws) + }) + const execution = new ExecutionHelper(fromPartial({}), ws) const simulator = new BackendSimulator(execution) const script = everyFrame(simulator) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts index a49e73b528d..68917603845 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -18,59 +18,23 @@ export interface Frame { } /** - * Which frames a fault applies to. A bare string keeps the workflow-wide - * selection every script already uses; add `jobId` to reach one prompt when - * several share a workflow id, and `occurrence` to reach one frame when a - * single prompt sends the same event more than once. - * - * `occurrence` is 1-based and counts within whatever `label` (+ `jobId`) already - * matched, so `{ label: 'wf-a:progress', occurrence: 2 }` is "the second - * progress frame" — the shape behind *progress stuck after a run*, which needs - * one of several identical events to go missing. Without it `label` + `jobId` - * cannot tell two `progress` frames of one prompt apart, and one prompt sends - * two of them as soon as a script uses both `progress()` and `nodeRunning()`. + * Which frames a fault applies to: the label a {@link Frame} carries, which is + * `:`. A label can match several frames, and every + * match is faulted, so a bare label spanning two prompts injects a wider fault + * than a test naming one frame probably means. */ -export type FrameSelector = - | string - | { label: string; jobId?: string; occurrence?: number } +export type FrameSelector = string function describeSelector(selector: FrameSelector): string { - if (typeof selector === 'string') return selector - const qualifiers = [ - selector.jobId === undefined ? undefined : `job ${selector.jobId}`, - selector.occurrence === undefined - ? undefined - : `occurrence ${selector.occurrence}` - ].filter((part) => part !== undefined) - return qualifiers.length === 0 - ? selector.label - : `${selector.label} (${qualifiers.join(', ')})` -} - -function matchesLabelAndJob(frame: Frame, selector: FrameSelector): boolean { - if (typeof selector === 'string') return frame.label === selector - if (frame.label !== selector.label) return false - return selector.jobId === undefined || frame.jobId === selector.jobId + return selector } function indicesOf(frames: Frame[], selector: FrameSelector): number[] { const found: number[] = [] frames.forEach((frame, index) => { - if (matchesLabelAndJob(frame, selector)) found.push(index) + if (frame.label === selector) found.push(index) }) - if (typeof selector === 'string' || selector.occurrence === undefined) { - return found - } - if (selector.occurrence < 1 || !Number.isInteger(selector.occurrence)) { - throw new Error( - `occurrence must be a positive integer, got ${selector.occurrence}` - ) - } - // `.at` rather than an index, so an occurrence past the end is `undefined` - // and falls through to `requireMatches` instead of being a hole in the array. - // The positive-integer check above is what keeps this from wrapping. - const picked = found.at(selector.occurrence - 1) - return picked === undefined ? [] : [picked] + return found } /** @@ -232,6 +196,13 @@ export class BackendSimulator { * drop one prompt's frame while a concurrent prompt of the same workflow keeps * its own. */ + +/** + * Remove selected frames, as a backend that never sent them would. + * + * Every match is dropped, so a bare label spanning two prompts injects a wider + * fault than a test naming one frame probably means. + */ export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { const drop = new Set(requireMatches(frames, selector, 'dropFrames')) return frames.filter((_, index) => !drop.has(index)) @@ -240,9 +211,8 @@ export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { /** * Send selected frames twice, as a retrying or reconnecting backend does. * - * Plural like {@link dropFrames}: every match is duplicated, so a bare label - * matching two prompts injects a wider fault than a test naming one frame - * probably means. Narrow it with `jobId`/`occurrence` when that matters. + * Every match is duplicated, so a bare label matching two prompts injects a + * wider fault than a test naming one frame probably means. */ export function duplicateFrames( frames: Frame[], diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 29084b5aca8..8ead9411bc1 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -104,9 +104,8 @@ interface QueuePromptRequestBody { * server never interprets the contents. Omitted when the workflow has no id. * * Three limits of the echo, so a reader does not assume more than the server - * gives (verified against ComfyUI#16763 at `52463316ca`, which adds this - * field and is **not yet merged** — a server without it ignores the field - * rather than rejecting the prompt): + * gives. Added by ComfyUI#16763, merged 2026-10-06; an older server ignores + * the field rather than rejecting the prompt: * * - Only JSON frames whose `data` object carries a `prompt_id` are stamped. * `status` and `logs` have none, and binary preview frames are not objects, From 088bc32f7a2da6f05cbd50af9107b173ee7fb9c4 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:04:04 -0700 Subject: [PATCH 12/52] test: make the remaining execution assertions able to fail Review on #20045 showed each of these by mutation. The only production caller of reconcileTerminalJobs had no test: deleting the call, or swapping its two arguments so every running job is treated as terminal, left the suite green. Covered now by asserting the eviction, which catches both. api.lastExecutingMessage had nothing pinning either half of its contract. Turning the `finally` into a no-op left every api and store test passing, and a field left set attributes the next frame to a stale prompt. isJobInitializing was asserted on a job that was never initializing, so it passed whatever the code did. It now marks the job first. Browser: the interleave seed put the foreign start before the visible run, so the old ordering never tore anything down and a count of one was satisfied before the foreign output arrived; seed 2 interleaves them and the test now also asserts which output was counted. The binary-preview test asserted image outputs, where a leaked preview can never appear, so it has moved to latent previews with the workflow's own preview as a barrier. The pasted-copy test polls the original tab reaching Completed before its absence checks, since WebSocketRoute.send does not await page-side handling. Deleted a two-tab test whose script and assertions were a subset of its neighbour. --- .../tests/twoTabExecutionLeak.spec.ts | 36 ++++++----------- .../tests/workflowScopedExecution.spec.ts | 27 +++++++++---- src/scripts/api.socketReset.test.ts | 39 +++++++++++++++++++ .../executionStore.workflowGating.test.ts | 4 ++ src/stores/queueStore.test.ts | 31 +++++++++++++++ 5 files changed, 105 insertions(+), 32 deletions(-) diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts index e273833d697..81be73e2acc 100644 --- a/browser_tests/tests/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/twoTabExecutionLeak.spec.ts @@ -100,29 +100,6 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { ).toHaveCount(0) }) - test('the background tab records its own run while the other is visible', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowAId = await activeWorkflowId(comfyPage) - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: workflowAId }) - simulator.play([running.start()]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - simulator.play([running.success()]) - - // The tab badge is per-workflow state, so it must still reach its terminal - // mark while another tab is in front — the regression CI caught earlier. - const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) - await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() - await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) - }) - test('the node the user is looking at does not take over a foreign run', async ({ comfyPage, getWebSocket @@ -279,7 +256,18 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { running.executed(SAVE_IMAGE_NODE, imageOutput('from-the-original.png')), running.success() ]) - await comfyPage.nextFrame() + + // A positive marker before the absence checks. `WebSocketRoute.send` does + // not await page-side handling and progress is coalesced per frame, so + // reading straight after `play` can assert on state the frames have not + // reached yet, and a gate that wrongly applied them would still pass. The + // original's own tab reaching Completed is driven by the same terminal + // frame, so polling it proves the batch was processed. + await expect( + comfyPage.menu.topbar + .getWorkflowTab(WORKFLOW_A) + .getByRole('img', { name: 'Completed' }) + ).toBeVisible() const leaked = await comfyPage.page.evaluate(() => JSON.stringify(window.app!.nodeOutputs) diff --git a/browser_tests/tests/workflowScopedExecution.spec.ts b/browser_tests/tests/workflowScopedExecution.spec.ts index 43866811a94..0877033992f 100644 --- a/browser_tests/tests/workflowScopedExecution.spec.ts +++ b/browser_tests/tests/workflowScopedExecution.spec.ts @@ -134,7 +134,9 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { workflowId: FOREIGN_WORKFLOW_ID }) - // Seed 11 fixes the order; it is printed here so a failure is reproducible. + // Seed 2 puts the foreign start inside the visible run, which is the + // ordering that leaks without the gate. Seed 11 put it before the visible + // run even started, so the old order could not tear anything down. simulator.play( interleave( [ @@ -149,11 +151,16 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { foreign.executed(SAVE_IMAGE_NODE, imageOutput('theirs.png')), foreign.success() ], - 11 + 2 ) ) await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + // Which one was counted matters: a count of one is also satisfied by the + // foreign output arriving instead of this workflow's own. + await expect( + comfyPage.appMode.outputHistory.imageOutputs.first() + ).not.toHaveAttribute('src', /theirs\.png/) }) test('an output that arrives after the terminal frame is ignored', async ({ @@ -332,13 +339,17 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) // Binary frames carry a JSON header with prompt_id but core does not stamp - // workflow_id on them, so this leans on the queue-time mapping. Pinned - // here because the binary contract is a separate follow-up. + // workflow_id on them, so this leans on the queue-time mapping. simulator.play([foreign.latentPreview(KSAMPLER_NODE)]) - await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(0) - await expect( - comfyPage.appMode.outputHistory.inProgressItems.first() - ).toBeVisible() + // A leaked preview surfaces as a latent preview, not an image output, so + // asserting on imageOutputs held whatever the gate did. Then this + // workflow's own preview is the barrier: it proves previews render at all + // and that the foreign one was processed and discarded rather than still + // in flight. + await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(0) + + simulator.play([mine.latentPreview(KSAMPLER_NODE)]) + await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(1) }) }) diff --git a/src/scripts/api.socketReset.test.ts b/src/scripts/api.socketReset.test.ts index 0302078c7db..1c3302fd1b3 100644 --- a/src/scripts/api.socketReset.test.ts +++ b/src/scripts/api.socketReset.test.ts @@ -153,4 +153,43 @@ describe('ComfyApi realtime socket reset', () => { expect(window.name).toBe('') expect(sessionStorage.getItem('clientId')).toBeNull() }) + + it('exposes the executing ids during dispatch and clears them after', async () => { + // The `executing` event carries only a node id for extension + // compatibility, so the store reads prompt_id and workflow_id off this + // field during the synchronous dispatch. Nothing pinned either half: + // turning the `finally` into a no-op left every api and executionStore + // test green, and a field left set would attribute the next frame to a + // stale prompt. + await api.resetSocket() + const socket = FakeWebSocket.instances[0] + socket.simulateOpen() + + let seenDuringDispatch: unknown = 'not called' + const listener = () => { + seenDuringDispatch = api.lastExecutingMessage + } + api.addEventListener('executing', listener) + + socket.handlers['message']?.({ + data: JSON.stringify({ + type: 'executing', + data: { + node: '3', + display_node: '3', + prompt_id: 'job-a', + workflow_id: 'wf-a' + } + }) + }) + + api.removeEventListener('executing', listener) + + expect(seenDuringDispatch).toMatchObject({ + prompt_id: 'job-a', + workflow_id: 'wf-a', + node: '3' + }) + expect(api.lastExecutingMessage).toBeNull() + }) }) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 153b3f2355b..ec9d6827156 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -376,6 +376,10 @@ describe('executionStore workflow gating', () => { it('execution_error from another workflow leaves active state alone but clears its initializing flag', () => { queueJobFrom('job-b', workflowB) + // Without this the job is never initializing, so the assertion below + // passes whatever the code does. + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) fire('execution_error', { prompt_id: 'job-b', diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 3965866c9f5..2c74178cd6c 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -4,6 +4,7 @@ import type { JobListItem } from '@/platform/remote/comfyui/jobs/jobTypes' import type { TaskOutput } from '@/platform/remote/comfyui/execution/types' import { api } from '@/scripts/api' import { useExecutionStore } from '@/stores/executionStore' +import { toNodeId } from '@/types/nodeId' import { TaskItemImpl, useQueuePendingTaskCountStore, @@ -776,6 +777,36 @@ describe('useQueueStore', () => { }) }) + describe('terminal-job reconciliation', () => { + it('clears progress for a job the backend reports only in history', async () => { + // This is the only production caller of reconcileTerminalJobs, and it was + // unverified: deleting the call, or swapping its two arguments so every + // running job is treated as terminal, left the whole suite green. + // Asserting the eviction rather than spying catches both. + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + 'hist-stuck': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'hist-stuck' + } + } + } + + mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([createHistoryJob(1, 'hist-stuck')]) + + await store.update() + + expect( + executionStore.nodeProgressStatesByJob['hist-stuck'] + ).toBeUndefined() + }) + }) + describe('update() - maxHistoryItems limit', () => { it('should enforce maxHistoryItems limit', async () => { store.maxHistoryItems = 3 From 9f5269bb680fb036cf154db9cbe62b2552788d77 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:06:32 -0700 Subject: [PATCH 13/52] refactor(e2e): make the metadata scope synchronous, and unmask its own test Review on #20045. withMetadata's only caller sends one frame synchronously. The async branch, its promise cast and the nesting guard existed to make an async callback safe, and were reachable only from the tests that described them. An async callback was also the shape that let a closed outer scope come back to life and stamp every later frame, so removing it removes the hazard rather than guarding it. Scope prompts as separate frames through BackendSimulator instead. The scope-restore test could not catch the defect it named: it sent the second frame through a second prompt, which opened its own scope and set the metadata itself, so a scope that never restored still passed. It now sends that frame straight through ExecutionHelper outside any scope, and fails against a never-restore mutant. --- .../fixtures/helpers/BackendSimulator.test.ts | 81 ++----------------- .../fixtures/helpers/ExecutionHelper.ts | 62 ++++---------- 2 files changed, 23 insertions(+), 120 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 8f1141cecd8..894c42d4b7c 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -72,11 +72,14 @@ describe('BackendSimulator frame scripting', () => { }) it('leaves the metadata scope clean after each frame', () => { - const { sent, simulator } = harness() + const { sent, simulator, execution } = harness() const scoped = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const legacy = simulator.prompt('job-b') - simulator.play([scoped.start(), legacy.start()]) + simulator.play([scoped.start()]) + // Straight through `execution`, outside any scope. Going through a second + // prompt masked the defect, because that prompt opened its own scope and + // set the metadata itself, so a scope that never restored still passed. + execution.executionStart('job-b') expect(sent[0].data.workflow_id).toBe('wf-a') expect(sent[1].data).not.toHaveProperty('workflow_id') @@ -115,78 +118,6 @@ describe('BackendSimulator frame scripting', () => { expect(sent[1].data).not.toHaveProperty('workflow_id') }) - it('keeps the workflow id scope open across awaits', async () => { - const { sent, execution } = harness() - - await execution.withWorkflowId('wf-a', async () => { - execution.executionStart('job-a') - await Promise.resolve() - execution.executionSuccess('job-a') - }) - execution.executionStart('job-b') - - expect(sent.map((frame) => frame.data.workflow_id)).toEqual([ - 'wf-a', - 'wf-a', - undefined - ]) - }) - - it('refuses a second scope while an async one is still in flight', async () => { - const { execution } = harness() - let release!: () => void - const blocked = new Promise((resolve) => { - release = resolve - }) - - const scope = execution.withWorkflowId('wf-a', () => blocked) - expect(() => execution.withWorkflowId('wf-b', () => {})).toThrow( - /in flight/ - ) - - release() - await scope - - expect(() => execution.withWorkflowId('wf-b', () => {})).not.toThrow() - }) - - it('refuses a scope nested inside another scope', () => { - const { execution } = harness() - - // Nesting slipped past the in-flight guard entirely, because the flag was - // only raised *after* `fn()` returned. Two live scopes cannot be attributed - // to their prompts, so the second one throws wherever it is opened from. - expect(() => - execution.withWorkflowId('wf-a', () => - execution.withWorkflowId('wf-b', () => {}) - ) - ).toThrow(/already in flight/) - }) - - it('does not resurrect a closed scope when a nested async one settles', async () => { - const { sent, execution } = harness() - let release!: () => void - const blocked = new Promise((resolve) => { - release = resolve - }) - - // The shape the nesting hole produced: the inner scope's `finally` restored - // its own saved value, bringing the already-closed outer scope back to life - // and stamping every later frame `wf-a` for the rest of the test. Before the - // guard this emitted `[undefined, 'wf-a']`. - expect(() => - execution.withWorkflowId('wf-a', () => { - void execution.withWorkflowId('wf-b', () => blocked) - }) - ).toThrow(/already in flight/) - - release() - await blocked - execution.executionStart('job-after') - - expect(sent.map((frame) => frame.data.workflow_id)).toEqual([undefined]) - }) - it('duplicates a frame', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 6cc9497c928..5342ad9a047 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -11,14 +11,6 @@ import { createMockJob } from '@e2e/fixtures/helpers/AssetsHelper' const PROMPT_ROUTE_PATTERN = /\/api\/prompt$/ -function isPromiseLike(value: unknown): value is PromiseLike { - return ( - typeof value === 'object' && - value !== null && - typeof (value as PromiseLike).then === 'function' - ) -} - type RunOptions = { nodeErrors?: Record onPromptRequest?: (requestBody: unknown) => void | Promise @@ -54,7 +46,6 @@ export function buildKSamplerError( export class ExecutionHelper { private jobCounter = 0 private metadata: Record | undefined - private scopeOpen = false private readonly completedJobs: RawJobListItem[] = [] private readonly page: ComfyPage['page'] private readonly command: ComfyPage['command'] @@ -85,10 +76,10 @@ export class ExecutionHelper { * messages. Leave `workflowId` undefined to emit frames the way a core build * without that support does — the legacy backend profile. */ - withWorkflowId(workflowId: string | undefined, fn: () => T): T { - return this.withMetadata( + withWorkflowId(workflowId: string | undefined, send: () => void): void { + this.withMetadata( workflowId === undefined ? undefined : { workflow_id: workflowId }, - fn + send ) } @@ -120,44 +111,25 @@ export class ExecutionHelper { * or script the prompts as separate synchronous frames via * `BackendSimulator`. */ - withMetadata( + /** + * Stamp `metadata` on every eligible JSON frame sent inside `send`. + * + * Synchronous on purpose. The only caller sends one frame, and an async + * callback was the shape that let an outer scope come back to life and stamp + * every later frame for the rest of a test. Script prompts as separate + * frames through `BackendSimulator` instead of awaiting inside a scope. + */ + withMetadata( metadata: Record | undefined, - fn: () => T - ): T { - if (this.scopeOpen) { - throw new Error( - 'ExecutionHelper: a metadata scope is already in flight. ' + - 'Await it before opening another one, and do not nest them.' - ) - } - + send: () => void + ): void { const previous = this.metadata this.metadata = metadata - // Set before `fn` runs, so a scope opened synchronously inside it is - // rejected too. Setting it afterwards only caught the overlapping-async - // case and left nesting — the strictly worse one — unguarded. - this.scopeOpen = true - const restore = () => { - this.scopeOpen = false - this.metadata = previous - } - - let result: T try { - result = fn() - } catch (error) { - restore() - throw error - } - - if (!isPromiseLike(result)) { - restore() - return result + send() + } finally { + this.metadata = previous } - - // Cast: `T` is promise-like here, and the returned promise settles with the - // same value or rejection, only after the scope has been restored. - return Promise.resolve(result).finally(restore) as T } /** From 4af9a665e2124c74d4ebbf6c5b6afc455fa173a1 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:17:49 -0700 Subject: [PATCH 14/52] refactor(execution): one release path for a finished job Three functions released a finished job and each deleted the same four records by hand, which is how the text-preview clear ended up ordered after the queuedJobs delete it depends on in one of them. evictTerminalJob and resetExecutionState now go through releaseFinishedJobRecords. Also fixed while in here: - handleExecutionInterrupted cleared the visible job's initializing flag instead of the interrupted job's. - reconcileTerminalJobs did not track queuedJobs, so a background job that got execution_start but no progress_state before a dropped terminal frame kept a Running badge for the session. - dropped a dead falsy-jobId guard and the reverse index loop. --- .../executionStore.terminalRecovery.test.ts | 32 +++++ src/stores/executionStore.ts | 110 +++++++----------- 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index 0a73449fcb6..b60f03e4547 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -336,4 +336,36 @@ describe('executionStore terminal-job recovery', () => { expect(store.nodeProgressStatesByJob[jobId]).toBeUndefined() expect(store.queuedJobs[jobId]).toBeUndefined() }) + + it('unsticks a tab whose job never reported progress', () => { + // The gap the tracked set had: execution_start sets the tab to Running, so + // a job whose terminal frame was dropped before any progress_state arrived + // was in no tracked collection and its badge stayed Running for the + // session. + // Must be a background job: a visible one is tracked through activeJobId, + // which execution_start only sets for the workflow in front. + const jobId = 'job-no-progress' + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + expect(store.getWorkflowStatus(workflowB)).toBe('running') + expect( + store.activeJobId, + 'B is behind, so it must not be adopted' + ).toBeNull() + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.getWorkflowStatus(workflowB)).toBeUndefined() + }) }) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 42a8d72f0af..d9d51062213 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -11,10 +11,7 @@ import type { WorkflowExecutionFailureReason, WorkflowExecutionIntent } from '@/platform/telemetry/types' -import type { - ComfyWorkflow, - LoadedComfyWorkflow -} from '@/platform/workflow/management/stores/workflowStore' +import type { ComfyWorkflow } from '@/platform/workflow/management/stores/workflowStore' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import type { ComfyApiWorkflow, @@ -537,7 +534,11 @@ export const useExecutionStore = defineStore('execution', () => { }) const workflow = jobIdToWorkflow.get(jobId) if (workflow) clearWorkflowStatus(workflow) - if (activeJobId.value) clearInitializationByJobId(activeJobId.value) + // This job's flag, not the visible job's. Before the gate existed a + // background interrupt also set activeJobId so the two coincided; now + // clearing activeJobId's flag wipes the visible run's "waiting for a + // machine" state and leaves the interrupted job's own flag set. + clearInitializationByJobId(jobId) if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { // The job is finished either way, so its own records have to be released // or they leak for the lifetime of the session. Only the shared UI state @@ -724,8 +725,14 @@ export const useExecutionStore = defineStore('execution', () => { * take unrelated features down with it. */ function activeWorkflowGraphId(): string | null { - const active: Partial | null = - workflowStore.activeWorkflow + // Typed by what is read rather than as a Partial of the whole workflow: + // both states are declared present on LoadedComfyWorkflow and the running + // store does not always honour that, and this is read from a watcher and a + // computed, so a missing state has to return null rather than throw. + const active: { + activeState?: { id?: string } + initialState?: { id?: string } + } | null = workflowStore.activeWorkflow if (!active) return null return active.activeState?.id ?? active.initialState?.id ?? null } @@ -1065,36 +1072,17 @@ export const useExecutionStore = defineStore('execution', () => { * Idempotent. */ function evictTerminalJob(jobId: JobId) { - if (!jobId) return - - if (jobId in nodeProgressStatesByJob.value) { - const map = { ...nodeProgressStatesByJob.value } - delete map[jobId] - nodeProgressStatesByJob.value = map - } - - // Before the queuedJobs entry goes: clearTextPreviewsForJob reads the job's - // node list out of it and returns early if it is already gone, so clearing - // after the delete removed nothing and left the progress_text widget on the - // node, which is the thing this eviction path exists to fix. - clearTextPreviewsForJob(jobId) + // Read before releasing: the release nulls activeJobId when it names this + // job, so asking afterwards always says no. + const isActive = activeJobId.value === jobId // A dropped terminal frame left the workflow marked running forever, since // handleExecutionStart set that and nothing else resets it. const evictedWorkflow = jobIdToWorkflow.get(jobId) if (evictedWorkflow) clearWorkflowStatus(evictedWorkflow) - jobIdToWorkflow.delete(jobId) - - if (jobId in queuedJobs.value) { - const next = { ...queuedJobs.value } - delete next[jobId] - queuedJobs.value = next - } - useJobPreviewStore().clearPreview(jobId) + releaseFinishedJobRecords(jobId) clearInitializationByJobId(jobId) - - const isActive = activeJobId.value === jobId // Only clear the shared mirror when it still belongs to the evicted job, // otherwise evicting an old job would blank a live run's progress. if (isActive || mirrorOwnerJobId() === jobId) { @@ -1131,18 +1119,12 @@ export const useExecutionStore = defineStore('execution', () => { // this feature already shipped once, so defer to the same resolver the // handlers use, and require ownership to be resolvable rather than adopting // a job we cannot place. - const jobIds = Object.keys(nodeProgressStatesByJob.value) - let matchedJobId: JobId | null = null - for (let i = jobIds.length - 1; i >= 0; i--) { - const jobId = jobIds[i] - if ( - canResolveWorkflowOwnership(jobId, undefined) && - messageMatchesActiveWorkflow(jobId, undefined) - ) { - matchedJobId = jobId - break - } - } + const matchedJobId = + Object.keys(nodeProgressStatesByJob.value).findLast( + (jobId) => + canResolveWorkflowOwnership(jobId, undefined) && + messageMatchesActiveWorkflow(jobId, undefined) + ) ?? null if (matchedJobId) { nodeProgressStates.value = @@ -1170,12 +1152,7 @@ export const useExecutionStore = defineStore('execution', () => { _executingNodeProgress.value = null } - watch( - () => workflowStore.activeWorkflow, - () => { - reconcileMirrorForActiveWorkflow() - } - ) + watch(() => workflowStore.activeWorkflow, reconcileMirrorForActiveWorkflow) /** * Reconcile tracked per-job state against the backend's authoritative job @@ -1189,8 +1166,13 @@ export const useExecutionStore = defineStore('execution', () => { activeJobIds: Set, terminalJobIds: Set ) { + // queuedJobs covers the case the other three miss: a background job that + // got execution_start, so its tab reads Running, but no progress_state + // before its terminal frame was dropped. Without it that badge stays + // Running until the tab closes. const tracked = new Set([ ...Object.keys(nodeProgressStatesByJob.value), + ...Object.keys(queuedJobs.value), ...initializingJobIds.value ]) if (activeJobId.value) tracked.add(activeJobId.value) @@ -1245,16 +1227,16 @@ export const useExecutionStore = defineStore('execution', () => { } } - /** - * Reset execution-related state after a run completes or is stopped. - */ function releaseFinishedJobRecords(jobId: JobId) { - if (jobId in queuedJobs.value) delete queuedJobs.value[jobId] - if (jobId in nodeProgressStatesByJob.value) { - const map = { ...nodeProgressStatesByJob.value } - delete map[jobId] - nodeProgressStatesByJob.value = map - } + // Must precede the queuedJobs delete: clearTextPreviewsForJob reads the + // job's node list out of that entry and returns early once it is gone, so + // clearing afterwards silently removes nothing. (It is also a no-op when + // this runs for a background job, since it only touches the front tab.) + clearTextPreviewsForJob(jobId) + delete queuedJobs.value[jobId] + const byJob = { ...nodeProgressStatesByJob.value } + delete byJob[jobId] + nodeProgressStatesByJob.value = byJob jobIdToWorkflow.delete(jobId) useJobPreviewStore().clearPreview(jobId) // This runs for a job whose tab is not in front, so activeJobId naming it @@ -1264,6 +1246,9 @@ export const useExecutionStore = defineStore('execution', () => { if (activeJobId.value === jobId) activeJobId.value = null } + /** + * Reset execution-related state after a run completes or is stopped. + */ function resetExecutionState(jobIdParam?: JobId | null) { cancelPendingProgressUpdates() @@ -1271,15 +1256,7 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStates.value = {} const jobId = jobIdParam ?? activeJobId.value ?? null const runErrorKey = jobId ? runErrorKeyForJob(jobId) : undefined - if (jobId) { - const map = { ...nodeProgressStatesByJob.value } - delete map[jobId] - nodeProgressStatesByJob.value = map - useJobPreviewStore().clearPreview(jobId) - jobIdToWorkflow.delete(jobId) - clearTextPreviewsForJob(jobId) - } - if (jobId) delete queuedJobs.value[jobId] + if (jobId) releaseFinishedJobRecords(jobId) activeJobId.value = null _executingNodeProgress.value = null executionErrorStore.clearPromptError(runErrorKey) @@ -1518,7 +1495,6 @@ export const useExecutionStore = defineStore('execution', () => { clearInitializationByJobIds, reconcileInitializingJobs, reconcileTerminalJobs, - reconcileMirrorForActiveWorkflow, clearActiveJobIfStale, bindExecutionEvents, unbindExecutionEvents, From 9ac24ca49d77bc730fe4fe97582f4bc1650bb995 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:21:03 -0700 Subject: [PATCH 15/52] refactor(execution): one name and one reader for workflow ownership frameBelongsToVisibleWorkflow becomes belongsToActiveWorkflow: it reads workflowStore.activeWorkflow like its two neighbours, so calling the same workflow 'visible' here and 'active' there left readers checking whether the two differed. Its parameter is JobId now, as elsewhere in the store. The graph-id fallback was written out by hand in four places; graphIdOf is the one reader, typed against ComfyWorkflow since both states are already nullable there. --- src/scripts/app.ts | 2 +- src/stores/executionStore.ts | 51 ++++++++++--------- .../executionStore.workflowGating.test.ts | 40 +++++---------- 3 files changed, 42 insertions(+), 51 deletions(-) diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 8e61556a96f..92d8c416c24 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -907,7 +907,7 @@ export class ComfyApp { // a job finishing in another tab would otherwise write its result onto // the same-numbered node of the tab in front. if ( - !useExecutionStore().frameBelongsToVisibleWorkflow( + !useExecutionStore().belongsToActiveWorkflow( detail.prompt_id, detail.workflow_id ) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index d9d51062213..54a968f47f2 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -724,17 +724,23 @@ export const useExecutionStore = defineStore('execution', () => { * computed now — so a missing state has to return null rather than throw and * take unrelated features down with it. */ + /** + * Graph id of a workflow: the id carried by the workflow's own JSON, which + * every copy of that content shares, so it identifies the content and not + * the tab holding it. + * + * Both states are nullable on ComfyWorkflow and this is read from a watcher + * and a computed, so an unloaded workflow returns undefined rather than + * throwing. + */ + function graphIdOf( + workflow: ComfyWorkflow | null | undefined + ): string | undefined { + return workflow?.activeState?.id ?? workflow?.initialState?.id + } + function activeWorkflowGraphId(): string | null { - // Typed by what is read rather than as a Partial of the whole workflow: - // both states are declared present on LoadedComfyWorkflow and the running - // store does not always honour that, and this is read from a watcher and a - // computed, so a missing state has to return null rather than throw. - const active: { - activeState?: { id?: string } - initialState?: { id?: string } - } | null = workflowStore.activeWorkflow - if (!active) return null - return active.activeState?.id ?? active.initialState?.id ?? null + return graphIdOf(workflowStore.activeWorkflow) ?? null } /** @@ -841,7 +847,7 @@ export const useExecutionStore = defineStore('execution', () => { */ function openWorkflowPathForGraph(graphId: WorkflowId): string | undefined { const matches = workflowStore.openWorkflows.filter( - (w) => (w.activeState?.id ?? w.initialState?.id) === graphId + (w) => graphIdOf(w) === graphId ) return matches.length === 1 ? matches[0].path : undefined } @@ -860,10 +866,7 @@ export const useExecutionStore = defineStore('execution', () => { */ function runErrorKeyForJob(jobId: string): string | null { const workflow = jobIdToWorkflow.get(jobId) - const graphId = - workflow?.activeState?.id ?? - workflow?.initialState?.id ?? - jobIdToWorkflowId.value.get(jobId) + const graphId = graphIdOf(workflow) ?? jobIdToWorkflowId.value.get(jobId) if (graphId === undefined) return null const path = @@ -1282,21 +1285,21 @@ export const useExecutionStore = defineStore('execution', () => { * visible graph. Without this gate a finished job writes its output onto the * same-numbered node of whatever tab is in front. */ - function frameBelongsToVisibleWorkflow( - promptId: string | undefined, + function belongsToActiveWorkflow( + jobId: JobId | undefined, workflowId: string | undefined ): boolean { - if (!promptId) return true - if (canResolveWorkflowOwnership(promptId, workflowId)) { - return messageMatchesActiveWorkflow(promptId, workflowId) + if (!jobId) return true + if (canResolveWorkflowOwnership(jobId, workflowId)) { + return messageMatchesActiveWorkflow(jobId, workflowId) } - return !activeJobId.value || promptId === activeJobId.value + return !activeJobId.value || jobId === activeJobId.value } function handleProgressText(e: CustomEvent) { const { nodeId, text, prompt_id, workflow_id } = e.detail if (!text || !nodeId) return - if (!frameBelongsToVisibleWorkflow(prompt_id, workflow_id)) return + if (!belongsToActiveWorkflow(prompt_id, workflow_id)) return const currentId = getNodeIdIfExecuting(nodeId) if (!currentId) return @@ -1348,7 +1351,7 @@ export const useExecutionStore = defineStore('execution', () => { queuedJob.shareId = workflow.shareId queuedJob.viewMode = mode queuedJob.isAppMode = isAppModeValue(mode) - const wid = workflow.activeState?.id ?? workflow.initialState?.id + const wid = graphIdOf(workflow) if (wid) { jobIdToWorkflowId.value.set(id, wid) } @@ -1511,6 +1514,6 @@ export const useExecutionStore = defineStore('execution', () => { getWorkflowStatus, clearWorkflowStatus, rewriteSessionWorkflowPaths, - frameBelongsToVisibleWorkflow + belongsToActiveWorkflow } }) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index ec9d6827156..f9f3bbc60a7 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1033,12 +1033,10 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowACopy - expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( - false + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) + expect(store.belongsToActiveWorkflow('job-copy', WORKFLOW_A_ID)).toBe( + true ) - expect( - store.frameBelongsToVisibleWorkflow('job-copy', WORKFLOW_A_ID) - ).toBe(true) }) it('does not project a foreign job progress onto the copy', () => { @@ -1088,9 +1086,7 @@ describe('executionStore workflow gating', () => { // The copy's graph id is identical, so the id leg would say yes. useWorkflowStore().activeWorkflow = workflowACopy - expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( - false - ) + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) }) it('falls back to the session path when no instance is known', () => { @@ -1099,13 +1095,13 @@ describe('executionStore workflow gating', () => { store.ensureSessionWorkflowPath('job-pathonly', workflowACopy.path) expect( - store.frameBelongsToVisibleWorkflow('job-pathonly', WORKFLOW_A_ID), + store.belongsToActiveWorkflow('job-pathonly', WORKFLOW_A_ID), 'the path says the copy owns it, even though the graph id matches A' ).toBe(false) useWorkflowStore().activeWorkflow = workflowACopy - expect( - store.frameBelongsToVisibleWorkflow('job-pathonly', WORKFLOW_A_ID) - ).toBe(true) + expect(store.belongsToActiveWorkflow('job-pathonly', WORKFLOW_A_ID)).toBe( + true + ) }) it('falls back to the graph id when neither instance nor path is known', () => { @@ -1113,10 +1109,10 @@ describe('executionStore workflow gating', () => { store.registerJobWorkflowIdMapping('job-elsewhere', WORKFLOW_B_ID) expect( - store.frameBelongsToVisibleWorkflow('job-elsewhere', WORKFLOW_B_ID) + store.belongsToActiveWorkflow('job-elsewhere', WORKFLOW_B_ID) ).toBe(false) expect( - store.frameBelongsToVisibleWorkflow('job-elsewhere', WORKFLOW_A_ID) + store.belongsToActiveWorkflow('job-elsewhere', WORKFLOW_A_ID) ).toBe(true) }) @@ -1125,9 +1121,7 @@ describe('executionStore workflow gating', () => { // A job from another browser session: no mapping, no frame id. Keeping // this permissive is what preserves single-tab behaviour. - expect( - store.frameBelongsToVisibleWorkflow('job-unknown', undefined) - ).toBe(true) + expect(store.belongsToActiveWorkflow('job-unknown', undefined)).toBe(true) }) it('does not give a running job to the copy that Save As creates', () => { @@ -1145,13 +1139,9 @@ describe('executionStore workflow gating', () => { }) useWorkflowStore().activeWorkflow = savedAs - expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( - false - ) + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) useWorkflowStore().activeWorkflow = workflowA - expect(store.frameBelongsToVisibleWorkflow('job-a', WORKFLOW_A_ID)).toBe( - true - ) + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) }) }) @@ -1228,9 +1218,7 @@ describe('executionStore workflow gating', () => { // The instance is the resolver's first leg, so a frame it can place must // not be reported as unresolvable by the other rule, or one gate accepts // the frame while the other rejects it. - expect(store.frameBelongsToVisibleWorkflow('job-a', undefined)).toBe( - false - ) + expect(store.belongsToActiveWorkflow('job-a', undefined)).toBe(false) }) it('does not leave activeJobId naming a finished background job', () => { From f8b8325db6f02ef5eeccad54cf8f2982f99c58ed Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:26:52 -0700 Subject: [PATCH 16/52] fix(execution): discard the all-zero workflow id on the way in queuePrompt refuses to send the all-zero sentinel as a routing key, but that never kept it off the wire: the same graph goes out under extra_pnginfo and merged core falls back to the id it finds there. So a graph still on LGraph's default id had the sentinel stamped on its frames, and the gate compared it as a real id, rejecting the tab's own frames. It is now read as absent at ingress, as the rest of the codebase reads it. The workflow_metadata docblock claimed binary previews are never stamped and that omitting the field keeps frames unstamped. ComfyUI#16809 and the extra_pnginfo fallback make both false. Narrowed the type to the one shape the only writer sends, and switched execution/types.ts to the exported WorkflowId alias so the store can use it too. --- .../remote/comfyui/execution/types.ts | 8 +--- .../remote/comfyui/useQueuePolling.test.ts | 18 +++----- src/scripts/api.queuePrompt.test.ts | 43 ++++++++----------- src/scripts/api.ts | 43 +++++++++---------- src/stores/executionStore.ts | 32 ++++++++++---- .../executionStore.workflowGating.test.ts | 15 +++++++ src/stores/queueStore.ts | 25 ++++++----- 7 files changed, 97 insertions(+), 87 deletions(-) diff --git a/src/platform/remote/comfyui/execution/types.ts b/src/platform/remote/comfyui/execution/types.ts index 223d41eb480..efd5c582f68 100644 --- a/src/platform/remote/comfyui/execution/types.ts +++ b/src/platform/remote/comfyui/execution/types.ts @@ -3,15 +3,9 @@ import { z } from 'zod' import { zNodeId } from '@/platform/workflow/validation/schemas/workflowSchema' import type { TaskStatus } from '@/platform/tasks/services/taskService' import { resultItemType } from '@/schemas/resultItemTypeSchema' +import type { WorkflowId } from '@/platform/workflow/validation/schemas/workflowSchema' export type JobId = string -/** - * The id of the workflow a prompt was queued from. Present on execution - * messages only when the server knows it — core omits the field entirely for a - * workflow with no id, and a server without `workflow_metadata` support never - * sends it, so every consumer must treat it as optional. - */ -type WorkflowId = string export const zResultItem = z.object({ filename: z.string().optional(), diff --git a/src/platform/remote/comfyui/useQueuePolling.test.ts b/src/platform/remote/comfyui/useQueuePolling.test.ts index 108a1daaf89..da6f032c1cd 100644 --- a/src/platform/remote/comfyui/useQueuePolling.test.ts +++ b/src/platform/remote/comfyui/useQueuePolling.test.ts @@ -30,21 +30,13 @@ describe('useQueuePolling', () => { expect(store.update).not.toHaveBeenCalled() }) - it('polls when activeJobsCount is exactly 1', async () => { + // The 2 row is what keeps this from being narrowed back to `=== 1`: polling + // with several jobs in flight is how per-job progress recovers when a + // terminal WebSocket frame is dropped. + it.for([1, 2])('polls when activeJobsCount is %i', async (count) => { mountUseQueuePolling() - Object.assign(store, { activeJobsCount: 1 }) - await vi.advanceTimersByTimeAsync(8_000) - - expect(store.update).toHaveBeenCalledOnce() - }) - - // Polling with several jobs in flight is what recovers per-job progress when - // a terminal WebSocket frame is dropped, so it must not be limited to one. - it('polls when activeJobsCount > 1', async () => { - mountUseQueuePolling() - - Object.assign(store, { activeJobsCount: 2 }) + Object.assign(store, { activeJobsCount: count }) await vi.advanceTimersByTimeAsync(8_000) expect(store.update).toHaveBeenCalledOnce() diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts index e6231f56bfe..3a74bfb9c22 100644 --- a/src/scripts/api.queuePrompt.test.ts +++ b/src/scripts/api.queuePrompt.test.ts @@ -1,3 +1,4 @@ +import type { MockInstance } from 'vitest' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { @@ -34,15 +35,13 @@ function workflow(id?: string): ComfyWorkflowJSON { } } -function sentBody( - fetchApiSpy: ReturnType -): Record { - const [, init] = fetchApiSpy.mock.calls[0] as [string, RequestInit] - return JSON.parse(String(init.body)) as Record +function sentBody(fetchApiSpy: MockInstance) { + const [, init] = fetchApiSpy.mock.calls[0] + return JSON.parse(String(init?.body)) as Record } describe('api.queuePrompt workflow_metadata', () => { - let fetchApiSpy: ReturnType + let fetchApiSpy: MockInstance beforeEach(() => { fetchApiSpy = vi.spyOn(api, 'fetchApi').mockResolvedValue(promptResponse()) @@ -59,24 +58,17 @@ describe('api.queuePrompt workflow_metadata', () => { }) }) - it('omits the field entirely when the workflow has no id', async () => { + // The sentinel row earns its place: `LGraph._id` defaults to `zeroUuid` and + // `LGraph.serialize()` returns it unfiltered, so a bare truthiness check + // ships it as a routing key. The rest of the codebase already reads it as + // "no id yet" (`ensureNonZeroUuid`). + it.for([ + ['no id', undefined], + ['the all-zero id sentinel', zeroUuid] + ] as const)('omits the field for %s', async ([, id]) => { await api.queuePrompt(0, { output: EMPTY_PROMPT, - workflow: workflow(undefined) - }) - - expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') - }) - - it('omits the field for the all-zero id sentinel', async () => { - // `LGraph._id` defaults to `zeroUuid` and `LGraph.serialize()` returns it - // unfiltered, so a bare truthiness check shipped the sentinel as a real - // routing key — and every workflow still carrying it would have shared one, - // which is how a WS frame gets attributed to the wrong workflow. The rest of - // the codebase already treats it as "no id yet" (`ensureNonZeroUuid`). - await api.queuePrompt(0, { - output: EMPTY_PROMPT, - workflow: workflow(zeroUuid) + workflow: workflow(id) }) expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') @@ -88,9 +80,8 @@ describe('api.queuePrompt workflow_metadata', () => { workflow: workflow(WORKFLOW_ID) }) - const body = sentBody(fetchApiSpy) as { - extra_data: { extra_pnginfo: { workflow: { id?: string } } } - } - expect(body.extra_data.extra_pnginfo.workflow.id).toBe(WORKFLOW_ID) + expect(sentBody(fetchApiSpy)).toMatchObject({ + extra_data: { extra_pnginfo: { workflow: { id: WORKFLOW_ID } } } + }) }) }) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 8ead9411bc1..d283a834bf0 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -99,26 +99,22 @@ interface QueuePromptRequestBody { prompt: ComfyApiWorkflow partial_execution_targets?: NodeExecutionId[] /** - * Opaque key/values the server echoes back on the WebSocket while this - * prompt runs. `workflow_id` is the only key the frontend relies on; the - * server never interprets the contents. Omitted when the workflow has no id. + * Echoed back by the server on every WebSocket frame this prompt produces + * that carries a `prompt_id` (ComfyUI#16763 for JSON frames, #16809 for the + * metadata header on binary previews). An older server ignores the field + * rather than rejecting the prompt. * - * Three limits of the echo, so a reader does not assume more than the server - * gives. Added by ComfyUI#16763, merged 2026-10-06; an older server ignores - * the field rather than rejecting the prompt: + * Two limits worth knowing, since the contract is the server's and not ours: * - * - Only JSON frames whose `data` object carries a `prompt_id` are stamped. - * `status` and `logs` have none, and binary preview frames are not objects, - * so none of those arrive stamped. A consumer still needs the - * `prompt_id`-to-workflow mapping for those. - * - A frame's own fields win on collision (`{**workflow_metadata, **data}`), - * so these values cannot override a frame's routing fields. - * - The cap is 256 characters of the server's JSON serialisation of this - * object, not 256 bytes of content: the `{"...": "..."}` punctuation counts, - * and non-ASCII characters are escaped to `\\uXXXX` before measuring, so - * they cost six characters each. - */ - workflow_metadata?: Record + * - `status` and `logs` frames have no `prompt_id`, so they are never + * stamped and a consumer still needs the `prompt_id`-to-workflow mapping. + * - A value the server cannot use is dropped rather than rejected: over 256 + * characters of its own JSON serialisation, not 256 bytes of content, with + * non-ASCII escaped to `\\uXXXX` first. The server then falls back to + * `extra_data.extra_pnginfo.workflow.id`, so omitting this field does not + * keep frames unstamped. + */ + workflow_metadata?: { workflow_id: string } extra_data: { extra_pnginfo: { @@ -1386,11 +1382,12 @@ export class ComfyApi extends EventTarget { }), // `LGraph.serialize()` returns `id` unfiltered and `LGraph._id` defaults // to the all-zero sentinel, which the codebase treats as "no id yet" and - // replaces on load (`adoptRootGraphId`) and on clear. A bare truthiness - // check ships that sentinel as a real routing key, so every workflow - // still carrying it would share one and WS frames could be attributed to - // the wrong workflow. Shape is already constrained upstream — the loaded - // schema is `z.string().uuid()` and the only other producer is + // replaces on load (`adoptRootGraphId`) and on clear, so it is not a + // routing key and is not sent as one. This does not keep it off the wire: + // the same graph goes out under `extra_pnginfo` and the server falls back + // to the id there, which is why `executionStore` also discards the + // sentinel on the way in. Shape is already constrained upstream — the + // loaded schema is `z.string().uuid()` and the only other producer is // `createUuidv4()` — so the sentinel is the one value to exclude here. ...(workflow.id && workflow.id !== zeroUuid && { diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 54a968f47f2..63fb0587349 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -46,6 +46,7 @@ import { isAppModeValue } from '@/utils/appMode' import { classifyCloudValidationError } from '@/utils/executionErrorUtil' import { executionIdToNodeLocatorId } from '@/utils/graphTraversalUtil' import { createRafCoalescer } from '@/utils/rafBatch' +import { zeroUuid } from '@/utils/uuid' type RuntimeExecutionError = Omit< ExecutionErrorWsMessage, @@ -765,10 +766,23 @@ export const useExecutionStore = defineStore('execution', () => { * to be consulted first. QA hit this with three unsaved copies of the default * graph, where every tab rendered every other tab's outputs. */ + /** + * The all-zero sentinel is `LGraph._id`'s default, which the codebase reads + * as "no id yet" and replaces on configure and on clear. Treating it as a + * real id would make every workflow still carrying it match every other one + * on the graph-id leg, which is the leak this gate exists to stop. The queue + * side does not send it, but core falls back to the id inside + * `extra_pnginfo.workflow`, so it still arrives on the wire. + */ + function workflowIdOrUndefined(id: string | undefined): string | undefined { + return !id || id === zeroUuid ? undefined : id + } + function messageMatchesActiveWorkflow( jobId: JobId, - messageWorkflowId: string | undefined + rawMessageWorkflowId: string | undefined ): boolean { + const messageWorkflowId = workflowIdOrUndefined(rawMessageWorkflowId) const activeWorkflow = workflowStore.activeWorkflow if (!activeWorkflow) return true @@ -782,9 +796,11 @@ export const useExecutionStore = defineStore('execution', () => { return mappedPath === activeWorkflow.path } - const activeId = activeWorkflowGraphId() + const activeId = workflowIdOrUndefined(activeWorkflowGraphId() ?? undefined) if (activeId) { - const ownerId = messageWorkflowId || jobIdToWorkflowId.value.get(jobId) + const ownerId = + messageWorkflowId ?? + workflowIdOrUndefined(jobIdToWorkflowId.value.get(jobId)) if (ownerId) return ownerId === activeId } @@ -802,12 +818,12 @@ export const useExecutionStore = defineStore('execution', () => { jobId: JobId, messageWorkflowId: string | undefined ): boolean { - // Must list every map messageMatchesActiveWorkflow consults, or the two - // disagree about the same frame: the instance is that resolver's first and - // strongest leg, and leaving it out here made a frame resolvable by one - // rule and unresolvable by the other. + // Must list every map messageMatchesActiveWorkflow consults, and reject the + // same ids it rejects, or the two disagree about the same frame: the + // instance is that resolver's first and strongest leg, and leaving it out + // here made a frame resolvable by one rule and unresolvable by the other. return ( - Boolean(messageWorkflowId) || + workflowIdOrUndefined(messageWorkflowId) !== undefined || jobIdToWorkflowInstanceId.has(jobId) || jobIdToWorkflowId.value.has(jobId) || jobIdToSessionWorkflowPath.value.has(jobId) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index f9f3bbc60a7..9d2917934f7 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -8,6 +8,7 @@ import { useWorkflowStore } from '@/platform/workflow/management/stores/workflow import { api } from '@/scripts/api' import { useExecutionStore } from '@/stores/executionStore' import { useNodeOutputStore } from '@/stores/nodeOutputStore' +import { zeroUuid } from '@/utils/uuid' /** * Workflow-ownership gating for execution WebSocket messages. @@ -1143,6 +1144,20 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowA expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) }) + + // Raised in review on #20045. The queue side refuses to send the sentinel + // as workflow_metadata, but merged core falls back to the id inside + // extra_pnginfo.workflow, so a graph still on LGraph's default id has it + // stamped onto its frames anyway. Compared as a real id it matches nothing, + // so the tab's own frames were rejected. + it('does not reject a frame whose only id is the all-zero sentinel', () => { + useWorkflowStore().activeWorkflow = workflowA + + expect( + store.belongsToActiveWorkflow('job-fresh-graph', zeroUuid), + 'the sentinel is not an id, so it cannot be grounds for rejection' + ).toBe(true) + }) }) /** diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index b2f8c8d7e90..e64fc3ceb5b 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -8,6 +8,7 @@ import type { TaskType } from '@/platform/remote/comfyui/jobs/jobTypes' import type { + JobId, StatusWsMessageStatus, TaskOutput } from '@/platform/remote/comfyui/execution/types' @@ -330,9 +331,15 @@ export const useQueueStore = defineStore('queue', () => { api.getHistory(maxHistoryItems.value) ]) - // Shared by the queue and history branches below. - - let activeJobIds: Set | null = null + // Shared by the queue and history branches below. `null` means the queue + // fetch failed, which is not the same as an empty queue. + const activeJobIds: Set | null = + queueResult.status === 'fulfilled' + ? new Set([ + ...queueResult.value.Running.map((j) => j.id), + ...queueResult.value.Pending.map((j) => j.id) + ]) + : null if (queueResult.status === 'fulfilled') { const queue = queueResult.value @@ -349,16 +356,14 @@ export const useQueueStore = defineStore('queue', () => { executionStore.registerJobWorkflowIdMapping(jobIdString, workflowId) } }) - - activeJobIds = new Set([ - ...queue.Running.map((j) => j.id), - ...queue.Pending.map((j) => j.id) - ]) - executionStore.reconcileInitializingJobs(activeJobIds) } else { console.error('Failed to fetch queue:', queueResult.reason) } + if (activeJobIds) { + useExecutionStore().reconcileInitializingJobs(activeJobIds) + } + if (historyResult.status === 'fulfilled') { const history = historyResult.value const currentHistory = toValue(historyTasks) @@ -366,7 +371,7 @@ export const useQueueStore = defineStore('queue', () => { // A job the backend has moved to history while we still hold progress // state for it lost its terminal WebSocket frame. Evicting it here is // the only path that unsticks node progress in that case. - if (history.length > 0 && activeJobIds) { + if (activeJobIds) { const terminalJobIds = new Set(history.map((j) => j.id)) useExecutionStore().reconcileTerminalJobs( activeJobIds, From c7f48c5dfee55483f0afd57159801c5513f0f468 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:47:16 -0700 Subject: [PATCH 17/52] test(execution): exercise the real isOpen, and trim what two suites repeat The two gating suites overrode openWorkflows and reimplemented isOpen as a path match. That is owned logic, not a boundary, so the suites would have kept passing if isOpen started keying by instance, which is the identity mix-up they exist to catch. They now register tabs through attachWorkflow and let the real isOpen and openWorkflows run; breaking isOpen turns four of them red, where before it changed nothing. Also from review: - assert the public executingNodeProgress rather than the underlying ref, and pin the exact null instead of .not.toBe(9) - flush a frame unconditionally in fire(), instead of keeping a copy of which store handlers are RAF-coalesced - the precedence row now sets all three legs in conflict, so it pins the order rather than restating the output-frame test - 'is idempotent' asserted only not.toThrow; it now asserts what a repeat pass must not touch, which is another workflow's status - dropped the repeated-switch cycles past the second, a redundant frame count in the simulator suite, two already-asserted mocks, and a duplicate browser_tests asset that was byte-identical to its original --- .../workflow_with_id_pasted_copy.json | 227 ------------------ .../fixtures/helpers/BackendSimulator.test.ts | 13 - .../tests/twoTabExecutionLeak.spec.ts | 12 +- .../executionStore.terminalRecovery.test.ts | 64 +++-- .../executionStore.workflowGating.test.ts | 203 +++++++--------- 5 files changed, 139 insertions(+), 380 deletions(-) delete mode 100644 browser_tests/assets/execution/workflow_with_id_pasted_copy.json diff --git a/browser_tests/assets/execution/workflow_with_id_pasted_copy.json b/browser_tests/assets/execution/workflow_with_id_pasted_copy.json deleted file mode 100644 index 669efb97138..00000000000 --- a/browser_tests/assets/execution/workflow_with_id_pasted_copy.json +++ /dev/null @@ -1,227 +0,0 @@ -{ - "last_node_id": 9, - "last_link_id": 9, - "nodes": [ - { - "id": 7, - "type": "CLIPTextEncode", - "pos": [413, 389], - "size": [425.27801513671875, 180.6060791015625], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 5 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [6], - "slot_index": 0 - } - ], - "properties": {}, - "widgets_values": ["text, watermark"] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "pos": [415, 186], - "size": [422.84503173828125, 164.31304931640625], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [4], - "slot_index": 0 - } - ], - "properties": {}, - "widgets_values": [ - "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," - ] - }, - { - "id": 5, - "type": "EmptyLatentImage", - "pos": [473, 609], - "size": [315, 106], - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [2], - "slot_index": 0 - } - ], - "properties": {}, - "widgets_values": [512, 512, 1] - }, - { - "id": 3, - "type": "KSampler", - "pos": [863, 186], - "size": [315, 262], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 6 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 2 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [7], - "slot_index": 0 - } - ], - "properties": {}, - "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [1209, 188], - "size": [210, 46], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [9], - "slot_index": 0 - } - ], - "properties": {} - }, - { - "id": 9, - "type": "SaveImage", - "pos": [1451, 189], - "size": [210, 26], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 9 - } - ], - "properties": {} - }, - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [26, 474], - "size": [315, 98], - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [1], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [3, 5], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [8], - "slot_index": 2 - } - ], - "properties": { - "models": [ - { - "name": "v1-5-pruned-emaonly-fp16.safetensors", - "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", - "directory": "checkpoints" - } - ] - }, - "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] - } - ], - "links": [ - [1, 4, 0, 3, 0, "MODEL"], - [2, 5, 0, 3, 3, "LATENT"], - [3, 4, 1, 6, 0, "CLIP"], - [4, 6, 0, 3, 1, "CONDITIONING"], - [5, 4, 1, 7, 0, "CLIP"], - [6, 7, 0, 3, 2, "CONDITIONING"], - [7, 3, 0, 8, 0, "LATENT"], - [8, 4, 2, 8, 1, "VAE"], - [9, 8, 0, 9, 0, "IMAGE"] - ], - "groups": [], - "config": {}, - "extra": { - "ds": { - "offset": [0, 0], - "scale": 1 - } - }, - "version": 0.4, - "id": "0199e3a3-6c01-7000-8000-aaaaaaaaaaaa" -} diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 894c42d4b7c..3325580b9d1 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -398,17 +398,4 @@ describe('BackendSimulator interleaving', () => { expect(() => interleave(script, [], 7)).toThrow(/both sides/) expect(() => interleave([], script, 7)).toThrow(/both sides/) }) - - it('keeps every frame exactly once when interleaving', () => { - const { sent, simulator } = harness() - const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) - - simulator.play( - interleave([a.start(), a.success()], [b.start(), b.success()], 42) - ) - - expect(positionsOf(sent, 'wf-a')).toHaveLength(2) - expect(positionsOf(sent, 'wf-b')).toHaveLength(2) - }) }) diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts index 81be73e2acc..59ec123bdfe 100644 --- a/browser_tests/tests/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/twoTabExecutionLeak.spec.ts @@ -56,7 +56,10 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { test.beforeEach(async ({ comfyPage }) => { await comfyPage.workflow.setupWorkflowsDirectory({ [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', - [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json' + [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json', + // The same asset under a second name: two files carrying one workflow + // id, which is what a copy-paste into a new tab produces. + [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id.json' }) await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) }) @@ -226,15 +229,10 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { // both. `ensureWorkflowId` keeps the id already in the pasted json, so the // two tabs share one workflow id while being different open workflows, and // the node ids are identical too. Outputs landed on the wrong tab. - test('a pasted copy sharing the workflow id does not receive the original output', async ({ + test('a second open workflow sharing the workflow id does not receive the output', async ({ comfyPage, getWebSocket }) => { - await comfyPage.workflow.setupWorkflowsDirectory({ - [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', - [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id_pasted_copy.json' - }) - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) const exec = new ExecutionHelper(comfyPage, await getWebSocket()) const simulator = new BackendSimulator(exec) const sharedId = await activeWorkflowId(comfyPage) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index b60f03e4547..ec9714efd8e 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -17,12 +17,9 @@ import { useExecutionStore } from '@/stores/executionStore' * reports the job in history while the frontend still holds progress for it. */ -const { mockShowTextPreview, mockRemoveTextPreview } = await vi.hoisted( - async () => ({ - mockShowTextPreview: vi.fn(), - mockRemoveTextPreview: vi.fn() - }) -) +const { mockRemoveTextPreview } = await vi.hoisted(async () => ({ + mockRemoveTextPreview: vi.fn() +})) vi.mock(import('@/composables/useAppMode')) vi.mock(import('@/platform/telemetry')) @@ -30,7 +27,7 @@ vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ useNodeProgressText: () => ({ - showTextPreview: mockShowTextPreview, + showTextPreview: vi.fn(), removeTextPreview: mockRemoveTextPreview }) })) @@ -97,9 +94,10 @@ function fire(event: string, detail: Record) { const handler = apiEventHandlers.get(event) if (!handler) throw new Error(`${event} handler not bound`) handler(new CustomEvent(event, { detail })) - if (event === 'progress_state' || event === 'progress') { - vi.advanceTimersToNextFrame() - } + // Some frames are RAF-coalesced in the store; flush unconditionally so the + // assertion sees the applied state, and so this does not have to track which + // handlers are coalesced. + vi.advanceTimersToNextFrame() } function runningNode(jobId: string, nodeId: string): NodeProgressState { @@ -114,16 +112,30 @@ function runningNode(jobId: string, nodeId: string): NodeProgressState { } } +/** + * Open these tabs through the real store, so `isOpen` and `openWorkflows` are + * the production implementations rather than a copy of them. A copy would keep + * passing if `isOpen` started keying by instance instead of path, which is + * exactly the identity mix-up this suite exists to catch. + * + * Additive, and a no-op for a tab already open. Pinia is rebuilt per test, so + * nothing accumulates across tests. + */ +function openTabs(...workflows: LoadedComfyWorkflow[]) { + const workflowStore = useWorkflowStore() + for (const workflow of workflows) { + if (workflowStore.isOpen(workflow)) continue + workflowStore.attachWorkflow(workflow, workflowStore.openWorkflows.length) + } +} + describe('executionStore terminal-job recovery', () => { let store: ReturnType beforeEach(() => { apiEventHandlers.clear() useWorkflowStore().activeWorkflow = workflowA - Object.assign(useWorkflowStore(), { openWorkflows: [workflowA, workflowB] }) - vi.mocked(useWorkflowStore().isOpen).mockImplementation((wf) => - useWorkflowStore().openWorkflows.some((open) => open.path === wf.path) - ) + openTabs(workflowA, workflowB) store = useExecutionStore() store.bindExecutionEvents() }) @@ -218,15 +230,30 @@ describe('executionStore terminal-job recovery', () => { expect(store.isJobInitializing('job-init')).toBe(false) }) - it('is idempotent', () => { + it('leaves a second workflow alone on a repeat pass', () => { const jobId = stuckActiveJob() + store.registerJobWorkflowIdMapping('job-b', WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) store.reconcileTerminalJobs(new Set(), new Set([jobId])) - expect(() => - store.reconcileTerminalJobs(new Set(), new Set([jobId])) - ).not.toThrow() + store.reconcileTerminalJobs(new Set(), new Set([jobId])) expect(store.nodeProgressStates['1']).toBeUndefined() + expect( + store.getWorkflowStatus(workflowB), + 'B never went terminal, so neither pass may release it' + ).toBe('running') }) it('still recovers when the terminal frame was dropped for a legacy backend', () => { @@ -280,7 +307,6 @@ describe('executionStore terminal-job recovery', () => { }) it('removes the text preview before the job record it reads is deleted', async () => { - mockRemoveTextPreview.mockClear() vi.mocked(useWorkflowStore().executionIdToCurrentId).mockReturnValue('1') const { useCanvasStore } = await import('@/renderer/core/canvas/canvasStore') diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 9d2917934f7..030a4029359 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1,5 +1,5 @@ import { fromPartial } from '@total-typescript/shoehorn' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { assert, beforeEach, describe, expect, it, vi } from 'vitest' import { nextTick } from 'vue' import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' @@ -94,8 +94,6 @@ const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') */ const workflowACopy = workflow(WORKFLOW_A_ID, 'workflows/a copy.json') -const RAF_COALESCED = new Set(['progress_state', 'progress']) - /** * `executing` is dispatched with a bare node id for extension compatibility, * so the store reads the ids off the raw message api recorded. Mirror that. @@ -114,9 +112,10 @@ function fire(event: string, detail: Record) { const handler = apiEventHandlers.get(event) if (!handler) throw new Error(`${event} handler not bound`) handler(new CustomEvent(event, { detail })) - // progress frames are RAF-coalesced in the store; flush so the assertion - // sees the applied state rather than the pending batch. - if (RAF_COALESCED.has(event)) vi.advanceTimersToNextFrame() + // Some frames are RAF-coalesced in the store; flush unconditionally so the + // assertion sees the applied state rather than the pending batch, and so + // this does not have to track which handlers are coalesced. + vi.advanceTimersToNextFrame() } function nodeState( @@ -136,22 +135,33 @@ function nodeState( } } +/** + * Open these tabs through the real store, so `isOpen` and `openWorkflows` are + * the production implementations rather than a copy of them. A copy would keep + * passing if `isOpen` started keying by instance instead of path, which is + * exactly the identity mix-up this suite exists to catch. + * + * Additive, and a no-op for a tab already open. Pinia is rebuilt per test, so + * nothing accumulates across tests. + */ +function openTabs(...workflows: LoadedComfyWorkflow[]) { + const workflowStore = useWorkflowStore() + for (const workflow of workflows) { + if (workflowStore.isOpen(workflow)) continue + workflowStore.attachWorkflow(workflow, workflowStore.openWorkflows.length) + } +} + describe('executionStore workflow gating', () => { let store: ReturnType - let revokePreviews: ReturnType - beforeEach(() => { apiEventHandlers.clear() api.lastExecutingMessage = null - mockShowTextPreview.mockClear() - revokePreviews = vi - .spyOn(useNodeOutputStore(), 'revokePreviewsByExecutionId') - .mockImplementation(() => {}) + vi.mocked( + useNodeOutputStore().revokePreviewsByExecutionId + ).mockImplementation(() => {}) useWorkflowStore().activeWorkflow = null - Object.assign(useWorkflowStore(), { openWorkflows: [workflowA, workflowB] }) - vi.mocked(useWorkflowStore().isOpen).mockImplementation((wf) => - useWorkflowStore().openWorkflows.some((open) => open.path === wf.path) - ) + openTabs(workflowA, workflowB) store = useExecutionStore() store.bindExecutionEvents() }) @@ -249,50 +259,25 @@ describe('executionStore workflow gating', () => { }) }) - it('marks nodes for a frame from the active workflow', () => { - fire('executed', { - prompt_id: 'job-a', - workflow_id: WORKFLOW_A_ID, - node: '1', - display_node: '1', - output: {} - }) - - expect(store.activeJob?.nodes['1']).toBe(true) - }) - - it('executed from another workflow does not mark active job nodes', () => { - fire('executed', { - prompt_id: 'job-b', - workflow_id: WORKFLOW_B_ID, - node: '1', - display_node: '1', - output: {} - }) + // Two handlers crossed with own-versus-foreign. `executed` and + // `execution_cached` are separate code paths, so all four rows stay. + it.for([ + { event: 'executed', own: true }, + { event: 'executed', own: false }, + { event: 'execution_cached', own: true }, + { event: 'execution_cached', own: false } + ])('$event marks the active job nodes: $own', ({ event, own }) => { + const ids = own + ? { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID } + : { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID } + const detail = + event === 'executed' + ? { ...ids, node: '1', display_node: '1', output: {} } + : { ...ids, nodes: ['1'], timestamp: 2 } - expect(store.activeJob?.nodes['1']).toBeUndefined() - }) + fire(event, detail) - it('execution_cached from another workflow does not mark active job nodes', () => { - fire('execution_cached', { - prompt_id: 'job-b', - workflow_id: WORKFLOW_B_ID, - nodes: ['1'], - timestamp: 2 - }) - - expect(store.activeJob?.nodes['1']).toBeUndefined() - }) - - it('execution_cached from the active workflow does mark them', () => { - fire('execution_cached', { - prompt_id: 'job-a', - workflow_id: WORKFLOW_A_ID, - nodes: ['1'], - timestamp: 2 - }) - - expect(store.activeJob?.nodes['1']).toBe(true) + expect(store.activeJob?.nodes['1']).toBe(own ? true : undefined) }) }) @@ -445,7 +430,7 @@ describe('executionStore workflow gating', () => { expect(store.nodeProgressStates['1']?.value).toBe(7) }) - it('updates _executingNodeProgress on a workflow_id match', () => { + it('updates the executing node progress on a workflow_id match', () => { fire('execution_start', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -462,10 +447,10 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-a', '1', 'running', 4) } }) - expect(store._executingNodeProgress?.value).toBe(4) + expect(store.executingNodeProgress).toBe(0.4) }) - it('skips _executingNodeProgress on a workflow_id mismatch', () => { + it('skips the executing node progress on a workflow_id mismatch', () => { fire('execution_start', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -482,7 +467,7 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-b', '1', 'running', 9) } }) - expect(store._executingNodeProgress?.value).not.toBe(9) + expect(store.executingNodeProgress).toBeNull() }) }) @@ -498,7 +483,9 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-a', '1', 'running') } }) - expect(revokePreviews).toHaveBeenCalled() + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalled() }) it('does not revoke previews for a frame from another workflow', () => { @@ -508,7 +495,9 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-b', '1', 'running') } }) - expect(revokePreviews).not.toHaveBeenCalled() + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).not.toHaveBeenCalled() }) it('revokes when a node transitions pending -> running', () => { @@ -517,7 +506,9 @@ describe('executionStore workflow gating', () => { workflow_id: WORKFLOW_A_ID, nodes: { '1': nodeState('job-a', '1', 'pending') } }) - expect(revokePreviews).not.toHaveBeenCalled() + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).not.toHaveBeenCalled() fire('progress_state', { prompt_id: 'job-a', @@ -525,7 +516,9 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-a', '1', 'running') } }) - expect(revokePreviews).toHaveBeenCalled() + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalled() }) it('does not revoke twice while a node stays running', () => { @@ -541,7 +534,9 @@ describe('executionStore workflow gating', () => { nodes: running }) - expect(revokePreviews).toHaveBeenCalledTimes(1) + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalledTimes(1) }) }) @@ -701,35 +696,18 @@ describe('executionStore workflow gating', () => { workflow_id: WORKFLOW_A_ID, nodes: { '1': nodeState('job-a', '1', 'running', 5) } }) - expect(store._executingNodeProgress?.value).toBe(5) + expect(store.executingNodeProgress).toBe(0.5) fireExecuting('9', { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID }) - expect(store._executingNodeProgress?.value).toBe(5) - }) - - // api clears the raw message straight after dispatch, so an `executing` - // dispatched later by an extension is not read as belonging to whichever - // run arrived last. Without that, a foreign frame would silently gate out - // every extension-driven executing event that followed it. - it('is not gated by a stale raw message from an earlier frame', () => { - fireExecuting(null, { - prompt_id: 'job-b', - workflow_id: WORKFLOW_B_ID - }) - expect(store.activeJobId).toBe('job-a') - - api.lastExecutingMessage = null - const handler = apiEventHandlers.get('executing')! - handler(new CustomEvent('executing', { detail: null })) - - expect(store.activeJobId).toBeNull() + expect(store.executingNodeProgress).toBe(0.5) }) it('still works when the raw message was never recorded', () => { // Defensive: an `executing` dispatched by an extension or a test rather // than by the socket has no raw message, and must behave as before. - const handler = apiEventHandlers.get('executing')! + const handler = apiEventHandlers.get('executing') + assert.exists(handler) handler(new CustomEvent('executing', { detail: null })) expect(store.activeJobId).toBeNull() @@ -755,7 +733,7 @@ describe('executionStore workflow gating', () => { max: 10 }) - expect(store._executingNodeProgress?.value).toBe(4) + expect(store.executingNodeProgress).toBe(0.4) }) it('drops a progress frame from another workflow', () => { @@ -767,7 +745,7 @@ describe('executionStore workflow gating', () => { max: 10 }) - expect(store._executingNodeProgress).toBeNull() + expect(store.executingNodeProgress).toBeNull() }) }) @@ -832,7 +810,7 @@ describe('executionStore workflow gating', () => { await nextTick() expect(store.nodeProgressStates['1']).toBeUndefined() - expect(store._executingNodeProgress).toBeNull() + expect(store.executingNodeProgress).toBeNull() }) it('replays the active workflow own progress when switching back', async () => { @@ -922,9 +900,9 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) - // Unrolled on purpose: each cycle depends on the state the previous one - // left, and the first arrival in B was clean while later ones were not, - // so the cycle number is part of what is being asserted. + // Two cycles, not a loop: each depends on the state the previous one + // left, and what QA saw was the first arrival in B being clean and a + // later one not, so first-versus-later is the distinction being made. const cycle = async (step: number) => { fire('progress_state', { prompt_id: 'job-a', @@ -946,14 +924,6 @@ describe('executionStore workflow gating', () => { const second = await cycle(5) expect(second.inB, 'second arrival in B').toBeUndefined() expect(second.inA).toBe(5) - - const third = await cycle(7) - expect(third.inB, 'third arrival in B').toBeUndefined() - expect(third.inA).toBe(7) - - const fourth = await cycle(9) - expect(fourth.inB, 'fourth arrival in B').toBeUndefined() - expect(fourth.inA).toBe(9) }) it('stays clean while the active workflow is queued behind the running one', async () => { @@ -1019,9 +989,7 @@ describe('executionStore workflow gating', () => { describe('two open workflows that share an id', () => { beforeEach(() => { - Object.assign(useWorkflowStore(), { - openWorkflows: [workflowA, workflowACopy] - }) + openTabs(workflowA, workflowACopy) }) // Node outputs are written by ComfyApp's own `executed` listener, not by @@ -1076,18 +1044,27 @@ describe('executionStore workflow gating', () => { */ describe('ownership resolution precedence', () => { beforeEach(() => { - Object.assign(useWorkflowStore(), { - openWorkflows: [workflowA, workflowACopy] - }) + openTabs(workflowA, workflowACopy) }) - it('prefers the tab instance over a matching graph id', () => { + it('prefers the tab instance over both weaker legs', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) + // Point the path leg at the copy as well, so both weaker legs name the + // copy and only the instance names A. Reordering the legs flips this. + store.ensureSessionWorkflowPath( + 'job-a', + workflowACopy.path, + workflowA.instanceId + ) - // The copy's graph id is identical, so the id leg would say yes. useWorkflowStore().activeWorkflow = workflowACopy - expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) + expect( + store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID), + 'the instance still says A, so the copy does not own it' + ).toBe(false) + useWorkflowStore().activeWorkflow = workflowA + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) }) it('falls back to the session path when no instance is known', () => { @@ -1135,9 +1112,7 @@ describe('executionStore workflow gating', () => { 'a-brand-new-graph-id', 'workflows/saved as.json' ) - Object.assign(useWorkflowStore(), { - openWorkflows: [workflowA, savedAs] - }) + openTabs(workflowA, savedAs) useWorkflowStore().activeWorkflow = savedAs expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) From b1dc69d5e9c012c94bab3299a6389ec3b3e8a9ff Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 8 Oct 2026 17:51:17 -0700 Subject: [PATCH 18/52] refactor(queue): split the queue and history snapshot appliers out of update update was 109 lines doing two unrelated things behind one try/finally, and the const refactor in the previous commit tipped it over the repo's CRAP gate. The two branches are now named functions; update reads as fetch both, apply each, reconcile. --- src/stores/queueStore.ts | 130 ++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index e64fc3ceb5b..156cd0df3d3 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -316,6 +316,71 @@ export const useQueueStore = defineStore('queue', () => { () => pendingTasks.value.length + runningTasks.value.length ) + function applyQueueSnapshot(queue: Awaited>) { + // API returns pre-sorted data (sort_by=create_time&order=desc) + runningTasks.value = queue.Running.map((job) => new TaskItemImpl(job)) + pendingTasks.value = queue.Pending.map((job) => new TaskItemImpl(job)) + + const appearedTasks = [...pendingTasks.value, ...runningTasks.value] + const executionStore = useExecutionStore() + appearedTasks.forEach((task) => { + const jobIdString = task.jobId + const workflowId = task.workflowId + if (workflowId && jobIdString) { + executionStore.registerJobWorkflowIdMapping(jobIdString, workflowId) + } + }) + } + + function applyHistorySnapshot( + history: Awaited>, + activeJobIds: Set | null + ) { + const currentHistory = toValue(historyTasks) + + // A job the backend has moved to history while we still hold progress + // state for it lost its terminal WebSocket frame. Evicting it here is + // the only path that unsticks node progress in that case. + if (activeJobIds) { + const terminalJobIds = new Set(history.map((j) => j.id)) + useExecutionStore().reconcileTerminalJobs(activeJobIds, terminalJobIds) + } + + // Sort by create_time descending and limit to maxItems + const sortedHistory = [...history] + .sort((a, b) => b.create_time - a.create_time) + .slice(0, toValue(maxHistoryItems)) + + // Reuse existing TaskItemImpl instances or create new + // Must recreate if outputs_count changed (e.g., API started returning it) + const existingByJobId = new Map( + currentHistory.map((impl) => [impl.jobId, impl]) + ) + + const nextHistoryTasks = sortedHistory.map((job) => { + const existing = existingByJobId.get(job.id) + if (!existing) return new TaskItemImpl(job) + // Recreate if outputs_count changed to ensure lazy loading works + if ( + existing.outputsCount !== (job.outputs_count ?? undefined) || + existing.previewableOutputsCount !== + (job.previewable_outputs_count ?? undefined) + ) { + return new TaskItemImpl(job) + } + return existing + }) + + const isHistoryUnchanged = + nextHistoryTasks.length === currentHistory.length && + nextHistoryTasks.every((task, index) => task === currentHistory[index]) + + if (!isHistoryUnchanged) { + historyTasks.value = nextHistoryTasks + } + hasFetchedHistorySnapshot.value = true + } + const update = async () => { if (updateState.inFlight) { updateState.dirty = true @@ -342,20 +407,7 @@ export const useQueueStore = defineStore('queue', () => { : null if (queueResult.status === 'fulfilled') { - const queue = queueResult.value - // API returns pre-sorted data (sort_by=create_time&order=desc) - runningTasks.value = queue.Running.map((job) => new TaskItemImpl(job)) - pendingTasks.value = queue.Pending.map((job) => new TaskItemImpl(job)) - - const appearedTasks = [...pendingTasks.value, ...runningTasks.value] - const executionStore = useExecutionStore() - appearedTasks.forEach((task) => { - const jobIdString = task.jobId - const workflowId = task.workflowId - if (workflowId && jobIdString) { - executionStore.registerJobWorkflowIdMapping(jobIdString, workflowId) - } - }) + applyQueueSnapshot(queueResult.value) } else { console.error('Failed to fetch queue:', queueResult.reason) } @@ -365,55 +417,7 @@ export const useQueueStore = defineStore('queue', () => { } if (historyResult.status === 'fulfilled') { - const history = historyResult.value - const currentHistory = toValue(historyTasks) - - // A job the backend has moved to history while we still hold progress - // state for it lost its terminal WebSocket frame. Evicting it here is - // the only path that unsticks node progress in that case. - if (activeJobIds) { - const terminalJobIds = new Set(history.map((j) => j.id)) - useExecutionStore().reconcileTerminalJobs( - activeJobIds, - terminalJobIds - ) - } - - // Sort by create_time descending and limit to maxItems - const sortedHistory = [...history] - .sort((a, b) => b.create_time - a.create_time) - .slice(0, toValue(maxHistoryItems)) - - // Reuse existing TaskItemImpl instances or create new - // Must recreate if outputs_count changed (e.g., API started returning it) - const existingByJobId = new Map( - currentHistory.map((impl) => [impl.jobId, impl]) - ) - - const nextHistoryTasks = sortedHistory.map((job) => { - const existing = existingByJobId.get(job.id) - if (!existing) return new TaskItemImpl(job) - // Recreate if outputs_count changed to ensure lazy loading works - if ( - existing.outputsCount !== (job.outputs_count ?? undefined) || - existing.previewableOutputsCount !== - (job.previewable_outputs_count ?? undefined) - ) { - return new TaskItemImpl(job) - } - return existing - }) - - const isHistoryUnchanged = - nextHistoryTasks.length === currentHistory.length && - nextHistoryTasks.every( - (task, index) => task === currentHistory[index] - ) - - if (!isHistoryUnchanged) { - historyTasks.value = nextHistoryTasks - } - hasFetchedHistorySnapshot.value = true + applyHistorySnapshot(historyResult.value, activeJobIds) } else { console.error('Failed to fetch history:', historyResult.reason) } From ad0d4635321950571e49e7305974050a27d77d64 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 9 Oct 2026 09:24:54 +0000 Subject: [PATCH 19/52] fix(execution): preserve workflow ownership across recovery Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#issuecomment-6076813229 --- .../fixtures/helpers/BackendSimulator.test.ts | 23 ++++- .../fixtures/helpers/ExecutionHelper.ts | 25 +++-- .../tests/workflowScopedExecution.spec.ts | 15 ++- .../remote/comfyui/execution/types.ts | 1 - .../remote/comfyui/useQueuePolling.test.ts | 21 ++++ .../remote/comfyui/useQueuePolling.ts | 18 +++- src/scripts/api.socketReset.test.ts | 40 ++++++++ src/scripts/api.ts | 7 +- src/scripts/app.test.ts | 96 +++++++++++++++++++ src/scripts/app.ts | 11 ++- src/stores/executionStore.ts | 62 +++++++----- .../executionStore.workflowGating.test.ts | 61 ++++++++++++ src/stores/queueStore.test.ts | 30 ++++++ src/stores/queueStore.ts | 44 ++++++--- 14 files changed, 397 insertions(+), 57 deletions(-) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 3325580b9d1..90bda5d8f77 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -23,13 +23,15 @@ const SEEDS = [0, 1, 3, 7, 42, 99, 12345] function harness() { const sent: SentFrame[] = [] + const binary: Buffer[] = [] const ws = fromPartial({ send: (message: string | Buffer) => { if (typeof message === 'string') sent.push(JSON.parse(message)) + else binary.push(message) } }) const execution = new ExecutionHelper(fromPartial({}), ws) - return { sent, simulator: new BackendSimulator(execution), execution } + return { sent, binary, simulator: new BackendSimulator(execution), execution } } /** The positions a workflow's frames occupy in the order they were sent. */ @@ -118,6 +120,25 @@ describe('BackendSimulator frame scripting', () => { expect(sent[1].data).not.toHaveProperty('workflow_id') }) + it('stamps validation errors and binary previews with workflow ownership', () => { + const { sent, binary, execution } = harness() + + execution.withWorkflowId('wf-a', () => { + execution.validationError('job-a', '1', {}) + execution.latentPreview('job-a', '1') + }) + + expect(sent[0].data.workflow_id).toBe('wf-a') + const metadataLength = binary[0].readUInt32BE(4) + const metadata = JSON.parse( + binary[0].subarray(8, 8 + metadataLength).toString() + ) + expect(metadata).toMatchObject({ + prompt_id: 'job-a', + workflow_id: 'wf-a' + }) + }) + it('duplicates a frame', () => { const { sent, simulator } = harness() const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 5342ad9a047..76d6329841a 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -219,12 +219,14 @@ export class ExecutionHelper { * Encodes the metadata and a tiny 1x1 PNG so the app creates a blob URL. */ latentPreview(jobId: string, nodeId: string): void { + const workflowId = this.metadata?.workflow_id const metadata = JSON.stringify({ node_id: nodeId, display_node_id: nodeId, parent_node_id: nodeId, real_node_id: nodeId, prompt_id: jobId, + ...(typeof workflowId === 'string' && { workflow_id: workflowId }), image_type: 'image/png' }) const metadataBytes = new TextEncoder().encode(metadata) @@ -305,20 +307,15 @@ export class ExecutionHelper { nodeId: string, nodeErrors: Record ): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_error', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - exception_message: JSON.stringify({ node_errors: nodeErrors }), - exception_type: 'prompt_outputs_failed_validation', - traceback: [] - } - }) - ) + this.emit('execution_error', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + exception_message: JSON.stringify({ node_errors: nodeErrors }), + exception_type: 'prompt_outputs_failed_validation', + traceback: [] + }) } /** Send `execution_interrupted` WS event (user-initiated stop). */ diff --git a/browser_tests/tests/workflowScopedExecution.spec.ts b/browser_tests/tests/workflowScopedExecution.spec.ts index 0877033992f..2425ad37fa5 100644 --- a/browser_tests/tests/workflowScopedExecution.spec.ts +++ b/browser_tests/tests/workflowScopedExecution.spec.ts @@ -184,6 +184,7 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { simulator.play( swapFrames(script, `${label}:executed`, `${label}:execution_success`) ) + await comfyPage.nextFrame() // A single WebSocket delivers in send order, so this ordering only happens // on a reconnect replay. The terminal frame has already torn down the run, @@ -338,9 +339,8 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) - // Binary frames carry a JSON header with prompt_id but core does not stamp - // workflow_id on them, so this leans on the queue-time mapping. simulator.play([foreign.latentPreview(KSAMPLER_NODE)]) + await comfyPage.nextFrame() // A leaked preview surfaces as a latent preview, not an image output, so // asserting on imageOutputs held whatever the gate did. Then this @@ -348,8 +348,19 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { // and that the foreign one was processed and discarded rather than still // in flight. await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(0) + expect( + await comfyPage.page.evaluate( + () => Object.values(window.app!.nodePreviewImages).flat().length + ) + ).toBe(0) simulator.play([mine.latentPreview(KSAMPLER_NODE)]) + await comfyPage.nextFrame() await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(1) + expect( + await comfyPage.page.evaluate( + () => Object.values(window.app!.nodePreviewImages).flat().length + ) + ).toBeGreaterThan(0) }) }) diff --git a/src/platform/remote/comfyui/execution/types.ts b/src/platform/remote/comfyui/execution/types.ts index efd5c582f68..9e2e6b0ea4e 100644 --- a/src/platform/remote/comfyui/execution/types.ts +++ b/src/platform/remote/comfyui/execution/types.ts @@ -103,7 +103,6 @@ export interface ProgressTextWsMessage { nodeId: NodeId text: string prompt_id?: string - workflow_id?: WorkflowId } export interface NotificationWsMessage { value: string diff --git a/src/platform/remote/comfyui/useQueuePolling.test.ts b/src/platform/remote/comfyui/useQueuePolling.test.ts index da6f032c1cd..b7f22586b85 100644 --- a/src/platform/remote/comfyui/useQueuePolling.test.ts +++ b/src/platform/remote/comfyui/useQueuePolling.test.ts @@ -3,7 +3,9 @@ import { nextTick } from 'vue' import { beforeEach, describe, expect, it, vi } from 'vitest' import { useQueuePolling } from '@/platform/remote/comfyui/useQueuePolling' +import { useExecutionStore } from '@/stores/executionStore' import { useQueueStore } from '@/stores/queueStore' +import { toNodeId } from '@/types/nodeId' function mountUseQueuePolling() { render({ @@ -52,6 +54,25 @@ describe('useQueuePolling', () => { expect(store.update).not.toHaveBeenCalled() }) + it('keeps polling while execution state awaits terminal reconciliation', async () => { + useExecutionStore().nodeProgressStatesByJob = { + 'stuck-job': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'stuck-job' + } + } + } + mountUseQueuePolling() + + await vi.advanceTimersByTimeAsync(8_000) + + expect(store.update).toHaveBeenCalledOnce() + }) + it('resets timer when loading completes', async () => { Object.assign(store, { activeJobsCount: 1 }) mountUseQueuePolling() diff --git a/src/platform/remote/comfyui/useQueuePolling.ts b/src/platform/remote/comfyui/useQueuePolling.ts index 8625ba622c8..3359c8ab20e 100644 --- a/src/platform/remote/comfyui/useQueuePolling.ts +++ b/src/platform/remote/comfyui/useQueuePolling.ts @@ -2,6 +2,7 @@ import { useTimeoutFn } from '@vueuse/core' import { ref, watch } from 'vue' import { useQueueStore } from '@/stores/queueStore' +import { useExecutionStore } from '@/stores/executionStore' const BASE_INTERVAL_MS = 8_000 const MAX_INTERVAL_MS = 32_000 @@ -9,11 +10,17 @@ const BACKOFF_MULTIPLIER = 1.5 export function useQueuePolling() { const queueStore = useQueueStore() + const executionStore = useExecutionStore() const delay = ref(BASE_INTERVAL_MS) const { start, stop } = useTimeoutFn( () => { - if (queueStore.activeJobsCount < 1 || queueStore.isLoading) return + if ( + (queueStore.activeJobsCount < 1 && + executionStore.trackedJobIds.length < 1) || + queueStore.isLoading + ) + return delay.value = Math.min(delay.value * BACKOFF_MULTIPLIER, MAX_INTERVAL_MS) void queueStore.update() }, @@ -22,12 +29,17 @@ export function useQueuePolling() { ) function scheduleNextPoll() { - if (queueStore.activeJobsCount >= 1 && !queueStore.isLoading) start() + if ( + (queueStore.activeJobsCount >= 1 || + executionStore.trackedJobIds.length >= 1) && + !queueStore.isLoading + ) + start() else stop() } watch( - () => queueStore.activeJobsCount, + () => [queueStore.activeJobsCount, executionStore.trackedJobIds.length], () => { delay.value = BASE_INTERVAL_MS scheduleNextPoll() diff --git a/src/scripts/api.socketReset.test.ts b/src/scripts/api.socketReset.test.ts index 1c3302fd1b3..3897167e3b8 100644 --- a/src/scripts/api.socketReset.test.ts +++ b/src/scripts/api.socketReset.test.ts @@ -51,6 +51,16 @@ class FakeWebSocket { const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) +function previewWithMetadata(metadata: Record): ArrayBuffer { + const metadataBytes = new TextEncoder().encode(JSON.stringify(metadata)) + const buffer = new ArrayBuffer(8 + metadataBytes.length + 1) + const view = new DataView(buffer) + view.setUint32(0, 4) + view.setUint32(4, metadataBytes.length) + new Uint8Array(buffer, 8, metadataBytes.length).set(metadataBytes) + return buffer +} + describe('ComfyApi realtime socket reset', () => { beforeEach(() => { FakeWebSocket.instances = [] @@ -192,4 +202,34 @@ describe('ComfyApi realtime socket reset', () => { }) expect(api.lastExecutingMessage).toBeNull() }) + + it('preserves workflow ownership from binary preview metadata', async () => { + await api.resetSocket() + const socket = FakeWebSocket.instances[0] + socket.simulateOpen() + + let received: unknown + const listener = (event: Event) => { + received = (event as CustomEvent).detail + } + api.addEventListener('b_preview_with_metadata', listener) + + socket.handlers['message']?.({ + data: previewWithMetadata({ + node_id: '3', + display_node_id: '3', + parent_node_id: '3', + real_node_id: '3', + prompt_id: 'job-b', + workflow_id: 'workflow-b', + image_type: 'image/png' + }) + }) + + api.removeEventListener('b_preview_with_metadata', listener) + expect(received).toMatchObject({ + jobId: 'job-b', + workflowId: 'workflow-b' + }) + }) }) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index d283a834bf0..867c6572288 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -73,7 +73,8 @@ import type { } from '@/platform/workflow/templates/types/template' import type { ComfyApiWorkflow, - ComfyWorkflowJSON + ComfyWorkflowJSON, + WorkflowId } from '@/platform/workflow/validation/schemas/workflowSchema' import type { SerializedNodeId } from '@/types/nodeId' import type { @@ -286,6 +287,7 @@ interface BackendApiCalls { displayNodeId: string realNodeId: string jobId: string + workflowId?: WorkflowId } progress_text: ProgressTextWsMessage progress_state: ProgressStateWsMessage @@ -1154,7 +1156,8 @@ export class ComfyApi extends EventTarget { displayNodeId: metadata.display_node_id, parentNodeId: metadata.parent_node_id, realNodeId: metadata.real_node_id, - jobId: metadata.prompt_id + jobId: metadata.prompt_id, + workflowId: metadata.workflow_id }) // Also dispatch legacy b_preview for backward compatibility diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 46ac69aad66..695ab6ac1f4 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -94,6 +94,7 @@ import { setTelemetryRegistry } from '@/platform/telemetry' import { TelemetryRegistry } from '@/platform/telemetry/TelemetryRegistry' import * as executionContextUtils from '@/platform/telemetry/utils/getExecutionContext' import { isCloud } from '@/platform/distribution/types' +import * as accountPreconditionDialogModule from '@/platform/cloud/subscription/composables/useAccountPreconditionDialog' import { PromptExecutionError, api } from '@/scripts/api' import { useExecutionErrorStore } from '@/stores/executionErrorStore' @@ -3060,6 +3061,54 @@ describe('ComfyApp', () => { expect(executionErrorStore.isErrorOverlayOpen).toBe(false) }) + it('does not open an account dialog for a background run error', () => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + useExecutionStore().storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: {}, + workflow: workflowB, + mode: 'graph' + }) + + const open = vi.fn() + vi.spyOn( + accountPreconditionDialogModule, + 'useAccountPreconditionDialog' + ).mockReturnValue({ open }) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const executionErrorHandler = addEventListener.mock.calls.find( + ([event]) => event === 'execution_error' + )?.[1] as EventListener | undefined + expect(executionErrorHandler).toBeTypeOf('function') + + executionErrorHandler?.( + new CustomEvent('execution_error', { + detail: { + prompt_id: 'job-b', + workflow_id: workflowBId, + node_id: '1', + node_type: 'PartnerApiNode', + exception_message: + 'Payment Required: Please add credits to your account to use this node.', + exception_type: 'InsufficientFundsError', + traceback: [] + } + }) + ) + + expect(open).not.toHaveBeenCalled() + }) + it('does not apply a background run output to the visible workflow', async () => { // The gate in the `executed` listener. Outputs are keyed by a locator // resolved against the visible graph, so without it a job finishing in @@ -3111,6 +3160,53 @@ describe('ComfyApp', () => { expect(setOutputs).not.toHaveBeenCalled() }) + it('does not apply a background preview to the visible workflow', () => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + + useExecutionStore().storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: {}, + workflow: workflowB, + mode: 'graph' + }) + + const setPreviews = vi.spyOn( + useNodeOutputStore(), + 'setNodePreviewsByExecutionId' + ) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const previewHandler = addEventListener.mock.calls.find( + ([event]) => event === 'b_preview_with_metadata' + )?.[1] as EventListener | undefined + expect(previewHandler).toBeTypeOf('function') + + previewHandler?.( + new CustomEvent('b_preview_with_metadata', { + detail: { + blob: new Blob(), + nodeId: '1', + displayNodeId: '1', + parentNodeId: '1', + realNodeId: '1', + jobId: 'job-b', + workflowId: workflowBId + } + }) + ) + + expect(setPreviews).not.toHaveBeenCalled() + }) + it('restores the failed run state when returning to a workflow tab', async () => { const workflowService = await useRealWorkflowService() const graph = new LGraph() diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 92d8c416c24..a830b5e4561 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -929,6 +929,13 @@ export class ComfyApp { }) api.addEventListener('execution_error', ({ detail }) => { + if ( + !useExecutionStore().belongsToActiveWorkflow( + detail.prompt_id, + detail.workflow_id + ) + ) + return const precondition = resolveAccountPrecondition({ exceptionType: detail.exception_type, exceptionMessage: detail.exception_message @@ -943,7 +950,9 @@ export class ComfyApp { api.addEventListener('b_preview_with_metadata', ({ detail }) => { // Enhanced preview with explicit node context - const { blob, displayNodeId, jobId } = detail + const { blob, displayNodeId, jobId, workflowId } = detail + if (!useExecutionStore().belongsToActiveWorkflow(jobId, workflowId)) + return const { setNodePreviewsByExecutionId, revokePreviewsByExecutionId } = useNodeOutputStore() const displayNodeExecutionId = tryNormalizeNodeExecutionId(displayNodeId) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 63fb0587349..e827ccb4947 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -45,7 +45,7 @@ import type { AppMode } from '@/utils/appMode' import { isAppModeValue } from '@/utils/appMode' import { classifyCloudValidationError } from '@/utils/executionErrorUtil' import { executionIdToNodeLocatorId } from '@/utils/graphTraversalUtil' -import { createRafCoalescer } from '@/utils/rafBatch' +import { createRafBatch, createRafCoalescer } from '@/utils/rafBatch' import { zeroUuid } from '@/utils/uuid' type RuntimeExecutionError = Omit< @@ -606,7 +606,7 @@ export const useExecutionStore = defineStore('execution', () => { } progressCoalescer.cancel() - if (e.detail == null) progressStateCoalescer.cancel() + if (e.detail == null) cancelPendingProgressStateUpdates() // Clear the current node progress when a new node starts executing _executingNodeProgress.value = null @@ -646,13 +646,21 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStatesByJob.value = pruned } - const progressStateCoalescer = createRafCoalescer( - applyProgressState, - 'raf:progress_state' - ) + const pendingProgressStates = new Map() + const progressStateBatch = createRafBatch(() => { + const pending = [...pendingProgressStates.values()] + pendingProgressStates.clear() + for (const detail of pending) applyProgressState(detail) + }) function handleProgressState(e: CustomEvent) { - progressStateCoalescer.push(e.detail) + pendingProgressStates.set(e.detail.prompt_id, e.detail) + progressStateBatch.schedule() + } + + function cancelPendingProgressStateUpdates() { + pendingProgressStates.clear() + progressStateBatch.cancel() } /** @@ -842,7 +850,7 @@ export const useExecutionStore = defineStore('execution', () => { function cancelPendingProgressUpdates() { progressCoalescer.cancel() - progressStateCoalescer.cancel() + cancelPendingProgressStateUpdates() } function handleStatus() { @@ -901,6 +909,12 @@ export const useExecutionStore = defineStore('execution', () => { const runErrorKey = runErrorKeyForJob(e.detail.prompt_id) if (runErrorKey === null) { bufferPendingExecutionError({ detail: e.detail, endTime }) + if ( + !messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id) + ) { + releaseFinishedJobRecords(e.detail.prompt_id) + return + } resetExecutionState(e.detail.prompt_id) return } @@ -977,6 +991,10 @@ export const useExecutionStore = defineStore('execution', () => { const workflow = jobIdToWorkflow.get(detail.prompt_id) if (workflow) clearWorkflowStatus(workflow) clearInitializationByJobId(detail.prompt_id) + if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) + return true + } resetExecutionState(detail.prompt_id) return true } @@ -1185,18 +1203,7 @@ export const useExecutionStore = defineStore('execution', () => { activeJobIds: Set, terminalJobIds: Set ) { - // queuedJobs covers the case the other three miss: a background job that - // got execution_start, so its tab reads Running, but no progress_state - // before its terminal frame was dropped. Without it that badge stays - // Running until the tab closes. - const tracked = new Set([ - ...Object.keys(nodeProgressStatesByJob.value), - ...Object.keys(queuedJobs.value), - ...initializingJobIds.value - ]) - if (activeJobId.value) tracked.add(activeJobId.value) - - for (const jobId of tracked) { + for (const jobId of trackedJobIds.value) { if (activeJobIds.has(jobId)) continue if (!terminalJobIds.has(jobId)) continue evictTerminalJob(jobId) @@ -1313,9 +1320,9 @@ export const useExecutionStore = defineStore('execution', () => { } function handleProgressText(e: CustomEvent) { - const { nodeId, text, prompt_id, workflow_id } = e.detail + const { nodeId, text, prompt_id } = e.detail if (!text || !nodeId) return - if (!belongsToActiveWorkflow(prompt_id, workflow_id)) return + if (!belongsToActiveWorkflow(prompt_id, undefined)) return const currentId = getNodeIdIfExecuting(nodeId) if (!currentId) return @@ -1478,6 +1485,16 @@ export const useExecutionStore = defineStore('execution', () => { return result }) + const trackedJobIds = computed(() => { + const tracked = new Set([ + ...Object.keys(nodeProgressStatesByJob.value), + ...Object.keys(queuedJobs.value), + ...initializingJobIds.value + ]) + if (activeJobId.value) tracked.add(activeJobId.value) + return [...tracked] + }) + const runningWorkflowCount = computed( () => runningJobIds.value.length ) @@ -1506,6 +1523,7 @@ export const useExecutionStore = defineStore('execution', () => { nodeLocationProgressStates, nodeProgressStatesByJob, runningJobIds, + trackedJobIds, runningWorkflowCount, initializingJobIds, isActiveWorkflowRunning, diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 030a4029359..c8e68b8655e 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -381,6 +381,39 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') expect(store.isJobInitializing('job-b')).toBe(false) }) + + it('does not reset the active workflow when a stamped background error arrives before storeJob', () => { + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('does not reset the active workflow for a background account precondition', () => { + queueJobFrom('job-b', workflowB) + + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'PartnerApiNode', + exception_message: + 'Payment Required: Please add credits to your account to use this node.', + exception_type: 'InsufficientFundsError', + traceback: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) }) describe('progress_state', () => { @@ -418,6 +451,34 @@ describe('executionStore workflow gating', () => { expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(7) }) + it('retains the latest progress snapshot for each job in one frame', () => { + const handler = apiEventHandlers.get('progress_state') + assert(handler) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + } + }) + ) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + } + }) + ) + + vi.advanceTimersToNextFrame() + + expect(store.nodeProgressStatesByJob['job-a']?.['1']?.value).toBe(3) + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(7) + }) + it('updates the mirror when there is no active workflow', () => { useWorkflowStore().activeWorkflow = null diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 2c74178cd6c..0cb895b5e65 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -71,6 +71,7 @@ vi.mock(import('@/scripts/api'), () => ({ api: { getQueue: vi.fn(), getHistory: vi.fn(), + getJobDetail: vi.fn(), clearItems: vi.fn(), deleteItem: vi.fn(), apiURL: vi.fn((path) => `/api${path}`), @@ -366,6 +367,7 @@ describe('useQueueStore', () => { beforeEach(() => { store = useQueueStore() + vi.mocked(api.getJobDetail).mockResolvedValue(undefined) }) const mockGetQueue = vi.mocked(api.getQueue) @@ -805,6 +807,34 @@ describe('useQueueStore', () => { executionStore.nodeProgressStatesByJob['hist-stuck'] ).toBeUndefined() }) + + it('looks up a tracked job that fell outside the history page', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + 'older-stuck': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'older-stuck' + } + } + } + + mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([createHistoryJob(2, 'newer-job')]) + vi.mocked(api.getJobDetail).mockResolvedValue( + createHistoryJob(1, 'older-stuck') + ) + + await store.update() + + expect(api.getJobDetail).toHaveBeenCalledWith('older-stuck') + expect( + executionStore.nodeProgressStatesByJob['older-stuck'] + ).toBeUndefined() + }) }) describe('update() - maxHistoryItems limit', () => { diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 156cd0df3d3..29ac5e3b253 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -5,6 +5,7 @@ import { extractWorkflow } from '@/platform/remote/comfyui/jobs/fetchJobs' import type { APITaskType, JobListItem, + JobStatus, TaskType } from '@/platform/remote/comfyui/jobs/jobTypes' import type { @@ -333,19 +334,10 @@ export const useQueueStore = defineStore('queue', () => { } function applyHistorySnapshot( - history: Awaited>, - activeJobIds: Set | null + history: Awaited> ) { const currentHistory = toValue(historyTasks) - // A job the backend has moved to history while we still hold progress - // state for it lost its terminal WebSocket frame. Evicting it here is - // the only path that unsticks node progress in that case. - if (activeJobIds) { - const terminalJobIds = new Set(history.map((j) => j.id)) - useExecutionStore().reconcileTerminalJobs(activeJobIds, terminalJobIds) - } - // Sort by create_time descending and limit to maxItems const sortedHistory = [...history] .sort((a, b) => b.create_time - a.create_time) @@ -381,6 +373,32 @@ export const useQueueStore = defineStore('queue', () => { hasFetchedHistorySnapshot.value = true } + const terminalStatuses = new Set([ + 'completed', + 'failed', + 'cancelled' + ]) + + async function reconcileTrackedJobs( + activeJobIds: Set, + history: JobListItem[] + ) { + const executionStore = useExecutionStore() + const terminalJobIds = new Set(history.map((job) => job.id)) + const unresolvedJobIds = executionStore.trackedJobIds.filter( + (jobId) => !activeJobIds.has(jobId) && !terminalJobIds.has(jobId) + ) + const details = await Promise.all( + unresolvedJobIds.map((jobId) => api.getJobDetail(jobId)) + ) + for (const detail of details) { + if (detail && terminalStatuses.has(detail.status)) { + terminalJobIds.add(detail.id) + } + } + executionStore.reconcileTerminalJobs(activeJobIds, terminalJobIds) + } + const update = async () => { if (updateState.inFlight) { updateState.dirty = true @@ -414,10 +432,14 @@ export const useQueueStore = defineStore('queue', () => { if (activeJobIds) { useExecutionStore().reconcileInitializingJobs(activeJobIds) + await reconcileTrackedJobs( + activeJobIds, + historyResult.status === 'fulfilled' ? historyResult.value : [] + ) } if (historyResult.status === 'fulfilled') { - applyHistorySnapshot(historyResult.value, activeJobIds) + applyHistorySnapshot(historyResult.value) } else { console.error('Failed to fetch history:', historyResult.reason) } From 279beb460e49e69b3390308958dcb01cb72cb1eb Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 9 Oct 2026 10:24:25 -0700 Subject: [PATCH 20/52] fix(execution): type execution_interrupted for the cancel Cloud actually sends Read out of the Cloud gateway at 968df21: a user cancel is synthesised there rather than forwarded from the worker, so the frame carries only synthetic, reason and the routing ids. node_id, node_type and executed were typed as required, which is true of core and false of Cloud. The handler never read them, so this is the type catching up with the wire rather than a behaviour change, and the new test pins that the cancelled job's own records are still released. Also noted in the workflow_metadata docblock that the two backends merge in opposite orders: core lets the frame win, the Cloud gateway assigns the routing ids afterwards. The old note described core's order as the contract. --- .../remote/comfyui/execution/types.ts | 15 ++++++++-- src/scripts/api.ts | 7 ++++- .../executionStore.workflowGating.test.ts | 28 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/platform/remote/comfyui/execution/types.ts b/src/platform/remote/comfyui/execution/types.ts index 9e2e6b0ea4e..595feb33210 100644 --- a/src/platform/remote/comfyui/execution/types.ts +++ b/src/platform/remote/comfyui/execution/types.ts @@ -85,9 +85,18 @@ export interface ExecutionCachedWsMessage extends ExecutionWsMessageBase { nodes: NodeId[] } export interface ExecutionInterruptedWsMessage extends ExecutionWsMessageBase { - node_id: NodeId - node_type: string - executed: NodeId[] + /** + * Optional because a user cancel on Cloud is synthesised by the gateway + * rather than forwarded from the worker, and carries only `synthetic`, + * `reason` and the routing ids. Core always sends all three. + */ + node_id?: NodeId + node_type?: string + executed?: NodeId[] + /** Cloud only, set on a gateway-synthesised cancel. */ + synthetic?: boolean + /** Cloud only, the human-readable cancel reason. */ + reason?: string } export interface ExecutionErrorWsMessage extends ExecutionWsMessageBase { node_id?: NodeId | null diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 867c6572288..6e5d80e53d0 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -105,7 +105,12 @@ interface QueuePromptRequestBody { * metadata header on binary previews). An older server ignores the field * rather than rejecting the prompt. * - * Two limits worth knowing, since the contract is the server's and not ours: + * Two limits worth knowing, since the contract is the server's and not ours. + * A third thing worth knowing is that the two backends disagree: core merges + * as `{**workflow_metadata, **data}`, so a frame's own fields win, while the + * Cloud gateway assigns `prompt_id` and `workflow_id` after the fact, so the + * backend wins. Do not rely on either order; the routing ids that arrive are + * authoritative whichever side set them. * * - `status` and `logs` frames have no `prompt_id`, so they are never * stamped and a consumer still needs the `prompt_id`-to-workflow mapping. diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index c8e68b8655e..898b4d01604 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -360,6 +360,34 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') }) + // Read out of Comfy-Org/cloud at 968df21: a user cancel is synthesised by + // the gateway rather than forwarded from the worker, so it arrives as + // { synthetic, reason, prompt_id, workflow_id } with none of the node + // fields core always sends. Raised in review on #20045. + it('handles a gateway-synthesised cancel with no node fields', () => { + queueJobFrom('job-b', workflowB) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 4) } + }) + // Without this the release assertion below passes vacuously. + expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() + + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + synthetic: true, + reason: 'Job cancelled by user' + }) + + expect(store.activeJobId, 'the visible run is untouched').toBe('job-a') + expect( + store.nodeProgressStatesByJob['job-b'], + "the cancelled job's own records are still released" + ).toBeUndefined() + }) + it('execution_error from another workflow leaves active state alone but clears its initializing flag', () => { queueJobFrom('job-b', workflowB) // Without this the job is never initializing, so the assertion below From aebf2b133f111cb3eb8743464cadc6675fd0ee66 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Fri, 9 Oct 2026 19:32:32 -0700 Subject: [PATCH 21/52] fix(test): drop an always-truthy guard in the onJobReset assertion oxlint's no-unnecessary-condition caught it: the hook's payload is a QueuedJob, never undefined, so `job &&` was dead. This is the error that failed lint-pr on the merge commit, which I had pushed with --no-verify so lint-staged never saw it. --- src/stores/executionStore.terminalRecovery.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index a569223e9fc..a3fed53528c 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -312,9 +312,9 @@ describe('executionStore terminal-job recovery', () => { // the delete would hand it nothing and silently remove no widget. Order is // the whole point here. const jobId = stuckQueuedJob() - const seen: (string[] | undefined)[] = [] + const seen: string[][] = [] store.onJobReset((job) => { - seen.push(job && Object.keys(job.nodes)) + seen.push(Object.keys(job.nodes)) }) store.reconcileTerminalJobs(new Set(), new Set([jobId])) From 710630b42e3436697d5ac483aef835991c938e5d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 10 Oct 2026 22:23:43 -0700 Subject: [PATCH 22/52] fix(execution): stop the ownership gate swallowing account errors and job previews Two of the gates added in ad0d463532 gate things that are not keyed by the visible graph, so a background run lost signal the user needs. An account precondition (sign-in, subscription, credits) belongs to the account, not to one workflow. The store returns from handleAccountPreconditionError before recording the error, so this dialog was the only signal a run ever produced; gated, a background run failed in total silence with no dialog, no badge and nothing to find on the tab afterwards. Ungated again, and the test that asserted the silence now asserts the dialog. useJobPreviewStore().setPreviewUrl is keyed by job rather than by graph, and the queue panel thumbnail and linearOutputStore's re-apply on return both read it for background jobs. The gate now sits after it and releases the blob url when it rejects, so only the node-keyed writes are gated. Also deleted a test that could not fail: ensureSessionWorkflowPath writes the instance and the path together, so the two ownership lists cannot disagree today and the assertion only tracked the line it was testing. --- src/scripts/app.test.ts | 26 ++++++++++++++++--- src/scripts/app.ts | 22 +++++++++------- .../executionStore.workflowGating.test.ts | 17 +----------- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 6c83b5437d7..984d972c4f0 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -4,6 +4,7 @@ import { useToast } from '@/components/ui/toast/toastStore' import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore' import { useSubgraphNavigationStore } from '@/stores/subgraphNavigationStore' import { useNodeOutputStore } from '@/stores/nodeOutputStore' +import { useJobPreviewStore } from '@/stores/jobPreviewStore' import { useSettingStore } from '@/platform/settings/settingStore' import { useAuthStore } from '@/stores/authStore' @@ -3099,7 +3100,11 @@ describe('ComfyApp', () => { expect(executionErrorStore.isErrorOverlayOpen).toBe(false) }) - it('does not open an account dialog for a background run error', () => { + // Deliberately the opposite of the node-output and node-preview gates. A + // credit or sign-in problem belongs to the account, and the store returns + // from handleAccountPreconditionError before recording anything, so this + // dialog is the only signal. Gated, a background run failed silently. + it('opens the account dialog even for a background run error', () => { const workflowA = markLoaded( new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) ) @@ -3144,7 +3149,10 @@ describe('ComfyApp', () => { }) ) - expect(open).not.toHaveBeenCalled() + expect(open).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ nodeType: 'PartnerApiNode' }) + ) }) it('does not apply a background run output to the visible workflow', async () => { @@ -3198,7 +3206,7 @@ describe('ComfyApp', () => { expect(setOutputs).not.toHaveBeenCalled() }) - it('does not apply a background preview to the visible workflow', () => { + it('keeps a background preview job-keyed but off the visible graph', () => { const workflowA = markLoaded( new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) ) @@ -3242,7 +3250,17 @@ describe('ComfyApp', () => { }) ) - expect(setPreviews).not.toHaveBeenCalled() + expect( + setPreviews, + 'the node-keyed write is gated: the visible graph has the same node ids' + ).not.toHaveBeenCalled() + // But the job-keyed one must still happen. The queue panel reads the + // running job's thumbnail from here, and linearOutputStore re-applies the + // last latent preview when the user switches back to that tab. + expect( + useJobPreviewStore().previewsByPromptId['job-b'], + 'the background job keeps its own thumbnail' + ).toBeTruthy() }) it('restores the failed run state when returning to a workflow tab', async () => { diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 68f9d40855c..80dff1ee6e7 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -944,13 +944,12 @@ export class ComfyApp { }) api.addEventListener('execution_error', ({ detail }) => { - if ( - !useExecutionStore().belongsToActiveWorkflow( - detail.prompt_id, - detail.workflow_id - ) - ) - return + // Deliberately ungated. A sign-in, subscription or credit problem belongs + // to the account rather than to one workflow, and the store's + // handleAccountPreconditionError returns before recording the error, so + // this dialog is the only signal the user ever gets. Gating it made a + // background run fail silently: no dialog, no badge, and nothing to find + // on the tab afterwards. const precondition = resolveAccountPrecondition({ exceptionType: detail.exception_type, exceptionMessage: detail.exception_message @@ -966,14 +965,19 @@ export class ComfyApp { api.addEventListener('b_preview_with_metadata', ({ detail }) => { // Enhanced preview with explicit node context const { blob, displayNodeId, jobId, workflowId } = detail - if (!useExecutionStore().belongsToActiveWorkflow(jobId, workflowId)) - return const { setNodePreviewsByExecutionId, revokePreviewsByExecutionId } = useNodeOutputStore() const displayNodeExecutionId = tryNormalizeNodeExecutionId(displayNodeId) if (!displayNodeExecutionId) return const blobUrl = createSharedObjectUrl(blob) + // Before the gate: this one is keyed by job rather than by the visible + // graph, and a background run needs it for its queue-panel thumbnail and + // for linearOutputStore's re-apply when the user switches back. useJobPreviewStore().setPreviewUrl(jobId, blobUrl, displayNodeId) + if (!useExecutionStore().belongsToActiveWorkflow(jobId, workflowId)) { + releaseSharedObjectUrl(blobUrl) + return + } // Ensure clean up if `executing` event is missed. revokePreviewsByExecutionId(displayNodeExecutionId) // Preview cleanup is handled in progress_state event to support multiple concurrent previews diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index ec9e06467ec..cae1b7fcb6b 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1238,22 +1238,7 @@ describe('executionStore workflow gating', () => { }) }) - describe('the two ownership rules agree', () => { - // Documents the invariant rather than catching a live bug: today - // ensureSessionWorkflowPath writes the instance and the path together, so - // the two lists cannot disagree in practice. It will fail if a future - // caller sets one without the other. - it('treats a job known by its tab instance as resolvable', () => { - useWorkflowStore().activeWorkflow = workflowA - queueJobFrom('job-a', workflowA) - useWorkflowStore().activeWorkflow = workflowB - - // The instance is the resolver's first leg, so a frame it can place must - // not be reported as unresolvable by the other rule, or one gate accepts - // the frame while the other rejects it. - expect(store.belongsToActiveWorkflow('job-a', undefined)).toBe(false) - }) - + describe('a background job that finishes', () => { it('does not leave activeJobId naming a finished background job', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) From 2dfbea59a8328a51819080d4b956a2dfa92ec208 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 10 Oct 2026 22:38:10 -0700 Subject: [PATCH 23/52] fix(execution): keep per-job state per job, and evict jobs no source can find Round 9 review. Three behaviour fixes, four tests that pin lines which were previously provable only by deleting them. A terminal 'executing' cleared every pending progress_state snapshot, not just the finished job's, so a background job's frame arriving in the same tick was discarded. Snapshots are keyed by job now, so the clear is too. A tracked job absent from the queue, from the history page and from /jobs/{id} stayed tracked forever. Since useQueuePolling began polling while anything is tracked, that one job kept polling awake for the rest of the session and logged a miss every cycle. It is now evicted on the second consecutive failed lookup, because fetchJobDetail returns undefined for a transient network error as well as for a 404. New tests, each verified by reverting exactly the line it covers: - a background pending snapshot survives the visible job ending - execution_interrupted clears the interrupted job's initializing flag, not the visible job's - one row per sentinel normalisation, since the old single test passed while either one remained - the preview test gained a foreground row, because an absence assertion also passes when the listener does nothing Also: the api.ts docblock leads with Cloud not reading the field rather than announcing two limits and then a third thing; the resolver's docblock is reattached to the resolver instead of the helper inserted under it; an orphaned docblock naming a no-longer-imported type is gone; and several comments recording review history rather than explaining code. --- src/scripts/api.ts | 12 +- src/scripts/app.test.ts | 22 ++++ .../executionStore.terminalRecovery.test.ts | 2 - src/stores/executionStore.ts | 55 ++++------ .../executionStore.workflowGating.test.ts | 103 ++++++++++++++++-- src/stores/queueStore.test.ts | 37 ++++++- src/stores/queueStore.ts | 30 ++++- 7 files changed, 208 insertions(+), 53 deletions(-) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 08d403423a3..2d700f85468 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -102,12 +102,12 @@ interface QueuePromptRequestBody { * metadata header on binary previews). An older server ignores the field * rather than rejecting the prompt. * - * Two limits worth knowing, since the contract is the server's and not ours. - * A third thing worth knowing is that the two backends disagree: core merges - * as `{**workflow_metadata, **data}`, so a frame's own fields win, while the - * Cloud gateway assigns `prompt_id` and `workflow_id` after the fact, so the - * backend wins. Do not rely on either order; the routing ids that arrive are - * authoritative whichever side set them. + * Cloud does not read this field: its gateway stamps `workflow_id` from + * `extra_pnginfo.workflow.id` and assigns the routing ids after the frame's + * own fields, where core lets the frame's fields win. Treat the ids that + * arrive as authoritative either way. + * + * Two limits worth knowing, since the contract is the server's and not ours: * * - `status` and `logs` frames have no `prompt_id`, so they are never * stamped and a consumer still needs the `prompt_id`-to-workflow mapping. diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 984d972c4f0..704c03cbef4 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3254,6 +3254,28 @@ describe('ComfyApp', () => { setPreviews, 'the node-keyed write is gated: the visible graph has the same node ids' ).not.toHaveBeenCalled() + // A foreground row, because the absence assertion above also passes if + // the listener does nothing at all: the reviewer deleted the + // setNodePreviewsByExecutionId loop and all 558 tests in src/scripts + // stayed green. + useWorkflowStore().activeWorkflow = workflowB + previewHandler?.( + new CustomEvent('b_preview_with_metadata', { + detail: { + blob: new Blob(), + nodeId: '1', + displayNodeId: '1', + parentNodeId: '1', + realNodeId: '1', + jobId: 'job-b', + workflowId: workflowBId + } + }) + ) + expect( + setPreviews, + "and applied once that job's own tab is in front" + ).toHaveBeenCalled() // But the job-keyed one must still happen. The queue panel reads the // running job's thumbnail from here, and linearOutputStore re-applies the // last latent preview when the user switches back to that tab. diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index a3fed53528c..343007d7eb6 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -371,8 +371,6 @@ describe('executionStore terminal-job recovery', () => { // a job whose terminal frame was dropped before any progress_state arrived // was in no tracked collection and its badge stayed Running for the // session. - // Must be a background job: a visible one is tracked through activeJobId, - // which execution_start only sets for the workflow in front. const jobId = 'job-no-progress' store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) store.storeJob({ diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 35afa865e90..02b38127dff 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -530,10 +530,6 @@ export const useExecutionStore = defineStore('execution', () => { }) const workflow = jobIdToWorkflow.get(jobId) if (workflow) clearWorkflowStatus(workflow) - // This job's flag, not the visible job's. Before the gate existed a - // background interrupt also set activeJobId so the two coincided; now - // clearing activeJobId's flag wipes the visible run's "waiting for a - // machine" state and leaves the interrupted job's own flag set. clearInitializationByJobId(jobId) if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { // The job is finished either way, so its own records have to be released @@ -601,7 +597,13 @@ export const useExecutionStore = defineStore('execution', () => { } progressCoalescer.cancel() - if (e.detail == null) cancelPendingProgressStateUpdates() + if (e.detail == null) { + // Per job: the snapshots are keyed by job now, so clearing the whole map + // here would discard a background job's pending frame that arrived in the + // same tick as the visible job's terminal `executing`. + if (raw) pendingProgressStates.delete(raw.prompt_id) + else cancelPendingProgressStateUpdates() + } // Clear the current node progress when a new node starts executing _executingNodeProgress.value = null @@ -720,14 +722,6 @@ export const useExecutionStore = defineStore('execution', () => { mirrorExecutingNodeProgress(nodes) } - /** - * Graph id of the active workflow, or null when it has none. - * - * `LoadedComfyWorkflow` types both states as present, but the running store - * does not always honour that, and this is read from a watcher and from a - * computed now — so a missing state has to return null rather than throw and - * take unrelated features down with it. - */ /** * Graph id of a workflow: the id carried by the workflow's own JSON, which * every copy of that content shares, so it identifies the content and not @@ -747,6 +741,18 @@ export const useExecutionStore = defineStore('execution', () => { return graphIdOf(workflowStore.activeWorkflow) ?? null } + /** + * The all-zero sentinel is `LGraph._id`'s default, which the codebase reads + * as "no id yet" and replaces on configure and on clear. Treating it as a + * real id would make every workflow still carrying it match every other one + * on the graph-id leg, which is the leak this gate exists to stop. The queue + * side does not send it, but core falls back to the id inside + * `extra_pnginfo.workflow`, so it still arrives on the wire. + */ + function workflowIdOrUndefined(id: string | undefined): string | undefined { + return !id || id === zeroUuid ? undefined : id + } + /** * Determines whether a WebSocket execution message belongs to the currently * active workflow tab, so a job started in another open workflow cannot leak @@ -769,18 +775,6 @@ export const useExecutionStore = defineStore('execution', () => { * to be consulted first. QA hit this with three unsaved copies of the default * graph, where every tab rendered every other tab's outputs. */ - /** - * The all-zero sentinel is `LGraph._id`'s default, which the codebase reads - * as "no id yet" and replaces on configure and on clear. Treating it as a - * real id would make every workflow still carrying it match every other one - * on the graph-id leg, which is the leak this gate exists to stop. The queue - * side does not send it, but core falls back to the id inside - * `extra_pnginfo.workflow`, so it still arrives on the wire. - */ - function workflowIdOrUndefined(id: string | undefined): string | undefined { - return !id || id === zeroUuid ? undefined : id - } - function messageMatchesActiveWorkflow( jobId: JobId, rawMessageWorkflowId: string | undefined @@ -1151,12 +1145,11 @@ export const useExecutionStore = defineStore('execution', () => { // this feature already shipped once, so defer to the same resolver the // handlers use, and require ownership to be resolvable rather than adopting // a job we cannot place. - const matchedJobId = - Object.keys(nodeProgressStatesByJob.value).findLast( - (jobId) => - canResolveWorkflowOwnership(jobId, undefined) && - messageMatchesActiveWorkflow(jobId, undefined) - ) ?? null + const matchedJobId = Object.keys(nodeProgressStatesByJob.value).findLast( + (jobId) => + canResolveWorkflowOwnership(jobId, undefined) && + messageMatchesActiveWorkflow(jobId, undefined) + ) if (matchedJobId) { nodeProgressStates.value = diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index cae1b7fcb6b..f452d712f96 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -349,10 +349,6 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') }) - // Read out of Comfy-Org/cloud at 968df21: a user cancel is synthesised by - // the gateway rather than forwarded from the worker, so it arrives as - // { synthetic, reason, prompt_id, workflow_id } with none of the node - // fields core always sends. Raised in review on #20045. it('handles a gateway-synthesised cancel with no node fields', () => { queueJobFrom('job-b', workflowB) fire('progress_state', { @@ -1163,11 +1159,9 @@ describe('executionStore workflow gating', () => { expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) }) - // Raised in review on #20045. The queue side refuses to send the sentinel - // as workflow_metadata, but merged core falls back to the id inside - // extra_pnginfo.workflow, so a graph still on LGraph's default id has it - // stamped onto its frames anyway. Compared as a real id it matches nothing, - // so the tab's own frames were rejected. + // Core falls back to the id inside extra_pnginfo.workflow, so a graph still + // on LGraph's default id has the sentinel stamped onto its frames. Compared + // as a real id it matches nothing, so the tab's own frames were rejected. it('does not reject a frame whose only id is the all-zero sentinel', () => { useWorkflowStore().activeWorkflow = workflowA @@ -1238,6 +1232,97 @@ describe('executionStore workflow gating', () => { }) }) + // Round 9 findings: each of these pins one line that was previously provable + // only by deleting it and watching nothing happen. + describe('per-job state that a neighbouring job must not disturb', () => { + it('keeps a background pending snapshot when the visible job ends', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + // B's frame and A's terminal `executing` land in the same tick. + const handler = apiEventHandlers.get('progress_state') + assert.exists(handler) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 6) } + } + }) + ) + fireExecuting(null, { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID }) + vi.advanceTimersToNextFrame() + + expect( + store.nodeProgressStatesByJob['job-b'], + "A ending must not drop B's pending snapshot" + ).toBeDefined() + }) + + it('clears only the interrupted job initializing flag', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-a', value: 'Waiting for a machine' }) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-a')).toBe(true) + expect(store.isJobInitializing('job-b')).toBe(true) + // Set directly rather than via execution_start, which would clear A's own + // initializing flag and make the assertion below vacuous. + store.activeJobId = 'job-a' + + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect( + store.isJobInitializing('job-a'), + 'the visible run is still waiting for a machine' + ).toBe(true) + expect(store.isJobInitializing('job-b')).toBe(false) + }) + }) + + describe('the all-zero sentinel, one row per normalization', () => { + it('leaves a sentinel-only frame unresolvable, so activeJobId rejects it', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + // Fails without the normalization in canResolveWorkflowOwnership: the + // sentinel would make the frame look resolvable and the graph-id leg + // would then compare it. + expect(store.belongsToActiveWorkflow('job-x', zeroUuid)).toBe(false) + }) + + it('adopts an unmapped job stamped with the sentinel', () => { + useWorkflowStore().activeWorkflow = workflowA + + // Fails without the normalization in messageMatchesActiveWorkflow: the + // sentinel would be compared against A's real graph id and lose. + fire('execution_start', { + prompt_id: 'job-fresh', + workflow_id: zeroUuid, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-fresh') + }) + }) + describe('a background job that finishes', () => { it('does not leave activeJobId naming a finished background job', () => { useWorkflowStore().activeWorkflow = workflowA diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 0cb895b5e65..a3e184980bc 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -830,11 +830,46 @@ describe('useQueueStore', () => { await store.update() - expect(api.getJobDetail).toHaveBeenCalledWith('older-stuck') expect( executionStore.nodeProgressStatesByJob['older-stuck'] ).toBeUndefined() }) + + // Raised in review: useQueuePolling now stays awake while anything is + // tracked, so a job that no source can find kept polling alive for the rest + // of the session and logged a miss every cycle. One miss is not enough to + // act on, because fetchJobDetail returns undefined for a transient network + // error as well as for a 404. + it('evicts a tracked job only after a second failed lookup', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + ghost: { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'ghost' + } + } + } + + mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([]) + vi.mocked(api.getJobDetail).mockResolvedValue(undefined) + + await store.update() + expect( + executionStore.nodeProgressStatesByJob['ghost'], + 'one miss could be a network blip, so it is still tracked' + ).toBeDefined() + + await store.update() + expect( + executionStore.nodeProgressStatesByJob['ghost'], + 'two consecutive misses means it is gone' + ).toBeUndefined() + }) }) describe('update() - maxHistoryItems limit', () => { diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 29ac5e3b253..929abcd937e 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -379,6 +379,12 @@ export const useQueueStore = defineStore('queue', () => { 'cancelled' ]) + /** + * Consecutive failed `/jobs/{id}` lookups per tracked job, for the eviction + * rule in {@link reconcileTrackedJobs}. + */ + const missedJobLookups = new Map() + async function reconcileTrackedJobs( activeJobIds: Set, history: JobListItem[] @@ -391,11 +397,27 @@ export const useQueueStore = defineStore('queue', () => { const details = await Promise.all( unresolvedJobIds.map((jobId) => api.getJobDetail(jobId)) ) - for (const detail of details) { - if (detail && terminalStatuses.has(detail.status)) { - terminalJobIds.add(detail.id) + unresolvedJobIds.forEach((jobId, index) => { + const detail = details[index] + if (detail) { + missedJobLookups.delete(jobId) + if (terminalStatuses.has(detail.status)) terminalJobIds.add(detail.id) + return } - } + // Absent from the queue, from this page of history, and from + // /jobs/{id}. Such a job is not running, and leaving it tracked keeps + // useQueuePolling awake for the rest of the session, refetching and + // logging a miss every cycle. fetchJobDetail returns undefined for a + // transient network error as well as for a 404, so evict on the second + // consecutive miss rather than the first. + const misses = (missedJobLookups.get(jobId) ?? 0) + 1 + if (misses >= 2) { + missedJobLookups.delete(jobId) + terminalJobIds.add(jobId) + } else { + missedJobLookups.set(jobId, misses) + } + }) executionStore.reconcileTerminalJobs(activeJobIds, terminalJobIds) } From 5bbe9f1c2ba598cca24ad79990e33495055bc2cf Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:11:58 +0000 Subject: [PATCH 24/52] refactor(execution): omit the absent workflow id Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4228823273 --- src/composables/node/useProgressTextPreviews.ts | 2 +- src/stores/executionStore.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/composables/node/useProgressTextPreviews.ts b/src/composables/node/useProgressTextPreviews.ts index 922ecd78c68..01753e51286 100644 --- a/src/composables/node/useProgressTextPreviews.ts +++ b/src/composables/node/useProgressTextPreviews.ts @@ -32,7 +32,7 @@ export function useProgressTextPreviews() { // gate falls back to the activeJobId check only when ownership cannot be // resolved at all. No workflow id to pass: core packs progress_text as a // binary frame that carries none, tracked in Comfy-Org/ComfyUI#16887. - if (!executionStore.belongsToActiveWorkflow(prompt_id, undefined)) return + if (!executionStore.belongsToActiveWorkflow(prompt_id)) return const executionId = String(nodeId) const currentId = executionId.includes(':') diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 02b38127dff..d4b64ef2d99 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -1271,7 +1271,7 @@ export const useExecutionStore = defineStore('execution', () => { */ function belongsToActiveWorkflow( jobId: JobId | undefined, - workflowId: string | undefined + workflowId?: string ): boolean { if (!jobId) return true if (canResolveWorkflowOwnership(jobId, workflowId)) { From 9a052ac0cc6c2b6759e765aa940cd12d1b515fb0 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:04 +0000 Subject: [PATCH 25/52] test(execution): remove the repeated cancel premise Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4233152398 --- src/stores/executionStore.workflowGating.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index f452d712f96..0bc73d8dadd 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -356,8 +356,6 @@ describe('executionStore workflow gating', () => { workflow_id: WORKFLOW_B_ID, nodes: { '1': nodeState('job-b', '1', 'running', 4) } }) - // Without this the release assertion below passes vacuously. - expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() fire('execution_interrupted', { prompt_id: 'job-b', From 51c453d94b1680e952ff3229df8b4fb600a80efe Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:09 +0000 Subject: [PATCH 26/52] test(execution): remove repeated recovery premises Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225672036 --- src/stores/executionStore.terminalRecovery.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index 343007d7eb6..c71712c2809 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -385,11 +385,6 @@ describe('executionStore terminal-job recovery', () => { workflow_id: WORKFLOW_B_ID, timestamp: 1 }) - expect(store.getWorkflowStatus(workflowB)).toBe('running') - expect( - store.activeJobId, - 'B is behind, so it must not be adopted' - ).toBeNull() store.reconcileTerminalJobs(new Set(), new Set([jobId])) From 0f12e18f9a87a035669faf7b07f0e63e4a835e49 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:16 +0000 Subject: [PATCH 27/52] refactor(app): trim the account error comment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240395667 --- src/scripts/app.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 80dff1ee6e7..ef241cf19e9 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -944,12 +944,7 @@ export class ComfyApp { }) api.addEventListener('execution_error', ({ detail }) => { - // Deliberately ungated. A sign-in, subscription or credit problem belongs - // to the account rather than to one workflow, and the store's - // handleAccountPreconditionError returns before recording the error, so - // this dialog is the only signal the user ever gets. Gating it made a - // background run fail silently: no dialog, no badge, and nothing to find - // on the tab afterwards. + // Account-scoped, so not gated by the visible workflow. const precondition = resolveAccountPrecondition({ exceptionType: detail.exception_type, exceptionMessage: detail.exception_message From 9d60965e3968e5e2d93716c7f0a98cf7de2e8b56 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:21 +0000 Subject: [PATCH 28/52] refactor(app): remove the repeated preview comment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240395672 --- src/scripts/app.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/scripts/app.ts b/src/scripts/app.ts index ef241cf19e9..b3ca6d1c44d 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -965,9 +965,6 @@ export class ComfyApp { const displayNodeExecutionId = tryNormalizeNodeExecutionId(displayNodeId) if (!displayNodeExecutionId) return const blobUrl = createSharedObjectUrl(blob) - // Before the gate: this one is keyed by job rather than by the visible - // graph, and a background run needs it for its queue-panel thumbnail and - // for linearOutputStore's re-apply when the user switches back. useJobPreviewStore().setPreviewUrl(jobId, blobUrl, displayNodeId) if (!useExecutionStore().belongsToActiveWorkflow(jobId, workflowId)) { releaseSharedObjectUrl(blobUrl) From 4cf45bc80d18948605e1ab35750a1b3e39089639 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:27 +0000 Subject: [PATCH 29/52] test(app): remove the repeated account error comment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240395675 --- src/scripts/app.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 704c03cbef4..1cbbabf11f4 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3100,10 +3100,6 @@ describe('ComfyApp', () => { expect(executionErrorStore.isErrorOverlayOpen).toBe(false) }) - // Deliberately the opposite of the node-output and node-preview gates. A - // credit or sign-in problem belongs to the account, and the store returns - // from handleAccountPreconditionError before recording anything, so this - // dialog is the only signal. Gated, a background run failed silently. it('opens the account dialog even for a background run error', () => { const workflowA = markLoaded( new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) From 98c055759d301ed8992e00221147b16d1341b4b9 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:32 +0000 Subject: [PATCH 30/52] test(app): remove the repeated thumbnail comment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240395677 --- src/scripts/app.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index 1cbbabf11f4..ef9308aa7e9 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3272,9 +3272,6 @@ describe('ComfyApp', () => { setPreviews, "and applied once that job's own tab is in front" ).toHaveBeenCalled() - // But the job-keyed one must still happen. The queue panel reads the - // running job's thumbnail from here, and linearOutputStore re-applies the - // last latent preview when the user switches back to that tab. expect( useJobPreviewStore().previewsByPromptId['job-b'], 'the background job keeps its own thumbnail' From 00ed31dab015197cf45d8690bc846947e66f2c1f Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:12:59 +0000 Subject: [PATCH 31/52] fix(queue): reset stale job lookup misses Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425959 --- src/stores/queueStore.test.ts | 33 +++++++++++++++++++++++++++++++++ src/stores/queueStore.ts | 3 +++ 2 files changed, 36 insertions(+) diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index a3e184980bc..8ab9dfb0613 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -870,6 +870,39 @@ describe('useQueueStore', () => { 'two consecutive misses means it is gone' ).toBeUndefined() }) + + it('resets the miss count when the tracked job reappears', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + ghost: { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'ghost' + } + } + } + + mockGetQueue + .mockResolvedValueOnce({ Running: [], Pending: [] }) + .mockResolvedValueOnce({ + Running: [createRunningJob(1, 'ghost')], + Pending: [] + }) + .mockResolvedValueOnce({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([]) + vi.mocked(api.getJobDetail) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + + await store.update() + await store.update() + await store.update() + + expect(executionStore.trackedJobIds).toContain('ghost') + }) }) describe('update() - maxHistoryItems limit', () => { diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 929abcd937e..505569a2a84 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -394,6 +394,9 @@ export const useQueueStore = defineStore('queue', () => { const unresolvedJobIds = executionStore.trackedJobIds.filter( (jobId) => !activeJobIds.has(jobId) && !terminalJobIds.has(jobId) ) + for (const jobId of missedJobLookups.keys()) { + if (!unresolvedJobIds.includes(jobId)) missedJobLookups.delete(jobId) + } const details = await Promise.all( unresolvedJobIds.map((jobId) => api.getJobDetail(jobId)) ) From 179961d35f68ebc0936a4c862418bd81ec155ab8 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:18 +0000 Subject: [PATCH 32/52] refactor(queue): trim the retry rationale Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425962 --- src/stores/queueStore.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 505569a2a84..b3729ae8e12 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -407,12 +407,8 @@ export const useQueueStore = defineStore('queue', () => { if (terminalStatuses.has(detail.status)) terminalJobIds.add(detail.id) return } - // Absent from the queue, from this page of history, and from - // /jobs/{id}. Such a job is not running, and leaving it tracked keeps - // useQueuePolling awake for the rest of the session, refetching and - // logging a miss every cycle. fetchJobDetail returns undefined for a - // transient network error as well as for a 404, so evict on the second - // consecutive miss rather than the first. + // fetchJobDetail returns undefined for a network error too, so wait for a + // second miss. const misses = (missedJobLookups.get(jobId) ?? 0) + 1 if (misses >= 2) { missedJobLookups.delete(jobId) From 760c28623ce8f228aa8c025fdbee449bec869e29 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:27 +0000 Subject: [PATCH 33/52] refactor(execution): remove the repeated snapshot comment Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425965 --- src/stores/executionStore.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index d4b64ef2d99..6b52347d1f7 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -598,9 +598,6 @@ export const useExecutionStore = defineStore('execution', () => { progressCoalescer.cancel() if (e.detail == null) { - // Per job: the snapshots are keyed by job now, so clearing the whole map - // here would discard a background job's pending frame that arrived in the - // same tick as the visible job's terminal `executing`. if (raw) pendingProgressStates.delete(raw.prompt_id) else cancelPendingProgressStateUpdates() } From c42a72031413c013e0404a8a8f3515bc8814b851 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:36 +0000 Subject: [PATCH 34/52] test(app): remove preview review history Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425969 --- src/scripts/app.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index ef9308aa7e9..d71852bbf18 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3250,10 +3250,6 @@ describe('ComfyApp', () => { setPreviews, 'the node-keyed write is gated: the visible graph has the same node ids' ).not.toHaveBeenCalled() - // A foreground row, because the absence assertion above also passes if - // the listener does nothing at all: the reviewer deleted the - // setNodePreviewsByExecutionId loop and all 558 tests in src/scripts - // stayed green. useWorkflowStore().activeWorkflow = workflowB previewHandler?.( new CustomEvent('b_preview_with_metadata', { From 554edb889287925487d7e3446ebf92495706db12 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:42 +0000 Subject: [PATCH 35/52] test(execution): remove round history from the suite Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425973 --- src/stores/executionStore.workflowGating.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 0bc73d8dadd..a4ec6fe89a8 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1230,8 +1230,6 @@ describe('executionStore workflow gating', () => { }) }) - // Round 9 findings: each of these pins one line that was previously provable - // only by deleting it and watching nothing happen. describe('per-job state that a neighbouring job must not disturb', () => { it('keeps a background pending snapshot when the visible job ends', () => { useWorkflowStore().activeWorkflow = workflowA From 6095847c6ffc45c971c6826ad41e488cee4cc51a Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:48 +0000 Subject: [PATCH 36/52] test(execution): remove the implied interrupt premise Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425978 --- src/stores/executionStore.workflowGating.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index a4ec6fe89a8..a7cf3c9cae2 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1268,7 +1268,6 @@ describe('executionStore workflow gating', () => { queueJobFrom('job-b', workflowB) fire('notification', { id: 'job-a', value: 'Waiting for a machine' }) fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) - expect(store.isJobInitializing('job-a')).toBe(true) expect(store.isJobInitializing('job-b')).toBe(true) // Set directly rather than via execution_start, which would clear A's own // initializing flag and make the assertion below vacuous. From 41fb2876a1696f7cf88bd96c0e9fe8f55efc1427 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:13:55 +0000 Subject: [PATCH 37/52] test(queue): remove lookup review history Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4240425980 --- src/stores/queueStore.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 8ab9dfb0613..f693c5c0121 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -835,11 +835,6 @@ describe('useQueueStore', () => { ).toBeUndefined() }) - // Raised in review: useQueuePolling now stays awake while anything is - // tracked, so a job that no source can find kept polling alive for the rest - // of the session and logged a miss every cycle. One miss is not enough to - // act on, because fetchJobDetail returns undefined for a transient network - // error as well as for a 404. it('evicts a tracked job only after a second failed lookup', async () => { const executionStore = useExecutionStore() executionStore.nodeProgressStatesByJob = { From 12a3391a7cd451c1d188a7dd414a01319573ec90 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:16:02 +0000 Subject: [PATCH 38/52] fix(execution): unify workflow ownership decisions Addresses review feedback: https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210627015 --- src/stores/executionStore.ts | 107 ++++++------------ .../executionStore.workflowGating.test.ts | 33 +++++- 2 files changed, 61 insertions(+), 79 deletions(-) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 6b52347d1f7..07b81063b17 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -371,8 +371,7 @@ export const useExecutionStore = defineStore('execution', () => { const states = nodeProgressStates.value // Apparently doing this inside `Object.entries` causes issues for (const state of Object.values(states)) { - if (!messageMatchesActiveWorkflow(state.prompt_id, state.workflow_id)) - continue + if (!belongsToActiveWorkflow(state.prompt_id, state.workflow_id)) continue const parts = String(state.display_node_id).split(':') for (let i = 0; i < parts.length; i++) { const executionId = parts.slice(0, i + 1).join(':') @@ -501,7 +500,7 @@ export const useExecutionStore = defineStore('execution', () => { // starting job belongs to the active workflow. Otherwise a job started // from another tab would steal activeJobId and clobber the active tab's // execution UI. - if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) return + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) return executionIdToLocatorCache.clear() executionErrorStore.clearExecutionStartErrors(runErrorKeyForJob(jobId)) @@ -510,7 +509,7 @@ export const useExecutionStore = defineStore('execution', () => { function handleExecutionCached(e: CustomEvent) { if (!activeJob.value) return - if (!messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) return for (const n of e.detail.nodes) { activeJob.value.nodes[n] = true @@ -531,7 +530,7 @@ export const useExecutionStore = defineStore('execution', () => { const workflow = jobIdToWorkflow.get(jobId) if (workflow) clearWorkflowStatus(workflow) clearInitializationByJobId(jobId) - if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { // The job is finished either way, so its own records have to be released // or they leak for the lifetime of the session. Only the shared UI state // below belongs to the visible tab. Text previews stay gated: they are @@ -544,7 +543,7 @@ export const useExecutionStore = defineStore('execution', () => { function handleExecuted(e: CustomEvent) { if (!activeJob.value) return - if (!messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) return activeJob.value.nodes[e.detail.node] = true } @@ -575,7 +574,7 @@ export const useExecutionStore = defineStore('execution', () => { }) } } - if (!messageMatchesActiveWorkflow(jobId, e.detail.workflow_id)) { + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { // Finished either way, so release this job's own records or they leak for // the session. Only the shared UI state below belongs to the visible tab, // and text previews stay gated because they are keyed by node and the @@ -592,7 +591,7 @@ export const useExecutionStore = defineStore('execution', () => { // `executing: null` of a job started in another workflow tab would clear // the visible tab's active job and its node progress. const raw = api.lastExecutingMessage - if (raw && !messageMatchesActiveWorkflow(raw.prompt_id, raw.workflow_id)) { + if (raw && !belongsToActiveWorkflow(raw.prompt_id, raw.workflow_id)) { return } @@ -694,7 +693,7 @@ export const useExecutionStore = defineStore('execution', () => { function applyProgressState(detail: ProgressStateWsMessage) { const { nodes, prompt_id: jobId, workflow_id: messageWorkflowId } = detail - const isActiveWorkflowMessage = messageMatchesActiveWorkflow( + const isActiveWorkflowMessage = belongsToActiveWorkflow( jobId, messageWorkflowId ) @@ -760,10 +759,6 @@ export const useExecutionStore = defineStore('execution', () => { * 2. {@link jobIdToSessionWorkflowPath}, unique per open workflow. * 3. the graph id, from the message or {@link jobIdToWorkflowId}. * - * When ownership cannot be resolved at all, for instance a job queued in a - * different browser session, the message is treated as belonging to the - * active workflow, which preserves single-tab behaviour. - * * A workflow id names a workflow, not an open tab, and several open tabs can * carry the same one: `ensureWorkflowId` keeps whatever id the json already * has, so copying a workflow into a new tab duplicates it, and loading one @@ -772,22 +767,22 @@ export const useExecutionStore = defineStore('execution', () => { * to be consulted first. QA hit this with three unsaved copies of the default * graph, where every tab rendered every other tab's outputs. */ - function messageMatchesActiveWorkflow( + function resolveWorkflowOwnership( jobId: JobId, - rawMessageWorkflowId: string | undefined - ): boolean { + rawMessageWorkflowId?: string + ): 'active' | 'foreign' | 'unresolved' { const messageWorkflowId = workflowIdOrUndefined(rawMessageWorkflowId) const activeWorkflow = workflowStore.activeWorkflow - if (!activeWorkflow) return true + if (!activeWorkflow) return 'unresolved' const mappedInstance = jobIdToWorkflowInstanceId.get(jobId) if (mappedInstance && activeWorkflow.instanceId) { - return mappedInstance === activeWorkflow.instanceId + return mappedInstance === activeWorkflow.instanceId ? 'active' : 'foreign' } const mappedPath = jobIdToSessionWorkflowPath.value.get(jobId) if (mappedPath && activeWorkflow.path) { - return mappedPath === activeWorkflow.path + return mappedPath === activeWorkflow.path ? 'active' : 'foreign' } const activeId = workflowIdOrUndefined(activeWorkflowGraphId() ?? undefined) @@ -795,38 +790,30 @@ export const useExecutionStore = defineStore('execution', () => { const ownerId = messageWorkflowId ?? workflowIdOrUndefined(jobIdToWorkflowId.value.get(jobId)) - if (ownerId) return ownerId === activeId + if (ownerId) return ownerId === activeId ? 'active' : 'foreign' } - return true + return 'unresolved' } /** - * Returns true when workflow ownership for {@link jobId} can be resolved - * — either by an explicit `workflow_id` on the incoming message or by a - * mapping registered when the job was queued. When this returns false - * the caller should fall back to whatever legacy guard applied before - * workflow gating was introduced. + * Whether a message belongs on the visible workflow. Unresolvable messages + * use the legacy active-prompt guard, preserving single-tab behavior without + * letting an unknown job mutate another active run. */ - function canResolveWorkflowOwnership( - jobId: JobId, - messageWorkflowId: string | undefined + function belongsToActiveWorkflow( + jobId: JobId | undefined, + workflowId?: string ): boolean { - // Must list every map messageMatchesActiveWorkflow consults, and reject the - // same ids it rejects, or the two disagree about the same frame: the - // instance is that resolver's first and strongest leg, and leaving it out - // here made a frame resolvable by one rule and unresolvable by the other. - return ( - workflowIdOrUndefined(messageWorkflowId) !== undefined || - jobIdToWorkflowInstanceId.has(jobId) || - jobIdToWorkflowId.value.has(jobId) || - jobIdToSessionWorkflowPath.value.has(jobId) - ) + if (!jobId) return true + const ownership = resolveWorkflowOwnership(jobId, workflowId) + if (ownership !== 'unresolved') return ownership === 'active' + return !activeJobId.value || jobId === activeJobId.value } const progressCoalescer = createRafCoalescer((detail) => { const { prompt_id: jobId, workflow_id: messageWorkflowId } = detail - if (!messageMatchesActiveWorkflow(jobId, messageWorkflowId)) return + if (!belongsToActiveWorkflow(jobId, messageWorkflowId)) return _executingNodeProgress.value = detail }, 'raf:progress') @@ -895,9 +882,7 @@ export const useExecutionStore = defineStore('execution', () => { const runErrorKey = runErrorKeyForJob(e.detail.prompt_id) if (runErrorKey === null) { bufferPendingExecutionError({ detail: e.detail, endTime }) - if ( - !messageMatchesActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id) - ) { + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) { releaseFinishedJobRecords(e.detail.prompt_id) return } @@ -957,7 +942,7 @@ export const useExecutionStore = defineStore('execution', () => { // dialog is already scoped by the run error key inside showExecutionError. // A background job is still finished, so release its own records either way // or they survive the session, same as the success and interrupt paths. - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { releaseFinishedJobRecords(detail.prompt_id) return } @@ -977,7 +962,7 @@ export const useExecutionStore = defineStore('execution', () => { const workflow = jobIdToWorkflow.get(detail.prompt_id) if (workflow) clearWorkflowStatus(workflow) clearInitializationByJobId(detail.prompt_id) - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { releaseFinishedJobRecords(detail.prompt_id) return true } @@ -994,7 +979,7 @@ export const useExecutionStore = defineStore('execution', () => { return false clearInitializationByJobId(detail.prompt_id) - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { releaseFinishedJobRecords(detail.prompt_id) return true } @@ -1021,7 +1006,7 @@ export const useExecutionStore = defineStore('execution', () => { if (!result) return false clearInitializationByJobId(detail.prompt_id) - if (!messageMatchesActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { releaseFinishedJobRecords(detail.prompt_id) return true } @@ -1143,9 +1128,7 @@ export const useExecutionStore = defineStore('execution', () => { // handlers use, and require ownership to be resolvable rather than adopting // a job we cannot place. const matchedJobId = Object.keys(nodeProgressStatesByJob.value).findLast( - (jobId) => - canResolveWorkflowOwnership(jobId, undefined) && - messageMatchesActiveWorkflow(jobId, undefined) + (jobId) => resolveWorkflowOwnership(jobId) === 'active' ) if (matchedJobId) { @@ -1253,30 +1236,6 @@ export const useExecutionStore = defineStore('execution', () => { executionErrorStore.clearPromptError(runErrorKey) } - /** - * Whether a text preview belongs on the visible canvas. - * - * Prefers the workflow-ownership gate and falls back to the legacy - * active-prompt guard only when ownership is unresolvable: activeJobId can - * point at another workflow's job, which would otherwise drop frames for the - * workflow the user is looking at. - * - * Exported because node outputs are applied outside this store, in ComfyApp's - * own `executed` listener, which resolves the execution id against the - * visible graph. Without this gate a finished job writes its output onto the - * same-numbered node of whatever tab is in front. - */ - function belongsToActiveWorkflow( - jobId: JobId | undefined, - workflowId?: string - ): boolean { - if (!jobId) return true - if (canResolveWorkflowOwnership(jobId, workflowId)) { - return messageMatchesActiveWorkflow(jobId, workflowId) - } - return !activeJobId.value || jobId === activeJobId.value - } - function storeJob({ nodes, id, diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index a7cf3c9cae2..3dbabc3ed88 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -268,6 +268,17 @@ describe('executionStore workflow gating', () => { expect(store.activeJob?.nodes['1']).toBe(own ? true : undefined) }) + + it('does not apply an unresolvable frame to another active job', () => { + fire('executed', { + prompt_id: 'job-x', + node: '1', + display_node: '1', + output: {} + }) + + expect(store.activeJob?.nodes['1']).toBeUndefined() + }) }) describe('terminal frames', () => { @@ -290,6 +301,23 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') }) + it('unresolvable execution_error does not reset another active job', () => { + fire('execution_error', { + prompt_id: 'job-x', + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [], + executed: [], + current_inputs: {}, + current_outputs: {} + }) + + expect(store.activeJobId).toBe('job-a') + }) + // Regression: gating the whole handler hid the completed badge on a // background tab, which workflowTabStatus.spec.ts caught on CI. Per-workflow // status is keyed by the job's own workflow and is never shared state. @@ -1297,17 +1325,12 @@ describe('executionStore workflow gating', () => { timestamp: 1 }) - // Fails without the normalization in canResolveWorkflowOwnership: the - // sentinel would make the frame look resolvable and the graph-id leg - // would then compare it. expect(store.belongsToActiveWorkflow('job-x', zeroUuid)).toBe(false) }) it('adopts an unmapped job stamped with the sentinel', () => { useWorkflowStore().activeWorkflow = workflowA - // Fails without the normalization in messageMatchesActiveWorkflow: the - // sentinel would be compared against A's real graph id and lose. fire('execution_start', { prompt_id: 'job-fresh', workflow_id: zeroUuid, From 2895d9d64611772328579bf33a205f684792b9f2 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:24:44 +0000 Subject: [PATCH 39/52] refactor(execution): satisfy audit findings --- src/stores/executionStore.ts | 43 +++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index 07b81063b17..a6731d6f085 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -53,6 +53,8 @@ type RuntimeExecutionError = Omit< traceback?: ExecutionErrorWsMessage['traceback'] | null } +type WorkflowOwnership = 'active' | 'foreign' | 'unresolved' + interface ExecutionNodeInfo { title?: string | null type?: string | null @@ -749,6 +751,14 @@ export const useExecutionStore = defineStore('execution', () => { return !id || id === zeroUuid ? undefined : id } + function compareWorkflowOwner( + owner: string | undefined, + active: string | undefined + ): WorkflowOwnership { + if (!owner || !active) return 'unresolved' + return owner === active ? 'active' : 'foreign' + } + /** * Determines whether a WebSocket execution message belongs to the currently * active workflow tab, so a job started in another open workflow cannot leak @@ -770,30 +780,28 @@ export const useExecutionStore = defineStore('execution', () => { function resolveWorkflowOwnership( jobId: JobId, rawMessageWorkflowId?: string - ): 'active' | 'foreign' | 'unresolved' { + ): WorkflowOwnership { const messageWorkflowId = workflowIdOrUndefined(rawMessageWorkflowId) const activeWorkflow = workflowStore.activeWorkflow if (!activeWorkflow) return 'unresolved' - const mappedInstance = jobIdToWorkflowInstanceId.get(jobId) - if (mappedInstance && activeWorkflow.instanceId) { - return mappedInstance === activeWorkflow.instanceId ? 'active' : 'foreign' - } + const instanceOwnership = compareWorkflowOwner( + jobIdToWorkflowInstanceId.get(jobId), + activeWorkflow.instanceId + ) + if (instanceOwnership !== 'unresolved') return instanceOwnership - const mappedPath = jobIdToSessionWorkflowPath.value.get(jobId) - if (mappedPath && activeWorkflow.path) { - return mappedPath === activeWorkflow.path ? 'active' : 'foreign' - } + const pathOwnership = compareWorkflowOwner( + jobIdToSessionWorkflowPath.value.get(jobId), + activeWorkflow.path + ) + if (pathOwnership !== 'unresolved') return pathOwnership const activeId = workflowIdOrUndefined(activeWorkflowGraphId() ?? undefined) - if (activeId) { - const ownerId = - messageWorkflowId ?? - workflowIdOrUndefined(jobIdToWorkflowId.value.get(jobId)) - if (ownerId) return ownerId === activeId ? 'active' : 'foreign' - } - - return 'unresolved' + const ownerId = + messageWorkflowId ?? + workflowIdOrUndefined(jobIdToWorkflowId.value.get(jobId)) + return compareWorkflowOwner(ownerId, activeId) } /** @@ -1454,7 +1462,6 @@ export const useExecutionStore = defineStore('execution', () => { ensureSessionWorkflowPath, getWorkflowStatus, clearWorkflowStatus, - rewriteSessionWorkflowPaths, belongsToActiveWorkflow } }) From 9ca66930a30e226531d2ca85a870ae8d5933f0c2 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:46:47 +0000 Subject: [PATCH 40/52] test(execution): address browser review feedback Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210664258 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210664897 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210665534 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210694432 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210695974 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210698334 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425765 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425777 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425823 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425834 --- .../fixtures/components/SidebarTab.ts | 9 +- browser_tests/fixtures/execution.ts | 20 ++ .../fixtures/helpers/BackendSimulator.test.ts | 222 ++++++-------- .../fixtures/helpers/BackendSimulator.ts | 173 ++--------- .../fixtures/helpers/ExecutionHelper.ts | 81 +---- .../fixtures/helpers/NodeOperationsHelper.ts | 9 + .../helpers/Preview3DPipelineFixture.ts | 6 +- .../fixtures/helpers/WorkflowHelper.ts | 32 +- .../execution/twoTabExecutionLeak.spec.ts | 199 ++++++++++++ .../workflowScopedExecution.spec.ts | 172 +++-------- .../tests/twoTabExecutionLeak.spec.ts | 282 ------------------ 11 files changed, 416 insertions(+), 789 deletions(-) create mode 100644 browser_tests/fixtures/execution.ts create mode 100644 browser_tests/tests/execution/twoTabExecutionLeak.spec.ts rename browser_tests/tests/{ => execution}/workflowScopedExecution.spec.ts (54%) delete mode 100644 browser_tests/tests/twoTabExecutionLeak.spec.ts diff --git a/browser_tests/fixtures/components/SidebarTab.ts b/browser_tests/fixtures/components/SidebarTab.ts index 08e5b049b5d..90def6486bc 100644 --- a/browser_tests/fixtures/components/SidebarTab.ts +++ b/browser_tests/fixtures/components/SidebarTab.ts @@ -221,10 +221,11 @@ export class WorkflowsSidebarTab extends SidebarTab { }) } - getPersistedItem(name: string) { - return this.root.locator('.comfyui-workflows-browse .node-label', { - hasText: name - }) + getPersistedItem(name: string, options: { exact?: boolean } = {}) { + const browse = this.root.locator('.comfyui-workflows-browse') + return options.exact + ? browse.getByText(name, { exact: true }) + : browse.locator('.node-label', { hasText: name }) } async renameWorkflow(locator: Locator, newName: string) { diff --git a/browser_tests/fixtures/execution.ts b/browser_tests/fixtures/execution.ts new file mode 100644 index 00000000000..a456954b8ad --- /dev/null +++ b/browser_tests/fixtures/execution.ts @@ -0,0 +1,20 @@ +import { mergeTests } from '@playwright/test' + +import { comfyPageFixture } from '@e2e/fixtures/ComfyPage' +import { BackendSimulator } from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const base = mergeTests(comfyPageFixture, webSocketFixture) + +export const executionFixture = base.extend<{ + execution: ExecutionHelper + backendSimulator: BackendSimulator +}>({ + execution: async ({ comfyPage, getWebSocket }, use) => { + await use(new ExecutionHelper(comfyPage, await getWebSocket())) + }, + backendSimulator: async ({ execution }, use) => { + await use(new BackendSimulator(execution)) + } +}) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts index 90bda5d8f77..ca542a1cce5 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.test.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -18,8 +18,7 @@ interface SentFrame { data: Record } -/** Seeds every schedule assertion runs over, so none of them can be lucky. */ -const SEEDS = [0, 1, 3, 7, 42, 99, 12345] +const OVERLAP_SEEDS = [0, 3] function harness() { const sent: SentFrame[] = [] @@ -34,7 +33,6 @@ function harness() { return { sent, binary, simulator: new BackendSimulator(execution), execution } } -/** The positions a workflow's frames occupy in the order they were sent. */ function positionsOf(sent: SentFrame[], workflowId: string): number[] { return sent.flatMap((frame, index) => frame.data.workflow_id === workflowId ? [index] : [] @@ -67,9 +65,9 @@ describe('BackendSimulator frame scripting', () => { const { sent, simulator } = harness() const legacy = simulator.prompt('job-a') - simulator.play([legacy.start(), legacy.progress('1', 1, 4)]) + simulator.play([legacy.start(), ...legacy.nodeRunning('1', 1, 4)]) - expect(sent).toHaveLength(2) + expect(sent).toHaveLength(3) expect(sent.every((frame) => !('workflow_id' in frame.data))).toBe(true) }) @@ -89,9 +87,6 @@ describe('BackendSimulator frame scripting', () => { it('never lets metadata overwrite a frame field', () => { const { sent, execution } = harness() - // Core merges `{**workflow_metadata, **data}`, so a client cannot hijack a - // frame's own routing fields by naming them in its metadata. Reversing the - // spread order in `emit` turns this red. execution.withMetadata( { workflow_id: 'wf-a', prompt_id: 'spoofed', node: 'spoofed' }, () => execution.executing('job-a', '7') @@ -113,8 +108,6 @@ describe('BackendSimulator frame scripting', () => { execution.status(2) }) - // The first frame proves the scope was live when `status` was sent, so the - // second assertion is about the exemption and not about an empty scope. expect(sent[0].data.workflow_id).toBe('wf-a') expect(sent[1].type).toBe('status') expect(sent[1].data).not.toHaveProperty('workflow_id') @@ -173,92 +166,84 @@ describe('BackendSimulator frame scripting', () => { ]) }) - it('points an ambiguous swap at the selector that can resolve it', () => { - const { simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [ - prompt.progress('1', 1, 4), - ...prompt.nodeRunning('1', 2, 4) - ] - - // The old message only offered `jobId`, which cannot disambiguate two - // frames of the *same* job — advice a test author could not act on. - expect(() => - swapFrames(script, 'wf-a:progress', 'wf-a:progress_state') - ).toThrow(/occurrence/) - }) - - it('throws rather than silently skipping an unknown swap target', () => { - const { simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - - expect(() => - swapFrames([prompt.start()], 'wf-a:execution_start', 'wf-a:nope') - ).toThrow(/not in the script/) - }) - - it('throws rather than swapping an ambiguously selected frame', () => { - const { simulator } = harness() - const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) - - expect(() => - swapFrames( - [first.start(), second.start(), first.success()], - 'wf-a:execution_start', - 'wf-a:execution_success' - ) - ).toThrow(/matches 2 frames/) - }) - - it('throws rather than swapping a frame with itself', () => { - const { simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [prompt.start(), prompt.success()] - - // Both selectors resolve, each unambiguously, and the swap is still a - // no-op — the same silent no-fault `requireMatches` rules out. - expect(() => - swapFrames(script, 'wf-a:execution_start', 'wf-a:execution_start') - ).toThrow(/same frame/) - }) - - it('throws rather than silently applying no fault at all', () => { - const { simulator } = harness() - const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) - const script = [prompt.start()] - - expect(() => dropFrames(script, 'wf-a:execution_success')).toThrow( - /not in the script/ - ) - expect(() => duplicateFrames(script, 'wf-a:progress')).toThrow( - /not in the script/ - ) + it.for([ + { + name: 'an unknown swap target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => + swapFrames([prompt.start()], 'wf-a:execution_start', 'wf-a:nope') + }, + message: /not in the script/ + }, + { + name: 'an ambiguous swap target', + makeCall: () => { + const { simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) + return () => + swapFrames( + [first.start(), second.start(), first.success()], + 'wf-a:execution_start', + 'wf-a:execution_success' + ) + }, + message: /matches 2 frames/ + }, + { + name: 'the same frame twice', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + const script = [prompt.start(), prompt.success()] + return () => + swapFrames(script, 'wf-a:execution_start', 'wf-a:execution_start') + }, + message: /same frame/ + }, + { + name: 'an unknown drop target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => dropFrames([prompt.start()], 'wf-a:execution_success') + }, + message: /not in the script/ + }, + { + name: 'an unknown duplicate target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => duplicateFrames([prompt.start()], 'wf-a:progress') + }, + message: /not in the script/ + } + ])('throws for $name', ({ makeCall, message }) => { + expect(makeCall()).toThrow(message) }) }) describe('BackendSimulator frame granularity', () => { - /** Every frame builder on `SimulatedPrompt`, scripted once. */ function everyFrame(simulator: BackendSimulator): Frame[] { const p = simulator.prompt('job-a', { workflowId: 'wf-a' }) return [ p.start(), p.executing('1'), - p.progress('1', 1, 4), ...p.nodeRunning('1', 1, 4), p.executed('1', {}), p.success(), - p.error('1', 'boom'), - p.interrupted('1'), p.latentPreview('1') ] } it('sends exactly one WebSocket frame per scheduled Frame', () => { - // The contract `Frame` documents, and the one a fault helper depends on: a - // Frame that sent two messages would leave the second unreachable to every - // selector and would make `interleave` schedule fewer units than it emits. - // Counts raw sends, so the binary `latentPreview` is included. let sends = 0 const ws = fromPartial({ send: () => { @@ -280,8 +265,6 @@ describe('BackendSimulator frame granularity', () => { simulator.play(script) - // `latentPreview` is binary and never reaches the JSON stub, so compare the - // JSON frames against the labels of the Frames that produce them. expect(sent.map((frame) => frame.type)).toEqual( script.slice(0, -1).map((frame) => frame.label.split(':')[1]) ) @@ -291,15 +274,7 @@ describe('BackendSimulator frame granularity', () => { const { simulator } = harness() const script = everyFrame(simulator) - // The carve-out above excludes exactly one builder from the label contract, - // so this is the only thing that holds it to it. `latentPreview` sends a - // type-4 message, which `api.ts` dispatches as `b_preview_with_metadata` - // (plus a legacy `b_preview` alias); labelling the frame with the alias made - // the primary event name throw `not in the script`. expect(script.at(-1)?.label).toBe('wf-a:b_preview_with_metadata') - expect(() => - dropFrames(script, 'wf-a:b_preview_with_metadata') - ).not.toThrow() }) it('scripts the two halves of a running node independently', () => { @@ -307,9 +282,6 @@ describe('BackendSimulator frame granularity', () => { const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) const script = [prompt.start(), ...prompt.nodeRunning('1', 1, 4)] - // A backend that reported the state change but never the step count is one - // of the shapes behind progress stuck after a run; it needs the `progress` - // half to be individually droppable. simulator.play(dropFrames(script, 'wf-a:progress')) expect(sent.map((frame) => frame.type)).toEqual([ @@ -326,8 +298,8 @@ describe('BackendSimulator interleaving', () => { const b = h.simulator.prompt('job-b', { workflowId: 'wf-b' }) h.simulator.play( interleave( - [a.start(), a.progress('1', 1, 4), a.success()], - [b.start(), b.progress('2', 1, 4), b.success()], + [a.start(), a.executed('1', {}), a.success()], + [b.start(), b.executed('2', {}), b.success()], seed ) ) @@ -339,43 +311,44 @@ describe('BackendSimulator interleaving', () => { twoPrompts(7).map((frame) => `${frame.data.workflow_id}:${frame.type}`) ).toEqual([ 'wf-a:execution_start', - 'wf-a:progress', + 'wf-a:executed', 'wf-b:execution_start', 'wf-a:execution_success', - 'wf-b:progress', + 'wf-b:executed', 'wf-b:execution_success' ]) }) - it.for(SEEDS)('keeps both prompts in flight at once for seed %i', (seed) => { - const sent = twoPrompts(seed) - const a = positionsOf(sent, 'wf-a') - const b = positionsOf(sent, 'wf-b') + it.for(OVERLAP_SEEDS)( + 'keeps both prompts in flight at once for seed %i', + (seed) => { + const sent = twoPrompts(seed) + const a = positionsOf(sent, 'wf-a') + const b = positionsOf(sent, 'wf-b') - // The two runs' spans intersect: neither prompt's script finishes before - // the other's starts. Without this the schedule is sequential and a - // concurrency spec can pass having never had two prompts in flight. - expect(a[0]).toBeLessThan(b[b.length - 1]) - expect(b[0]).toBeLessThan(a[a.length - 1]) - }) + expect(a[0]).toBeLessThan(b[b.length - 1]) + expect(b[0]).toBeLessThan(a[a.length - 1]) + } + ) - it.for(SEEDS)( + it.for(OVERLAP_SEEDS)( "preserves each prompt's own dependency order for seed %i", (seed) => { const sent = twoPrompts(seed) - const typesOf = (workflowId: string) => - sent + function typesOf(workflowId: string) { + return sent .filter((frame) => frame.data.workflow_id === workflowId) .map((frame) => frame.type) + } expect(typesOf('wf-a')).toEqual([ 'execution_start', - 'progress', + 'executed', 'execution_success' ]) expect(typesOf('wf-b')).toEqual([ 'execution_start', - 'progress', + 'executed', 'execution_success' ]) } @@ -391,20 +364,14 @@ describe('BackendSimulator interleaving', () => { it('gives different seeds different schedules', () => { const schedules = new Set( - SEEDS.map((seed) => + OVERLAP_SEEDS.map((seed) => twoPrompts(seed) .map((frame) => frame.data.workflow_id) .join(',') ) ) - // `seed` has to name a schedule, so near-distinct is the bar: these seven - // seeds produce six distinct orders. A coin that merely alternates — LCG - // bit 0, which flips every step whatever the seed — collapses them to two - // (`ababab` and `bababa`) and still passes a `> 1` check. The exact-32-bit - // multiply is pinned separately by the seed 7 order above; this assertion - // is about the coin, not the arithmetic. - expect(schedules.size).toBeGreaterThanOrEqual(5) + expect(schedules.size).toBe(2) }) it('throws rather than returning a schedule that cannot overlap', () => { @@ -412,11 +379,20 @@ describe('BackendSimulator interleaving', () => { const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) const script = [a.start(), a.success()] - // Both forcing rules are gated on the other side being non-empty, so an - // empty side used to return the other script verbatim: a schedule with no - // overlap at all, passing every concurrency assertion vacuously because - // there is only one prompt to assert about. expect(() => interleave(script, [], 7)).toThrow(/both sides/) expect(() => interleave([], script, 7)).toThrow(/both sides/) + expect(() => interleave([script[0]], [script[1]], 7)).toThrow( + /at least three/ + ) + }) + + it.for([11.9, 2 ** 32 + 11])('rejects unsupported seed %s', (seed) => { + const { simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + + expect(() => + interleave([a.start(), a.success()], [b.start()], seed) + ).toThrow(/signed 32-bit integer/) }) }) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts index 68917603845..e9004c78c66 100644 --- a/browser_tests/fixtures/helpers/BackendSimulator.ts +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -1,46 +1,18 @@ import type { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' -/** - * One scheduled WebSocket frame. Frames are built up front and sent by - * {@link BackendSimulator.play}, so a test states the exact arrival order of - * two prompts' messages instead of relying on timing. - */ export interface Frame { - /** `:` — used by the fault helpers. */ - label: string - /** - * The prompt that sent this frame. Two concurrent prompts of the *same* - * workflow share a label, so a fault that must hit one of them selects on - * this as well — see {@link FrameSelector}. - */ - jobId: string - send: () => void + readonly label: string + readonly send: () => void } -/** - * Which frames a fault applies to: the label a {@link Frame} carries, which is - * `:`. A label can match several frames, and every - * match is faulted, so a bare label spanning two prompts injects a wider fault - * than a test naming one frame probably means. - */ export type FrameSelector = string -function describeSelector(selector: FrameSelector): string { - return selector -} - function indicesOf(frames: Frame[], selector: FrameSelector): number[] { - const found: number[] = [] - frames.forEach((frame, index) => { - if (frame.label === selector) found.push(index) - }) - return found + return frames.flatMap((frame, index) => + frame.label === selector ? [index] : [] + ) } -/** - * A fault that silently matches nothing is a test that silently stops - * exercising its fault, so every helper fails loudly instead. - */ function requireMatches( frames: Frame[], selector: FrameSelector, @@ -48,18 +20,11 @@ function requireMatches( ): number[] { const found = indicesOf(frames, selector) if (found.length === 0) { - throw new Error( - `${helper}: ${describeSelector(selector)} is not in the script` - ) + throw new Error(`${helper}: ${selector} is not in the script`) } return found } -/** - * A prompt the simulated backend is running. Every method returns a {@link Frame} - * rather than sending immediately, and each frame carries this prompt's - * `workflow_id` when one was given. - */ export class SimulatedPrompt { constructor( private readonly execution: ExecutionHelper, @@ -70,7 +35,6 @@ export class SimulatedPrompt { private frame(type: string, send: () => void): Frame { return { label: `${this.workflowId ?? this.jobId}:${type}`, - jobId: this.jobId, send: () => this.execution.withWorkflowId(this.workflowId, send) } } @@ -87,24 +51,6 @@ export class SimulatedPrompt { ) } - progress(nodeId: string, value: number, max: number): Frame { - return this.frame('progress', () => - this.execution.progress(this.jobId, nodeId, value, max) - ) - } - - /** - * The two frames a running node produces — `progress_state`, then `progress`. - * - * Returned as two {@link Frame}s rather than one because a Frame is one - * WebSocket frame. A single Frame that sent both would make the `progress` - * half unreachable to every fault helper (no Frame carries its label, so - * `dropFrames(script, 'wf:progress')` would throw `not in the script` while a - * `progress` frame was in fact on the wire), would force a drop or duplicate - * of one half to hit both, and would make {@link interleave} schedule fewer - * units than it emits. Spread it into a script: - * `[p.start(), ...p.nodeRunning('1', 1, 4)]`. - */ nodeRunning(nodeId: string, value: number, max: number): [Frame, Frame] { return [ this.frame('progress_state', () => @@ -134,24 +80,6 @@ export class SimulatedPrompt { ) } - error(nodeId: string, message: string): Frame { - return this.frame('execution_error', () => - this.execution.executionError(this.jobId, nodeId, message) - ) - } - - interrupted(nodeId: string): Frame { - return this.frame('execution_interrupted', () => - this.execution.executionInterrupted(this.jobId, nodeId) - ) - } - - /** - * The binary preview frame. Labelled `b_preview_with_metadata`, which is the - * event `api.ts` dispatches for the type-4 message this sends; the `b_preview` - * it also dispatches is a back-compat alias, and labelling the frame with the - * alias made the primary event name unselectable. - */ latentPreview(nodeId: string): Frame { return this.frame('b_preview_with_metadata', () => this.execution.latentPreview(this.jobId, nodeId) @@ -159,25 +87,9 @@ export class SimulatedPrompt { } } -/** - * Scripts the WebSocket side of execution for one or more prompts. - * - * The bugs this exists for — node progress stuck after a run, progress from one - * workflow tab landing on another, outputs applied to the wrong workflow — only - * reproduce under specific frame orders, drops and overlaps. Hand-testing cannot - * produce them reliably, so they are expressed as explicit frame scripts here. - * - * Socket drop/reconnect is not modelled: use `nextWebSocket` from the - * `webSocketFixture` to arm a waiter, then close the current route. - */ export class BackendSimulator { constructor(private readonly execution: ExecutionHelper) {} - /** - * Declare a prompt the backend is running. `jobId` is what - * {@link ExecutionHelper.run} returned for the tab that submitted it. - * Omit `workflowId` to emulate a core build that does not send metadata. - */ prompt( jobId: string, options: { workflowId?: string } = {} @@ -185,35 +97,16 @@ export class BackendSimulator { return new SimulatedPrompt(this.execution, jobId, options.workflowId) } - /** Send frames in the given order. */ play(frames: Frame[]): void { for (const frame of frames) frame.send() } } -/** - * Remove every selected frame — a backend that never sent it. Pass a `jobId` to - * drop one prompt's frame while a concurrent prompt of the same workflow keeps - * its own. - */ - -/** - * Remove selected frames, as a backend that never sent them would. - * - * Every match is dropped, so a bare label spanning two prompts injects a wider - * fault than a test naming one frame probably means. - */ export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { const drop = new Set(requireMatches(frames, selector, 'dropFrames')) return frames.filter((_, index) => !drop.has(index)) } -/** - * Send selected frames twice, as a retrying or reconnecting backend does. - * - * Every match is duplicated, so a bare label matching two prompts injects a - * wider fault than a test naming one frame probably means. - */ export function duplicateFrames( frames: Frame[], selector: FrameSelector @@ -224,16 +117,6 @@ export function duplicateFrames( ) } -/** - * Swap two frames, producing out-of-order arrival. - * - * Each selector must identify exactly one frame, and the two must not be the - * same frame. A swap is a pair operation, so quietly taking the first of several - * matches is how a two-prompt script ends up reordering the wrong prompt's - * frame; add a `jobId` or an `occurrence` to disambiguate. Swapping a frame with - * itself is the same silent no-fault that {@link requireMatches} exists to rule - * out, so it throws rather than returning the script unchanged. - */ export function swapFrames( frames: Frame[], first: FrameSelector, @@ -247,15 +130,14 @@ export function swapFrames( ] as const) { if (found.length > 1) { throw new Error( - `swapFrames: ${describeSelector(selector)} matches ${found.length} frames; ` + - 'add a jobId (different prompts) or an occurrence (same prompt, ' + - 'repeated event) to select one' + `swapFrames: ${selector} matches ${found.length} frames; ` + + 'pick a label that matches exactly one frame' ) } } if (a[0] === b[0]) { throw new Error( - `swapFrames: ${describeSelector(first)} and ${describeSelector(second)} ` + + `swapFrames: ${first} and ${second} ` + 'select the same frame; swapping it with itself applies no fault' ) } @@ -264,29 +146,15 @@ export function swapFrames( return swapped } -/** - * Deterministic interleave of two prompts' frames. `seed` selects the order, so - * a failure is reproducible from the seed printed in the test name. - * - * Each input's own order is preserved — frames of one prompt never arrive out of - * dependency order relative to each other, only relative to the other prompt's. - * - * The result is guaranteed to **overlap**: neither prompt's script finishes - * before the other's starts, so the two runs are genuinely concurrent. Without - * that guarantee a schedule can come out sequential and a concurrency spec - * passes without ever having two prompts in flight. (A one-frame-each pair - * cannot overlap; everything larger does.) - * - * An empty input throws. Both forcing rules below are gated on the other side - * being non-empty, so an empty side would return the other script verbatim with - * no overlap at all — the same silent degradation to no-fault that - * {@link requireMatches} refuses, and with the same consequence: a concurrency - * spec passing having never been concurrent. - */ export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { - if (a.length === 0 || b.length === 0) { + if (!Number.isInteger(seed) || seed !== (seed | 0)) { + throw new Error( + `interleave: seed must be a signed 32-bit integer, got ${seed}` + ) + } + if (a.length === 0 || b.length === 0 || a.length + b.length < 3) { throw new Error( - `interleave: needs frames on both sides to overlap, got ${a.length} and ${b.length}` + `interleave: needs frames on both sides and at least three in total to overlap, got ${a.length} and ${b.length}` ) } const out: Frame[] = [] @@ -294,16 +162,9 @@ export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { let i = 0 let j = 0 while (i < a.length || j < b.length) { - // Exact 32-bit arithmetic. `state * 1103515245` exceeds 2^53 as a double, - // so a plain multiply rounds the low bits away: every seed then produced - // the same constant coin and the helper emitted all of `a` before any of - // `b`. `Math.imul` is the multiply that does not lose them. + // Math.imul: the plain product exceeds 2^53 and rounds the low bits away. state = (Math.imul(state, 1103515245) + 12345) | 0 - // Bit 0 of a power-of-two-modulus LCG flips on every step whatever the - // seed, which would make the schedule a fixed A/B/A/B. Take a high bit. let takeA = j >= b.length || (i < a.length && ((state >>> 16) & 1) === 0) - // Overlap guarantee, in two symmetric rules: do not emit one prompt's last - // frame while the other has not started. if (takeA && b.length > 0 && j === 0 && i === a.length - 1) takeA = false else if (!takeA && a.length > 0 && i === 0 && j === b.length - 1) takeA = true diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index 76d6329841a..2606c29374f 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -17,6 +17,10 @@ type RunOptions = { beforePromptResponse?: (jobId: string) => void | Promise } +export function imageOutput(filename: string) { + return { images: [{ filename, subfolder: '', type: 'output' }] } +} + /** * Build a `NodeError` describing a single failed input on a KSampler node. * Shared between specs that surface validation rings via 400 responses. @@ -70,12 +74,6 @@ export class ExecutionHelper { return this.ws } - /** - * Stamp `workflow_id` on every eligible JSON frame sent inside `fn`, the way - * core's `send_sync` spreads a prompt's `workflow_metadata` onto outgoing - * messages. Leave `workflowId` undefined to emit frames the way a core build - * without that support does — the legacy backend profile. - */ withWorkflowId(workflowId: string | undefined, send: () => void): void { this.withMetadata( workflowId === undefined ? undefined : { workflow_id: workflowId }, @@ -83,42 +81,6 @@ export class ExecutionHelper { ) } - /** - * The general form of {@link withWorkflowId}: core accepts an arbitrary - * `workflow_metadata` dict, so a scenario can set keys that collide with a - * frame's own fields and assert the frame still wins. - * - * The scope survives `await`s inside `fn`. A callback that returns a promise - * keeps the scope open until that promise settles, so frames emitted after an - * `await` are still stamped — returning the promise used to restore the - * previous scope immediately, silently unstamping (or mis-stamping) every - * frame after the first suspension point. - * - * The scope is ambient for as long as it is open, not lexical to `fn`: a frame - * emitted from anywhere while an async scope is pending is stamped with it. - * That is the point — it models core holding a prompt's `workflow_metadata` - * for the duration of the run — but it means an un-awaited scope keeps - * stamping, so always await the returned promise. - * - * **Exactly one scope may be open at a time, and a second one throws** — - * whether it overlaps an async scope or is nested inside `fn`. Without - * ambient async context two live scopes cannot be attributed to their - * prompts, and the wrong `workflow_id` on a frame is precisely the bug this - * harness exists to catch. Nesting used to slip past this guard and was - * worse than ambiguous: the inner scope's `finally` restored *its* saved - * value, so an outer scope that had already closed came back to life and - * stamped every later frame for the rest of the test. Await the first scope, - * or script the prompts as separate synchronous frames via - * `BackendSimulator`. - */ - /** - * Stamp `metadata` on every eligible JSON frame sent inside `send`. - * - * Synchronous on purpose. The only caller sends one frame, and an async - * callback was the shape that let an outer scope come back to life and stamp - * every later frame for the rest of a test. Script prompts as separate - * frames through `BackendSimulator` instead of awaiting inside a scope. - */ withMetadata( metadata: Record | undefined, send: () => void @@ -132,15 +94,6 @@ export class ExecutionHelper { } } - /** - * Metadata first so a frame's own fields always win on collision, matching - * core's `{**workflow_metadata, **data}`. - * - * Only frames carrying a `prompt_id` are stamped, which is core's own gate - * (`send_sync` checks `"prompt_id" in data`) rather than an event-name - * carve-out. `status` is exempt because it has no `prompt_id`, so the - * exemption cannot drift out of sync with the list of frame builders. - */ private emit(type: string, data: Record): void { const payload = this.metadata && 'prompt_id' in data @@ -253,14 +206,7 @@ export class ExecutionHelper { this.emit('execution_start', { prompt_id: jobId, timestamp: Date.now() }) } - /** - * Send `executing` WS event to signal which node is currently running. - * - * Shape matches the server: a node-start frame carries `display_node` - * (execution.py), the terminal `node: null` frame does not (main.py). The - * app reads `display_node || node`, so omitting it would silently exercise - * the fallback instead of the normal path. - */ + /** Send `executing` WS event to signal which node is currently running. */ executing(jobId: string, nodeId: string | null): void { this.emit('executing', { prompt_id: jobId, @@ -334,13 +280,6 @@ export class ExecutionHelper { this.emit('progress', { prompt_id: jobId, node: nodeId, value, max }) } - /** - * The `progress_state` payload for a single node in the `running` state. - * - * Exposed separately from {@link nodeRunning} so a caller that needs the two - * events as individually schedulable frames can send them itself without - * rebuilding this payload — see `SimulatedPrompt.nodeRunning`. - */ runningNodeState( jobId: string, nodeId: string, @@ -404,15 +343,7 @@ export class ExecutionHelper { this.status(0) } - /** - * Send `status` WS event to update queue count. - * - * Goes through {@link emit} like every other frame and comes out unstamped - * because it carries no `prompt_id` — core exempts it for exactly that - * reason: one server-wide socket serves every workflow, so a queue frame has - * no single owner. Routing it through `emit` is what makes the exemption - * testable; sending it directly would make any assertion about it vacuous. - */ + /** Send `status` WS event to update queue count. */ status(queueRemaining: number): void { this.emit('status', { status: { exec_info: { queue_remaining: queueRemaining } } diff --git a/browser_tests/fixtures/helpers/NodeOperationsHelper.ts b/browser_tests/fixtures/helpers/NodeOperationsHelper.ts index 61d10a556df..e5ce9a91457 100644 --- a/browser_tests/fixtures/helpers/NodeOperationsHelper.ts +++ b/browser_tests/fixtures/helpers/NodeOperationsHelper.ts @@ -49,6 +49,15 @@ export class NodeOperationsHelper { return selectedNodeIds.map(toNodeId) } + async getExecutionRender(nodeId: string) { + return this.page.evaluate((id) => { + const node = window.app!.graph.getNodeById(id) + if (!node) return null + const stroke = node.strokeStyles['running']?.call(node) + return { progress: node.progress ?? null, outlined: stroke !== undefined } + }, toNodeId(nodeId)) + } + /** * Add a node to the graph by type. * @param type - The node type (e.g. 'KSampler', 'VAEDecode') diff --git a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts index 272ae43b0f5..6a1f64f495e 100644 --- a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts +++ b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts @@ -215,11 +215,7 @@ export class Preview3DPipelineContext { } async openPersistedWorkflowFromSidebar(workflowName: string): Promise { - const tab = this.comfyPage.menu.workflowsTab - await tab.open() - await tab.getPersistedItem(workflowName).click() - await this.comfyPage.workflow.waitForWorkflowIdle(30_000) - await this.comfyPage.vueNodes.waitForNodes() + await this.comfyPage.workflow.openPersistedWorkflow(workflowName) } async assertPreview3dModelPathAndLastTime(path: string): Promise { diff --git a/browser_tests/fixtures/helpers/WorkflowHelper.ts b/browser_tests/fixtures/helpers/WorkflowHelper.ts index bc2aafd8f13..3907d23e1be 100644 --- a/browser_tests/fixtures/helpers/WorkflowHelper.ts +++ b/browser_tests/fixtures/helpers/WorkflowHelper.ts @@ -188,6 +188,14 @@ export class WorkflowHelper { }) } + async getActiveWorkflowId(): Promise { + return this.comfyPage.page.evaluate(() => { + const workflow = (window.app!.extensionManager as WorkspaceStore).workflow + .activeWorkflow + return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined + }) + } + async getActiveWorkflowActiveAppMode(): Promise { return this.comfyPage.page.evaluate(() => { return (window.app!.extensionManager as WorkspaceStore).workflow @@ -222,26 +230,18 @@ export class WorkflowHelper { }) } - /** - * `workflowStore.openWorkflow` registers the tab but never loads the graph, - * so going through the store leaves the canvas on whatever it already had. - * Clicking the sidebar entry is the path a user takes and the only one that - * puts the file's nodes on the canvas. - */ async openPersistedWorkflow(workflowName: string): Promise { const sidebar = this.comfyPage.menu.workflowsTab + const wasOpen = await sidebar.selectedTabButton.isVisible() await sidebar.open() - await sidebar.getPersistedItem(workflowName).click() - await sidebar.close() + await sidebar.getPersistedItem(workflowName, { exact: true }).click() + if (!wasOpen) await sidebar.close() await this.waitForWorkflowIdle() - await this.comfyPage.page.waitForFunction( - (name) => - ( - window.app!.extensionManager as WorkspaceStore - ).workflow.activeWorkflow?.path.endsWith(`${name}.json`) ?? false, - workflowName, - { timeout: 5000 } - ) + await expect + .poll(async () => + (await this.getActiveWorkflowPath())?.endsWith(`${workflowName}.json`) + ) + .toBe(true) await this.comfyPage.vueNodes.waitForNodes() } diff --git a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts new file mode 100644 index 00000000000..4430d743ae4 --- /dev/null +++ b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts @@ -0,0 +1,199 @@ +import { comfyExpect as expect } from '@e2e/fixtures/ComfyPage' +import { executionFixture as test } from '@e2e/fixtures/execution' +import { imageOutput } from '@e2e/fixtures/helpers/ExecutionHelper' + +const KSAMPLER_NODE = '3' +const SAVE_IMAGE_NODE = '9' +const WORKFLOW_A = 'exec_leak_a' +const WORKFLOW_B = 'exec_leak_b' +const WORKFLOW_A_COPY = 'pasted_twin' + +test.describe('cross-tab execution leak', { tag: '@ui' }, () => { + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.workflow.setupWorkflowsDirectory({ + [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', + [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json', + // The same asset under a second name: two files carrying one workflow + // id, which is what a copy-paste into a new tab produces. + [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id.json' + }) + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) + }) + + test('switching back shows the finished run, not a stuck one', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) + + simulator.play([ + running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), + running.success() + ]) + await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() + await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) + + await comfyPage.workflow.switchToTab(WORKFLOW_A) + + await expect( + comfyPage.menu.topbar.workflowTabs.getByRole('img', { name: 'Running' }) + ).toHaveCount(0) + }) + + test('the node the user is looking at does not take over a foreign run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.25, outlined: true }) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + + await comfyPage.workflow.switchToTab(WORKFLOW_A) + + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) + + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) + + test('stays clean across repeated switches during one run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + + async function switchAndAssert(step: number, arrival: number) { + await comfyPage.workflow.switchToTab(WORKFLOW_A) + simulator.play([...running.nodeRunning(KSAMPLER_NODE, step, 10)]) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: step / 10, outlined: true }) + + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect( + await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE), + `arrival ${arrival} in B must not inherit A` + ).toEqual({ progress: null, outlined: false }) + } + + await switchAndAssert(3, 1) + await switchAndAssert(7, 2) + }) + + test('stays clean while the visible workflow is queued behind the running one', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobA = await exec.run() + const running = simulator.prompt(jobA, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + + await exec.run() + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) + + test('a second open workflow sharing the workflow id does not receive the output', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const sharedId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: sharedId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A_COPY) + expect( + await comfyPage.workflow.getActiveWorkflowId(), + 'the copy must carry the same id, which is what makes this reachable' + ).toBe(sharedId) + + simulator.play([ + ...running.nodeRunning(KSAMPLER_NODE, 2, 4), + running.executed(SAVE_IMAGE_NODE, imageOutput('from-the-original.png')), + running.success() + ]) + + await expect( + comfyPage.menu.topbar + .getWorkflowTab(WORKFLOW_A) + .getByRole('img', { name: 'Completed' }) + ).toBeVisible() + + const leaked = await comfyPage.page.evaluate(() => + JSON.stringify(window.app!.nodeOutputs) + ) + expect( + leaked, + 'the original output must not land on the copy' + ).not.toContain('from-the-original.png') + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) +}) diff --git a/browser_tests/tests/workflowScopedExecution.spec.ts b/browser_tests/tests/execution/workflowScopedExecution.spec.ts similarity index 54% rename from browser_tests/tests/workflowScopedExecution.spec.ts rename to browser_tests/tests/execution/workflowScopedExecution.spec.ts index 2425ad37fa5..042c67ca3ac 100644 --- a/browser_tests/tests/workflowScopedExecution.spec.ts +++ b/browser_tests/tests/execution/workflowScopedExecution.spec.ts @@ -1,81 +1,42 @@ -import { mergeTests } from '@playwright/test' - -import { - comfyExpect as expect, - comfyPageFixture -} from '@e2e/fixtures/ComfyPage' -import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import { comfyExpect as expect } from '@e2e/fixtures/ComfyPage' +import { executionFixture as test } from '@e2e/fixtures/execution' import { BackendSimulator, duplicateFrames, interleave, swapFrames } from '@e2e/fixtures/helpers/BackendSimulator' -import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' -import { webSocketFixture } from '@e2e/fixtures/ws' -import type { WorkspaceStore } from '@e2e/types/globals' - -/** - * Execution WebSocket frames must only move the visible workflow's UI. - * - * Every scenario here is a frame script rather than a timing race: the backend - * is simulated by {@link BackendSimulator}, so "a frame from the other tab - * arrives between this tab's progress and its terminal frame" is an ordered - * list, not a sleep. See `BackendSimulator.test.ts` for the harness's own - * tests. - * - * Every scenario loads a workflow that carries an `id`. Ownership gating has - * nothing to resolve for an unsaved workflow — no id on the message's side to - * compare against, and no queue-time mapping — so it stays deliberately - * permissive there, and a foreign-frame assertion would pass for the wrong - * reason. `assets/default.json` has no id, which is why these use - * `assets/execution/workflow_with_id.json`. - */ - -const test = mergeTests(comfyPageFixture, webSocketFixture) +import { + ExecutionHelper, + imageOutput +} from '@e2e/fixtures/helpers/ExecutionHelper' const KSAMPLER_NODE = '3' const SAVE_IMAGE_NODE = '9' const FOREIGN_WORKFLOW_ID = '0199e3a3-6c01-7000-8000-ffffffffffff' -/** Carries `id`, unlike `assets/default.json`. See the describe block comment. */ const WORKFLOW_WITH_ID = 'execution/workflow_with_id' -function imageOutput(filename: string) { - return { - images: [{ filename, subfolder: '', type: 'output' }] - } -} - -/** The id the gate compares against; undefined for an unsaved workflow. */ -async function activeWorkflowId( - comfyPage: ComfyPage -): Promise { - return comfyPage.page.evaluate(() => { - const workflow = (window.app!.extensionManager as WorkspaceStore).workflow - .activeWorkflow - return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined - }) -} - test.describe('workflow-scoped execution', { tag: '@ui' }, () => { + let workflowId: string + test.beforeEach(async ({ comfyPage }) => { await comfyPage.workflow.loadWorkflow(WORKFLOW_WITH_ID) await comfyPage.appMode.enterAppModeWithInputs([[KSAMPLER_NODE, 'seed']]) await expect(comfyPage.appMode.linearWidgets).toBeVisible() + const loadedWorkflowId = await comfyPage.workflow.getActiveWorkflowId() + if (!loadedWorkflowId) throw new Error('loaded workflow has no id') + workflowId = loadedWorkflowId }) test('renders the visible workflow own frames', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) const jobId = await exec.run() await comfyPage.nextFrame() - const mine = simulator.prompt(jobId, { - workflowId: await activeWorkflowId(comfyPage) - }) + const mine = simulator.prompt(jobId, { workflowId }) simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) await expect( @@ -92,13 +53,9 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { test('an output from another workflow does not land on the visible tab', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() - const jobId = await exec.run() await comfyPage.nextFrame() const mine = simulator.prompt(jobId, { workflowId }) @@ -120,13 +77,9 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { test('interleaved prompts keep the visible tab on its own run', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() - const jobId = await exec.run() await comfyPage.nextFrame() const mine = simulator.prompt(jobId, { workflowId }) @@ -134,9 +87,6 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { workflowId: FOREIGN_WORKFLOW_ID }) - // Seed 2 puts the foreign start inside the visible run, which is the - // ordering that leaks without the gate. Seed 11 put it before the visible - // run even started, so the old order could not tear anything down. simulator.play( interleave( [ @@ -156,55 +106,44 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { ) await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) - // Which one was counted matters: a count of one is also satisfied by the - // foreign output arriving instead of this workflow's own. await expect( comfyPage.appMode.outputHistory.imageOutputs.first() - ).not.toHaveAttribute('src', /theirs\.png/) + ).toHaveAttribute('src', /mine\.png/) }) test('an output that arrives after the terminal frame is ignored', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) const jobId = await exec.run() await comfyPage.nextFrame() - const workflowId = await activeWorkflowId(comfyPage) const mine = simulator.prompt(jobId, { workflowId }) const script = [ mine.start(), mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), mine.success() ] - const label = workflowId ?? jobId + const label = workflowId simulator.play( swapFrames(script, `${label}:executed`, `${label}:execution_success`) ) await comfyPage.nextFrame() - // A single WebSocket delivers in send order, so this ordering only happens - // on a reconnect replay. The terminal frame has already torn down the run, - // so the late output is dropped rather than applied to a finished job — - // outputs for a completed job come back through history instead. Pinned - // here so a future change to the teardown is a deliberate decision. await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(0) await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) }) test('a repeated terminal frame does not duplicate the output', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) const jobId = await exec.run() await comfyPage.nextFrame() - const workflowId = await activeWorkflowId(comfyPage) const mine = simulator.prompt(jobId, { workflowId }) const script = [ mine.start(), @@ -212,27 +151,28 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { mine.success() ] - simulator.play( - duplicateFrames(script, `${workflowId ?? jobId}:execution_success`) - ) + simulator.play(duplicateFrames(script, `${workflowId}:execution_success`)) await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) }) test('a backend that sends no workflow id behaves as before', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) const jobId = await exec.run() await comfyPage.nextFrame() - // No workflowId: every frame is emitted the way a core build without - // `workflow_metadata` emits it. This is the control for the 90% of users - // whose server has not been updated. const legacy = simulator.prompt(jobId) - simulator.play([legacy.start(), ...legacy.nodeRunning(KSAMPLER_NODE, 1, 4)]) + const foreign = simulator.prompt('foreign-job') + simulator.play([ + legacy.start(), + foreign.start(), + ...foreign.nodeRunning(KSAMPLER_NODE, 3, 4), + foreign.success(), + ...legacy.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) await expect( comfyPage.appMode.outputHistory.inProgressItems.first() @@ -249,13 +189,9 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { test('another workflow finishing does not end the visible run', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - expect(workflowId, 'the loaded workflow must carry an id').toBeDefined() - const jobId = await exec.run() await comfyPage.nextFrame() const mine = simulator.prompt(jobId, { workflowId }) @@ -268,9 +204,6 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { comfyPage.appMode.outputHistory.inProgressItems.first() ).toBeVisible() - // The other tab's run ends. Its terminal `executing: null` must not tear - // down this tab's run — the frame carries only a node id on the public - // event, so this is the path that needed the raw message to gate. simulator.play([ foreign.start(), foreign.executing(null), @@ -292,13 +225,11 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { test('a reconnect mid-run does not duplicate the output', async ({ comfyPage, getWebSocket, - nextWebSocket + nextWebSocket, + execution: exec, + backendSimulator: simulator }) => { const ws = await getWebSocket() - const exec = new ExecutionHelper(comfyPage, ws) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - const jobId = await exec.run() await comfyPage.nextFrame() const mine = simulator.prompt(jobId, { workflowId }) @@ -307,8 +238,8 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { comfyPage.appMode.outputHistory.inProgressItems.first() ).toBeVisible() - // Arm the waiter before dropping the socket, then replay the run's frames - // on the reconnected socket the way a catch-up would. + simulator.play([mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png'))]) + const reconnected = nextWebSocket() await ws.close() const nextRoute = await reconnected @@ -324,12 +255,9 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { test('a binary preview for another workflow does not reach the visible tab', async ({ comfyPage, - getWebSocket + execution: exec, + backendSimulator: simulator }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowId = await activeWorkflowId(comfyPage) - const jobId = await exec.run() await comfyPage.nextFrame() const mine = simulator.prompt(jobId, { workflowId }) @@ -342,18 +270,6 @@ test.describe('workflow-scoped execution', { tag: '@ui' }, () => { simulator.play([foreign.latentPreview(KSAMPLER_NODE)]) await comfyPage.nextFrame() - // A leaked preview surfaces as a latent preview, not an image output, so - // asserting on imageOutputs held whatever the gate did. Then this - // workflow's own preview is the barrier: it proves previews render at all - // and that the foreign one was processed and discarded rather than still - // in flight. - await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(0) - expect( - await comfyPage.page.evaluate( - () => Object.values(window.app!.nodePreviewImages).flat().length - ) - ).toBe(0) - simulator.play([mine.latentPreview(KSAMPLER_NODE)]) await comfyPage.nextFrame() await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(1) diff --git a/browser_tests/tests/twoTabExecutionLeak.spec.ts b/browser_tests/tests/twoTabExecutionLeak.spec.ts deleted file mode 100644 index 59ec123bdfe..00000000000 --- a/browser_tests/tests/twoTabExecutionLeak.spec.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { mergeTests } from '@playwright/test' - -import { - comfyExpect as expect, - comfyPageFixture -} from '@e2e/fixtures/ComfyPage' -import type { ComfyPage } from '@e2e/fixtures/ComfyPage' -import { BackendSimulator } from '@e2e/fixtures/helpers/BackendSimulator' -import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' -import { webSocketFixture } from '@e2e/fixtures/ws' -import { toNodeId } from '@/types/nodeId' -import type { WorkspaceStore } from '@e2e/types/globals' - -/** - * The reported bug, with two real workflow tabs rather than simulated ones. - * - * `workflowScopedExecution.spec.ts` injects frames tagged with a foreign - * workflow id, which exercises the gate but takes the frontend's own tab - * bookkeeping on trust. These tests open two persisted workflows, queue from - * one, switch to the other, and assert the second tab is untouched — the - * actual sequence users reported. - */ - -const test = mergeTests(comfyPageFixture, webSocketFixture) - -const KSAMPLER_NODE = '3' -const SAVE_IMAGE_NODE = '9' -const WORKFLOW_A = 'exec_leak_a' -const WORKFLOW_B = 'exec_leak_b' -const WORKFLOW_A_COPY = 'pasted_twin' - -function imageOutput(filename: string) { - return { images: [{ filename, subfolder: '', type: 'output' }] } -} - -async function activeWorkflowId( - comfyPage: ComfyPage -): Promise { - return comfyPage.page.evaluate(() => { - const workflow = (window.app!.extensionManager as WorkspaceStore).workflow - .activeWorkflow - return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined - }) -} - -async function canvasNodeRender(comfyPage: ComfyPage, nodeId: string) { - return comfyPage.page.evaluate((id) => { - const node = window.app!.graph.getNodeById(id) - if (!node) return null - const stroke = node.strokeStyles['running']?.call(node) - return { progress: node.progress ?? null, outlined: stroke !== undefined } - }, toNodeId(nodeId)) -} - -test.describe('cross-tab execution leak', { tag: '@ui' }, () => { - test.beforeEach(async ({ comfyPage }) => { - await comfyPage.workflow.setupWorkflowsDirectory({ - [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', - [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json', - // The same asset under a second name: two files carrying one workflow - // id, which is what a copy-paste into a new tab produces. - [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id.json' - }) - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) - }) - - test('switching back shows the finished run, not a stuck one', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowAId = await activeWorkflowId(comfyPage) - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: workflowAId }) - simulator.play([ - running.start(), - ...running.nodeRunning(KSAMPLER_NODE, 1, 4) - ]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) - - // Terminal frames arrive while tab A is in the background. That is the - // stuck-progress report: the run must reach its end, not stay Running. - simulator.play([ - running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), - running.success() - ]) - await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() - await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) - - await comfyPage.workflow.switchToTab(WORKFLOW_A) - - // An active tab deliberately carries no status badge (WorkflowTab.vue:235 - // — the user is already looking at it), so the absence of Completed here - // proves nothing. What must hold is that nothing anywhere still reads as - // running once the run has ended. - await expect( - comfyPage.menu.topbar.workflowTabs.getByRole('img', { name: 'Running' }) - ).toHaveCount(0) - }) - - test('the node the user is looking at does not take over a foreign run', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowAId = await activeWorkflowId(comfyPage) - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: workflowAId }) - simulator.play([ - running.start(), - ...running.nodeRunning(KSAMPLER_NODE, 1, 4) - ]) - await comfyPage.nextFrame() - - // Both workflows own a KSampler numbered 3, which is what makes this - // reachable: node progress is keyed by a locator resolved against the - // graph that happens to be in front, so A's entry lands on B's node. - await expect - .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) - .toEqual({ progress: 0.25, outlined: true }) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - await comfyPage.nextFrame() - - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false - }) - - // A keeps running out of sight; none of it may surface on B. - simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) - await comfyPage.nextFrame() - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false - }) - - await comfyPage.workflow.switchToTab(WORKFLOW_A) - await comfyPage.nextFrame() - - // Returning to A shows where its run actually got to, not where it was - // when the user left. - await expect - .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) - .toEqual({ progress: 0.75, outlined: true }) - }) - - test('stays clean across repeated switches during one run', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowAId = await activeWorkflowId(comfyPage) - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: workflowAId }) - simulator.play([running.start()]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - await comfyPage.nextFrame() - - // Round 2 of the GPU QA: the first arrival in B was clean and later ones - // were not, so one switch proves nothing here. - for (const [index, step] of [3, 5, 7, 9].entries()) { - await comfyPage.workflow.switchToTab(WORKFLOW_A) - await comfyPage.nextFrame() - simulator.play([...running.nodeRunning(KSAMPLER_NODE, step, 10)]) - await comfyPage.nextFrame() - await expect - .poll(() => canvasNodeRender(comfyPage, KSAMPLER_NODE)) - .toEqual({ progress: step / 10, outlined: true }) - - await comfyPage.workflow.switchToTab(WORKFLOW_B) - await comfyPage.nextFrame() - expect( - await canvasNodeRender(comfyPage, KSAMPLER_NODE), - `arrival ${index + 1} in B must not inherit A` - ).toEqual({ progress: null, outlined: false }) - } - }) - - test('stays clean while the visible workflow is queued behind the running one', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const workflowAId = await activeWorkflowId(comfyPage) - - const jobA = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobA, { workflowId: workflowAId }) - simulator.play([ - running.start(), - ...running.nodeRunning(KSAMPLER_NODE, 1, 4) - ]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - await comfyPage.nextFrame() - - // B is queued behind A and has not started. QA saw A's progress land on - // B's KSampler the moment B was queued. - await exec.run() - await comfyPage.nextFrame() - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false - }) - - simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) - await comfyPage.nextFrame() - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false - }) - }) - - // QA on the core PR thread: copy a workflow, paste it into a new tab, run - // both. `ensureWorkflowId` keeps the id already in the pasted json, so the - // two tabs share one workflow id while being different open workflows, and - // the node ids are identical too. Outputs landed on the wrong tab. - test('a second open workflow sharing the workflow id does not receive the output', async ({ - comfyPage, - getWebSocket - }) => { - const exec = new ExecutionHelper(comfyPage, await getWebSocket()) - const simulator = new BackendSimulator(exec) - const sharedId = await activeWorkflowId(comfyPage) - - const jobId = await exec.run() - await comfyPage.nextFrame() - const running = simulator.prompt(jobId, { workflowId: sharedId }) - simulator.play([running.start()]) - - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A_COPY) - await comfyPage.nextFrame() - expect( - await activeWorkflowId(comfyPage), - 'the copy must carry the same id, which is what makes this reachable' - ).toBe(sharedId) - - simulator.play([ - ...running.nodeRunning(KSAMPLER_NODE, 2, 4), - running.executed(SAVE_IMAGE_NODE, imageOutput('from-the-original.png')), - running.success() - ]) - - // A positive marker before the absence checks. `WebSocketRoute.send` does - // not await page-side handling and progress is coalesced per frame, so - // reading straight after `play` can assert on state the frames have not - // reached yet, and a gate that wrongly applied them would still pass. The - // original's own tab reaching Completed is driven by the same terminal - // frame, so polling it proves the batch was processed. - await expect( - comfyPage.menu.topbar - .getWorkflowTab(WORKFLOW_A) - .getByRole('img', { name: 'Completed' }) - ).toBeVisible() - - const leaked = await comfyPage.page.evaluate(() => - JSON.stringify(window.app!.nodeOutputs) - ) - expect( - leaked, - 'the original output must not land on the copy' - ).not.toContain('from-the-original.png') - expect(await canvasNodeRender(comfyPage, KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false - }) - }) -}) From 40c750e3ef0e2df30172f3d2812748ef3fefb6a9 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:49:17 +0000 Subject: [PATCH 41/52] test(execution): close store review gaps Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210692227 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210702105 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210764228 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210770492 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4222232781 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4222232833 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425863 --- .../executionStore.terminalRecovery.test.ts | 65 +++---- src/stores/executionStore.ts | 105 +---------- .../executionStore.workflowGating.test.ts | 165 +++++++----------- src/stores/queueStore.test.ts | 4 - 4 files changed, 88 insertions(+), 251 deletions(-) diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts index c71712c2809..bae369a0342 100644 --- a/src/stores/executionStore.terminalRecovery.test.ts +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -1,22 +1,19 @@ import { fromPartial } from '@total-typescript/shoehorn' import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' +import type { + ExecutingWsMessage, + ExecutionErrorWsMessage, + ExecutionStartWsMessage, + NodeProgressState, + NotificationWsMessage, + ProgressStateWsMessage +} from '@/platform/remote/comfyui/execution/types' import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import { api } from '@/scripts/api' import { useExecutionStore } from '@/stores/executionStore' -/** - * Recovery from a dropped terminal WebSocket frame. - * - * `execution_success` / `execution_error` / `execution_interrupted` are - * broadcast once and never retried, so a single dropped frame leaves node - * progress pinned forever — the user sees a node stuck mid-run with no way to - * clear it. Queue polling is the only authority that can notice: the backend - * reports the job in history while the frontend still holds progress for it. - */ - const { mockRemoveTextPreview } = await vi.hoisted(async () => ({ mockRemoveTextPreview: vi.fn() })) @@ -33,6 +30,12 @@ vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ })) type EventHandler = (event: CustomEvent) => void +interface EventPayloads { + execution_error: ExecutionErrorWsMessage + execution_start: ExecutionStartWsMessage + notification: NotificationWsMessage + progress_state: ProgressStateWsMessage +} const apiEventHandlers = new Map() vi.mock(import('@/scripts/api'), () => ({ @@ -75,14 +78,9 @@ function workflow(id: string, path: string): LoadedComfyWorkflow { const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') -/** - * `executing` is dispatched with a bare node id for extension compatibility, so - * the store reads the ids off the raw message api recorded. Mirror that rather - * than passing an object as the detail, which would pass for the wrong reason. - */ function fireExecuting( node: string | null, - ids: { prompt_id: string; workflow_id?: string } + ids: Pick ) { api.lastExecutingMessage = { node, ...ids } const handler = apiEventHandlers.get('executing') @@ -90,7 +88,10 @@ function fireExecuting( handler(new CustomEvent('executing', { detail: node })) } -function fire(event: string, detail: Record) { +function fire( + event: K, + detail: EventPayloads[K] +) { const handler = apiEventHandlers.get(event) if (!handler) throw new Error(`${event} handler not bound`) handler(new CustomEvent(event, { detail })) @@ -140,7 +141,6 @@ describe('executionStore terminal-job recovery', () => { store.bindExecutionEvents() }) - /** A job of the active workflow, mid-run, whose terminal frame never came. */ function stuckActiveJob(jobId = 'job-a') { store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) fire('execution_start', { @@ -180,8 +180,6 @@ describe('executionStore terminal-job recovery', () => { it('ignores a job that is neither running nor in history yet', () => { const jobId = stuckActiveJob() - // Submitted but not yet visible in either list: evicting here would blank - // a run that is about to report progress. store.reconcileTerminalJobs(new Set(), new Set()) expect(store.nodeProgressStates['1']?.state).toBe('running') @@ -206,8 +204,6 @@ describe('executionStore terminal-job recovery', () => { }) it('clears a stuck job belonging to another workflow', () => { - // The other tab's job is tracked per-job even though it never wrote the - // shared mirror, so it needs evicting too or that tab stays stuck. store.registerJobWorkflowIdMapping('job-b', WORKFLOW_B_ID) fire('progress_state', { prompt_id: 'job-b', @@ -257,7 +253,6 @@ describe('executionStore terminal-job recovery', () => { }) it('still recovers when the terminal frame was dropped for a legacy backend', () => { - // No workflow_id anywhere: recovery must not depend on the new field. const jobId = 'job-legacy' store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) fire('execution_start', { prompt_id: jobId, timestamp: 1 }) @@ -271,8 +266,6 @@ describe('executionStore terminal-job recovery', () => { expect(store.nodeProgressStates['1']).toBeUndefined() }) - /** As above, but queued through storeJob, which is what associates the job - * with its workflow. The status and text-preview paths both read that. */ function stuckQueuedJob(jobId = 'job-queued') { store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) store.storeJob({ @@ -301,16 +294,10 @@ describe('executionStore terminal-job recovery', () => { store.reconcileTerminalJobs(new Set(), new Set([jobId])) - // handleExecutionStart set this and nothing else resets it, so without the - // release the tab keeps claiming it is running for the rest of the session. expect(store.getWorkflowStatus(workflowA)).toBeUndefined() }) it('fires onJobReset while the job record its consumer reads still exists', () => { - // useProgressTextPreviews.clearTextPreviews reads the job's node list off - // the queuedJobs entry to find the widgets to remove, so triggering after - // the delete would hand it nothing and silently remove no widget. Order is - // the whole point here. const jobId = stuckQueuedJob() const seen: string[][] = [] store.onJobReset((job) => { @@ -325,8 +312,11 @@ describe('executionStore terminal-job recovery', () => { ).toEqual([['1']]) }) - it('releases a background job records when it ends in an error', () => { - const jobId = 'job-bg-error' + it.for([ + { name: 'node error', nodeId: '1' }, + { name: 'service-level error', nodeId: null } + ])('releases background job records after a $name', ({ nodeId }) => { + const jobId = `job-bg-error-${nodeId ?? 'service'}` store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) store.storeJob({ nodes: ['1'], @@ -345,14 +335,11 @@ describe('executionStore terminal-job recovery', () => { workflow_id: WORKFLOW_B_ID, nodes: { '1': runningNode(jobId, '1') } }) - expect(store.nodeProgressStatesByJob[jobId]).toBeDefined() - - // A is in front, so this error belongs to a background tab. It is still a - // finished job, so its own records have to go, like the success path. fire('execution_error', { prompt_id: jobId, workflow_id: WORKFLOW_B_ID, - node_id: '1', + timestamp: 2, + node_id: nodeId, node_type: 'TestNode', exception_type: 'RuntimeError', exception_message: 'boom', diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index a6731d6f085..223ccc93129 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -498,10 +498,6 @@ export const useExecutionStore = defineStore('execution', () => { executionStartedAt: queuedJobs.value[jobId].executionStartedAt }) - // Only adopt as the global active job and clear shared UI state when the - // starting job belongs to the active workflow. Otherwise a job started - // from another tab would steal activeJobId and clobber the active tab's - // execution UI. if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) return executionIdToLocatorCache.clear() @@ -533,10 +529,6 @@ export const useExecutionStore = defineStore('execution', () => { if (workflow) clearWorkflowStatus(workflow) clearInitializationByJobId(jobId) if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { - // The job is finished either way, so its own records have to be released - // or they leak for the lifetime of the session. Only the shared UI state - // below belongs to the visible tab. Text previews stay gated: they are - // keyed by node, and the visible graph may hold the same node ids. releaseFinishedJobRecords(jobId) return } @@ -554,9 +546,6 @@ export const useExecutionStore = defineStore('execution', () => { const jobId = e.detail.prompt_id pendingExecutionErrorsByJobId.delete(jobId) clearInitializationByJobId(jobId) - // Per-workflow status is keyed by this job's own workflow, so it is set - // for every job — a background tab must still show Completed. Only the - // shared execution state below is gated. setWorkflowStatus(jobId, { status: 'completed', endTime: performance.now() @@ -577,10 +566,6 @@ export const useExecutionStore = defineStore('execution', () => { } } if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { - // Finished either way, so release this job's own records or they leak for - // the session. Only the shared UI state below belongs to the visible tab, - // and text previews stay gated because they are keyed by node and the - // visible graph may hold the same node ids. releaseFinishedJobRecords(jobId) return } @@ -588,10 +573,7 @@ export const useExecutionStore = defineStore('execution', () => { } function handleExecuting(e: CustomEvent): void { - // The event detail is just the node id, for extension compatibility, so - // the ids come from the raw message. Without this, the final - // `executing: null` of a job started in another workflow tab would clear - // the visible tab's active job and its node progress. + // Bare node id for extensions; the ids come from api.lastExecutingMessage. const raw = api.lastExecutingMessage if (raw && !belongsToActiveWorkflow(raw.prompt_id, raw.workflow_id)) { return @@ -658,13 +640,6 @@ export const useExecutionStore = defineStore('execution', () => { progressStateBatch.cancel() } - /** - * Revoke previews for nodes that just started executing. - * - * Uses the *actual* node id rather than the display node id intentionally, - * so the preview is not cleared every time a new node inside an expanded - * graph starts. - */ function revokeStartedNodePreviews( nodes: Record, previousForJob: Record @@ -678,7 +653,6 @@ export const useExecutionStore = defineStore('execution', () => { } } - /** Mirror the executing node's progress for backwards compatibility. */ function mirrorExecutingNodeProgress( nodes: Record ) { @@ -713,8 +687,6 @@ export const useExecutionStore = defineStore('execution', () => { } evictOldProgressJobs() - // Per-job state is recorded for every workflow; only the active workflow - // writes the shared mirror the canvas reads. if (!isActiveWorkflowMessage) return nodeProgressStates.value = nodes mirrorExecutingNodeProgress(nodes) @@ -759,24 +731,6 @@ export const useExecutionStore = defineStore('execution', () => { return owner === active ? 'active' : 'foreign' } - /** - * Determines whether a WebSocket execution message belongs to the currently - * active workflow tab, so a job started in another open workflow cannot leak - * its progress into the one on screen. - * - * Resolution order, strongest first: - * 1. {@link jobIdToWorkflowInstanceId}, the id of the open tab that queued it. - * 2. {@link jobIdToSessionWorkflowPath}, unique per open workflow. - * 3. the graph id, from the message or {@link jobIdToWorkflowId}. - * - * A workflow id names a workflow, not an open tab, and several open tabs can - * carry the same one: `ensureWorkflowId` keeps whatever id the json already - * has, so copying a workflow into a new tab duplicates it, and loading one - * can leave both a temporary and a persisted entry holding it. So the id is - * the weakest signal here, not the strongest, and anything unique per tab has - * to be consulted first. QA hit this with three unsaved copies of the default - * graph, where every tab rendered every other tab's outputs. - */ function resolveWorkflowOwnership( jobId: JobId, rawMessageWorkflowId?: string @@ -804,11 +758,6 @@ export const useExecutionStore = defineStore('execution', () => { return compareWorkflowOwner(ownerId, activeId) } - /** - * Whether a message belongs on the visible workflow. Unresolvable messages - * use the legacy active-prompt guard, preserving single-tab behavior without - * letting an unknown job mutate another active run. - */ function belongsToActiveWorkflow( jobId: JobId | undefined, workflowId?: string @@ -946,10 +895,6 @@ export const useExecutionStore = defineStore('execution', () => { executionErrorStore.recordExecutionError(detail, runErrorKey) executionErrorStore.showExecutionError(detail, runErrorKey) clearInitializationByJobId(detail.prompt_id) - // Only the active workflow's error resets the shared execution state; the - // dialog is already scoped by the run error key inside showExecutionError. - // A background job is still finished, so release its own records either way - // or they survive the session, same as the success and interrupt paths. if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { releaseFinishedJobRecords(detail.prompt_id) return @@ -1077,30 +1022,14 @@ export const useExecutionStore = defineStore('execution', () => { return entries[0].prompt_id } - /** - * Evict per-job execution artifacts for a job that has reached a terminal - * state, without disturbing state owned by a different running job. - * - * Unlike {@link resetExecutionState} this is safe for any jobId, including - * one that is not {@link activeJobId}. It is the recovery path for a dropped - * terminal WebSocket frame, which would otherwise leave node progress pinned - * forever: the backend broadcasts `execution_success` once and never retries. - * Idempotent. - */ function evictTerminalJob(jobId: JobId) { - // Read before releasing: the release nulls activeJobId when it names this - // job, so asking afterwards always says no. const isActive = activeJobId.value === jobId - // A dropped terminal frame left the workflow marked running forever, since - // handleExecutionStart set that and nothing else resets it. const evictedWorkflow = jobIdToWorkflow.get(jobId) if (evictedWorkflow) clearWorkflowStatus(evictedWorkflow) releaseFinishedJobRecords(jobId) clearInitializationByJobId(jobId) - // Only clear the shared mirror when it still belongs to the evicted job, - // otherwise evicting an old job would blank a live run's progress. if (isActive || mirrorOwnerJobId() === jobId) { nodeProgressStates.value = {} executionIdToLocatorCache.clear() @@ -1116,25 +1045,10 @@ export const useExecutionStore = defineStore('execution', () => { } } - /** - * Re-point the shared mirror at the now-active workflow's own job. - * - * Gating stops a background workflow's frames from *writing* the mirror, but - * whatever was written while that workflow was in front stays there — and - * `nodeLocationProgressStates` resolves node ids against the *currently* - * active graph, so two workflows that share a node id will show the stale - * entry on the newly visible node. Replay the active workflow's own per-job - * progress instead, or clear the mirror when it has no running job. - */ function reconcileMirrorForActiveWorkflow() { const activeWorkflow = workflowStore.activeWorkflow if (!activeWorkflow) return - // Ownership has exactly one definition, and it is not the graph id: several - // open tabs can carry the same one. Matching here on the graph id is the bug - // this feature already shipped once, so defer to the same resolver the - // handlers use, and require ownership to be resolvable rather than adopting - // a job we cannot place. const matchedJobId = Object.keys(nodeProgressStatesByJob.value).findLast( (jobId) => resolveWorkflowOwnership(jobId) === 'active' ) @@ -1143,15 +1057,7 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStates.value = nodeProgressStatesByJob.value[matchedJobId] ?? {} executionIdToLocatorCache.clear() - // handleExecutionStart deliberately does not adopt a job whose tab is in - // the background, so nothing had adopted it by the time the user came - // back: the tab showed node progress while isIdle stayed true, which left - // the title and overlay reading idle and made handleExecuted drop the - // completion frames. Per-job progress still existing means the job has - // not reached a terminal frame, so it is the one this tab should show. - if (activeJobId.value !== matchedJobId) { - activeJobId.value = matchedJobId - } + activeJobId.value = matchedJobId if (_executingNodeProgress.value?.prompt_id !== matchedJobId) { _executingNodeProgress.value = null } @@ -1171,9 +1077,6 @@ export const useExecutionStore = defineStore('execution', () => { * Reconcile tracked per-job state against the backend's authoritative job * sets. A job the backend reports as terminal but which still holds progress * state lost its terminal frame, so evict it. - * - * @param activeJobIds jobs the backend reports as Running or Pending - * @param terminalJobIds jobs the backend reports in history */ function reconcileTerminalJobs( activeJobIds: Set, @@ -1221,10 +1124,6 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStatesByJob.value = byJob jobIdToWorkflow.delete(jobId) useJobPreviewStore().clearPreview(jobId) - // This runs for a job whose tab is not in front, so activeJobId naming it - // is stale: the gated terminal branches return early and the gated - // `executing: null` is dropped, which left the actionbar, favicon and title - // non-idle with nothing running until the next queue poll evicted it. if (activeJobId.value === jobId) activeJobId.value = null } diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 3dbabc3ed88..1ce70407cb3 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -2,7 +2,19 @@ import { fromPartial } from '@total-typescript/shoehorn' import { assert, beforeEach, describe, expect, it, vi } from 'vitest' import { nextTick } from 'vue' -import type { NodeProgressState } from '@/platform/remote/comfyui/execution/types' +import type { + ExecutedWsMessage, + ExecutingWsMessage, + ExecutionCachedWsMessage, + ExecutionErrorWsMessage, + ExecutionInterruptedWsMessage, + ExecutionStartWsMessage, + ExecutionSuccessWsMessage, + NodeProgressState, + NotificationWsMessage, + ProgressStateWsMessage, + ProgressWsMessage +} from '@/platform/remote/comfyui/execution/types' import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import { api } from '@/scripts/api' @@ -10,26 +22,22 @@ import { useExecutionStore } from '@/stores/executionStore' import { useNodeOutputStore } from '@/stores/nodeOutputStore' import { zeroUuid } from '@/utils/uuid' -/** - * Workflow-ownership gating for execution WebSocket messages. - * - * Core stamps `workflow_id` on every JSON frame of a running prompt, so a frame - * that belongs to another open workflow tab must not write the shared "current - * execution" state. Three resolution paths have to keep working, because a - * frontend talks to backends of three different vintages: - * - * - **modern**: the frame carries `workflow_id` - * - **legacy**: no `workflow_id` anywhere — ownership resolves from the - * mapping registered when the job was queued, or from the session path - * - **unresolvable**: neither is available, and behaviour must stay exactly as - * it was before gating existed (permissive) - */ - vi.mock(import('@/composables/useAppMode')) vi.mock(import('@/platform/telemetry')) vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) type EventHandler = (event: CustomEvent) => void +interface EventPayloads { + executed: ExecutedWsMessage + execution_cached: ExecutionCachedWsMessage + execution_error: ExecutionErrorWsMessage + execution_interrupted: ExecutionInterruptedWsMessage + execution_start: ExecutionStartWsMessage + execution_success: ExecutionSuccessWsMessage + notification: NotificationWsMessage + progress: ProgressWsMessage + progress_state: ProgressStateWsMessage +} const apiEventHandlers = new Map() vi.mock(import('@/scripts/api'), () => ({ @@ -66,8 +74,6 @@ function workflow(id: string, path: string): LoadedComfyWorkflow { activeState: { id }, initialState: { id }, path, - // Every open tab carries its own instanceId in the running app, and it is - // the only identifier that is unique per tab rather than per workflow. instanceId: `instance-${path}` }) } @@ -75,12 +81,6 @@ function workflow(id: string, path: string): LoadedComfyWorkflow { const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') -/** - * Copy a workflow, open a new tab and paste it: `ensureWorkflowId` keeps the - * id already in the pasted json, so both tabs carry the same workflow id while - * being different open workflows. Reported by QA on the core PR thread, with - * three unsaved copies of the default graph. - */ const workflowACopy = workflow(WORKFLOW_A_ID, 'workflows/a copy.json') /** @@ -89,7 +89,7 @@ const workflowACopy = workflow(WORKFLOW_A_ID, 'workflows/a copy.json') */ function fireExecuting( node: string | null, - ids: { prompt_id: string; workflow_id?: string } + ids: Pick ) { api.lastExecutingMessage = { node, ...ids } const handler = apiEventHandlers.get('executing') @@ -97,7 +97,10 @@ function fireExecuting( handler(new CustomEvent('executing', { detail: node })) } -function fire(event: string, detail: Record) { +function fire( + event: K, + detail: EventPayloads[K] +) { const handler = apiEventHandlers.get(event) if (!handler) throw new Error(`${event} handler not bound`) handler(new CustomEvent(event, { detail })) @@ -155,7 +158,6 @@ describe('executionStore workflow gating', () => { store.bindExecutionEvents() }) - /** Queue a job from `wf` so the id mapping exists, as storeJob does. */ function queueLegacyJobFrom(jobId: string, wf: LoadedComfyWorkflow) { store.storeJob({ nodes: ['1'], @@ -259,12 +261,11 @@ describe('executionStore workflow gating', () => { const ids = own ? { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID } : { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID } - const detail = - event === 'executed' - ? { ...ids, node: '1', display_node: '1', output: {} } - : { ...ids, nodes: ['1'], timestamp: 2 } - - fire(event, detail) + if (event === 'executed') { + fire('executed', { ...ids, node: '1', display_node: '1', output: {} }) + } else { + fire('execution_cached', { ...ids, nodes: ['1'], timestamp: 2 }) + } expect(store.activeJob?.nodes['1']).toBe(own ? true : undefined) }) @@ -318,9 +319,6 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') }) - // Regression: gating the whole handler hid the completed badge on a - // background tab, which workflowTabStatus.spec.ts caught on CI. Per-workflow - // status is keyed by the job's own workflow and is never shared state. it('still marks another workflow completed on its own tab', () => { queueJobFrom('job-b', workflowB) fire('execution_start', { @@ -340,6 +338,24 @@ describe('executionStore workflow gating', () => { expect(store.activeJobId).toBe('job-a') }) + it('records a background completion only on its owning workflow', () => { + queueJobFrom('job-b', workflowB) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.getWorkflowStatus(workflowA)).toBeUndefined() + expect(store.getWorkflowStatus(workflowB)).toBe('completed') + }) + it('execution_success from the active workflow clears it', () => { fire('execution_success', { prompt_id: 'job-a', @@ -388,6 +404,7 @@ describe('executionStore workflow gating', () => { fire('execution_interrupted', { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID, + timestamp: 2, synthetic: true, reason: 'Job cancelled by user' }) @@ -401,8 +418,6 @@ describe('executionStore workflow gating', () => { it('execution_error from another workflow leaves active state alone but clears its initializing flag', () => { queueJobFrom('job-b', workflowB) - // Without this the job is never initializing, so the assertion below - // passes whatever the code does. fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) expect(store.isJobInitializing('job-b')).toBe(true) @@ -414,7 +429,8 @@ describe('executionStore workflow gating', () => { node_type: 'TestNode', exception_message: 'fail', exception_type: 'Error', - traceback: [] + traceback: [], + executed: [] }) expect(store.activeJobId).toBe('job-a') @@ -430,7 +446,8 @@ describe('executionStore workflow gating', () => { node_type: 'TestNode', exception_message: 'fail', exception_type: 'Error', - traceback: [] + traceback: [], + executed: [] }) expect(store.activeJobId).toBe('job-a') @@ -448,7 +465,8 @@ describe('executionStore workflow gating', () => { exception_message: 'Payment Required: Please add credits to your account to use this node.', exception_type: 'InsufficientFundsError', - traceback: [] + traceback: [], + executed: [] }) expect(store.activeJobId).toBe('job-a') @@ -670,9 +688,6 @@ describe('executionStore workflow gating', () => { it('stays permissive when ownership cannot be resolved at all', () => { useWorkflowStore().activeWorkflow = workflowA - // No workflow_id on the frame and no mapping for this job: the job could - // have been queued by another browser session, and single-tab behaviour - // must not regress. fire('progress_state', { prompt_id: 'job-from-elsewhere', nodes: { '1': nodeState('job-from-elsewhere', '1', 'running', 5) } @@ -696,8 +711,6 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-b', workflowB) - // Core omits the field entirely when unknown, never sends "" — but a - // third-party server might, and an empty id must not read as "mine". fire('progress_state', { prompt_id: 'job-b', workflow_id: '', @@ -709,8 +722,6 @@ describe('executionStore workflow gating', () => { it('gates on workflow_id even when the job mapping disagrees', () => { useWorkflowStore().activeWorkflow = workflowA - // Mapping says workflow A, the frame says workflow B. The frame wins: - // the server is the authority on which workflow produced the prompt. store.registerJobWorkflowIdMapping('job-x', WORKFLOW_A_ID) fire('progress_state', { @@ -733,9 +744,6 @@ describe('executionStore workflow gating', () => { }) }) - // The event detail is a bare node id for extension compatibility, so this - // path can only gate via the raw message api records. Before that, the - // other tab's final `executing: null` cleared the visible tab's run. it('does not let another workflow clear the visible active job', () => { fireExecuting(null, { prompt_id: 'job-b', @@ -769,8 +777,6 @@ describe('executionStore workflow gating', () => { }) it('still works when the raw message was never recorded', () => { - // Defensive: an `executing` dispatched by an extension or a test rather - // than by the socket has no raw message, and must behave as before. const handler = apiEventHandlers.get('executing') assert.exists(handler) handler(new CustomEvent('executing', { detail: null })) @@ -856,11 +862,6 @@ describe('executionStore workflow gating', () => { }) describe('switching tabs mid-run', () => { - // The QA run on a real generation found this: gating stops a background - // workflow writing the mirror, but whatever it wrote while it was in front - // stays there, and `nodeLocationProgressStates` resolves node ids against - // the *currently* active graph. Two workflows sharing a node id therefore - // show the stale entry on the newly visible node. it('clears the mirror when switching to a workflow with no running job', async () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) @@ -891,7 +892,6 @@ describe('executionStore workflow gating', () => { await nextTick() expect(store.nodeProgressStates['1']).toBeUndefined() - // A's run advanced while it was in the background; per-job state kept it. fire('progress_state', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -942,10 +942,6 @@ describe('executionStore workflow gating', () => { expect(store.nodeProgressStates['1']?.value).toBe(7) expect(store.nodeProgressStates['1']?.prompt_id).toBe('job-b') }) - // A legacy server sends no workflow_id at all, so ownership resolves - // through the session path the client recorded at queue time. The QA pass - // against core f1072eb0 saw foreign progress here; it must clear on switch - // on that backend too, not only on one that stamps the frames. it('clears the mirror on a backend that sends no workflow id', async () => { useWorkflowStore().activeWorkflow = workflowA queueLegacyJobFrom('job-a', workflowA) @@ -960,15 +956,11 @@ describe('executionStore workflow gating', () => { expect(store.nodeProgressStates['1']).toBeUndefined() }) - // Round 2 of the GPU QA: the first switch is clean, a later one is not. it('stays clean across repeated switches during one run', async () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) - // Two cycles, not a loop: each depends on the state the previous one - // left, and what QA saw was the first arrival in B being clean and a - // later one not, so first-versus-later is the distinction being made. - const cycle = async (step: number) => { + async function cycle(step: number) { fire('progress_state', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -1005,7 +997,6 @@ describe('executionStore workflow gating', () => { await nextTick() expect(store.nodeProgressStates['1']).toBeUndefined() - // A keeps running; B's own job has not started yet. fire('progress_state', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID, @@ -1014,11 +1005,6 @@ describe('executionStore workflow gating', () => { expect(store.nodeProgressStates['1']).toBeUndefined() }) - // The canvas stamps progress onto the new graph's nodes when the graph - // changes (nodeProgressCanvasSync.replaceGraph), reading whatever the - // mirror holds at that instant. Clearing the mirror on a watcher is a race - // against that, and it is the race the GPU QA lost. So the projection has - // to be correct even when the mirror still holds the foreign entry. it('does not project a foreign entry that is still in the mirror', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) @@ -1029,8 +1015,6 @@ describe('executionStore workflow gating', () => { }) expect(Object.keys(store.nodeLocationProgressStates)).toHaveLength(1) - // Switch without letting the watcher run, which is what the canvas sees - // when it observes the graph change first. useWorkflowStore().activeWorkflow = workflowB expect(store.nodeProgressStates['1']?.value).toBe(3) @@ -1057,9 +1041,6 @@ describe('executionStore workflow gating', () => { openTabs(workflowA, workflowACopy) }) - // Node outputs are written by ComfyApp's own `executed` listener, not by - // this store, and it resolves the node against the visible graph. The gate - // it consults is this predicate. it('rejects a foreign job output frame for the copy', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) @@ -1102,11 +1083,6 @@ describe('executionStore workflow gating', () => { }) }) - /** - * The resolution order is load-bearing and was wrong once: keying on the - * graph id first let every tab sharing an id claim every job. These pin the - * order itself, so reordering the legs fails here rather than in QA. - */ describe('ownership resolution precedence', () => { beforeEach(() => { openTabs(workflowA, workflowACopy) @@ -1115,8 +1091,6 @@ describe('executionStore workflow gating', () => { it('prefers the tab instance over both weaker legs', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) - // Point the path leg at the copy as well, so both weaker legs name the - // copy and only the instance names A. Reordering the legs flips this. store.ensureSessionWorkflowPath( 'job-a', workflowACopy.path, @@ -1134,7 +1108,6 @@ describe('executionStore workflow gating', () => { it('falls back to the session path when no instance is known', () => { useWorkflowStore().activeWorkflow = workflowA - // No storeJob, so no instance mapping: register only the path. store.ensureSessionWorkflowPath('job-pathonly', workflowACopy.path) expect( @@ -1162,8 +1135,6 @@ describe('executionStore workflow gating', () => { it('stays permissive when ownership cannot be resolved at all', () => { useWorkflowStore().activeWorkflow = workflowA - // A job from another browser session: no mapping, no frame id. Keeping - // this permissive is what preserves single-tab behaviour. expect(store.belongsToActiveWorkflow('job-unknown', undefined)).toBe(true) }) @@ -1171,8 +1142,6 @@ describe('executionStore workflow gating', () => { useWorkflowStore().activeWorkflow = workflowA queueJobFrom('job-a', workflowA) - // Save As mints a new workflow with its own instanceId and a new graph - // id, and switches to it. The original tab keeps the run. const savedAs = workflow( 'a-brand-new-graph-id', 'workflows/saved as.json' @@ -1198,16 +1167,6 @@ describe('executionStore workflow gating', () => { }) }) - /** - * Reported by review on #20045. handleExecutionStart deliberately does not - * adopt activeJobId for a job whose tab is in the background, so that a - * background run cannot steal the visible tab's execution UI. But nothing - * adopted it on the way back either, so a tab could show node progress while - * isIdle stayed true: the title, favicon and overlay read idle, and - * handleExecuted dropped completion frames at `if (!activeJob.value) return`. - * Reachable by the second prompt of a batch, or anything queued behind - * another job while the user is on a different tab. - */ describe('returning to a tab whose job started in the background', () => { it('adopts the job so the tab is not both running and idle', async () => { useWorkflowStore().activeWorkflow = workflowA @@ -1226,8 +1185,6 @@ describe('executionStore workflow gating', () => { workflow_id: WORKFLOW_A_ID, nodes: { '1': nodeState('job-a', '1', 'running', 3) } }) - expect(store.activeJobId, 'B must not adopt A job').toBeNull() - useWorkflowStore().activeWorkflow = workflowA await nextTick() @@ -1350,8 +1307,6 @@ describe('executionStore workflow gating', () => { workflow_id: WORKFLOW_A_ID, timestamp: 1 }) - expect(store.activeJobId).toBe('job-a') - useWorkflowStore().activeWorkflow = workflowB fire('execution_success', { prompt_id: 'job-a', diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index f693c5c0121..630780f2db4 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -781,10 +781,6 @@ describe('useQueueStore', () => { describe('terminal-job reconciliation', () => { it('clears progress for a job the backend reports only in history', async () => { - // This is the only production caller of reconcileTerminalJobs, and it was - // unverified: deleting the call, or swapping its two arguments so every - // running job is treated as terminal, left the whole suite green. - // Asserting the eviction rather than spying catches both. const executionStore = useExecutionStore() executionStore.nodeProgressStatesByJob = { 'hist-stuck': { From bc47f28f36c2196683ca1fcd9ae1d87de590ca52 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sun, 11 Oct 2026 09:49:55 +0000 Subject: [PATCH 42/52] test(api): address execution routing review Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210702840 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210754541 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210755227 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4210756548 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225148233 Addresses https://github.com/Comfy-Org/ComfyUI_frontend/pull/20045#discussion_r4225425844 --- src/scripts/api.queuePrompt.test.ts | 8 +-- src/scripts/api.socketReset.test.ts | 6 -- src/scripts/api.ts | 23 +------ src/scripts/app.test.ts | 98 +++++++++++++++-------------- src/scripts/app.ts | 3 - 5 files changed, 57 insertions(+), 81 deletions(-) diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts index 3a74bfb9c22..a73aeb102b7 100644 --- a/src/scripts/api.queuePrompt.test.ts +++ b/src/scripts/api.queuePrompt.test.ts @@ -8,17 +8,15 @@ import type { import { api } from '@/scripts/api' import { zeroUuid } from '@/utils/uuid' -// Tests for the workflow_metadata field api.queuePrompt sends; fetchApi is stubbed. const WORKFLOW_ID = '0199e3a3-6c01-7000-8000-c1a1f7a8d9b2' const EMPTY_PROMPT: ComfyApiWorkflow = {} -// A real `Response`, so the stub cannot drift out of shape with the contract -// `queuePrompt` reads. A fresh one per test: a Response body reads once. -const promptResponse = (): Response => - Response.json( +function promptResponse(): Response { + return Response.json( { prompt_id: 'p1', number: 1, node_errors: {} }, { status: 200 } ) +} function workflow(id?: string): ComfyWorkflowJSON { return { diff --git a/src/scripts/api.socketReset.test.ts b/src/scripts/api.socketReset.test.ts index 16a651d7398..8d54dc1121a 100644 --- a/src/scripts/api.socketReset.test.ts +++ b/src/scripts/api.socketReset.test.ts @@ -167,12 +167,6 @@ describe('ComfyApi realtime socket reset', () => { }) it('exposes the executing ids during dispatch and clears them after', async () => { - // The `executing` event carries only a node id for extension - // compatibility, so the store reads prompt_id and workflow_id off this - // field during the synchronous dispatch. Nothing pinned either half: - // turning the `finally` into a no-op left every api and executionStore - // test green, and a field left set would attribute the next frame to a - // stale prompt. await api.resetSocket() const socket = FakeWebSocket.instances[0] socket.simulateOpen() diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 2d700f85468..2a39c4324ff 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -466,10 +466,9 @@ export class ComfyApi extends EventTarget { */ clientId?: string /** - * The last `executing` message as the server sent it, including `prompt_id` - * and `workflow_id`. The public `executing` event carries only the node id - * for backwards compatibility with extensions, so a consumer that needs to - * know which run a frame belongs to reads it here. + * The `executing` message being dispatched, including `prompt_id` and + * `workflow_id`. Non-null only while `executing` listeners run, so read it + * synchronously inside the listener. */ lastExecutingMessage: ExecutingWsMessage | null = null /** @@ -1111,13 +1110,6 @@ export class ComfyApi extends EventTarget { this.dispatchCustomEvent('status', msg.data.status ?? null) break case 'executing': - // The public `executing` event keeps its bare NodeId detail — - // extensions depend on that shape — so the ids travel here - // instead, for consumers that must know which run the frame - // belongs to. Dispatch is synchronous, so clearing it straight - // after keeps the window to this frame's own handlers: an - // `executing` dispatched later by an extension must not be read - // as belonging to whichever run happened to arrive last. this.lastExecutingMessage = msg.data try { this.dispatchCustomEvent( @@ -1311,15 +1303,6 @@ export class ComfyApi extends EventTarget { ...(options?.partialExecutionTargets && { partial_execution_targets: options.partialExecutionTargets }), - // `LGraph.serialize()` returns `id` unfiltered and `LGraph._id` defaults - // to the all-zero sentinel, which the codebase treats as "no id yet" and - // replaces on load (`adoptRootGraphId`) and on clear, so it is not a - // routing key and is not sent as one. This does not keep it off the wire: - // the same graph goes out under `extra_pnginfo` and the server falls back - // to the id there, which is why `executionStore` also discards the - // sentinel on the way in. Shape is already constrained upstream — the - // loaded schema is `z.string().uuid()` and the only other producer is - // `createUuidv4()` — so the sentinel is the one value to exclude here. ...(workflow.id && workflow.id !== zeroUuid && { workflow_metadata: { workflow_id: workflow.id } diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index d71852bbf18..498b5364494 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -3151,56 +3151,60 @@ describe('ComfyApp', () => { ) }) - it('does not apply a background run output to the visible workflow', async () => { - // The gate in the `executed` listener. Outputs are keyed by a locator - // resolved against the visible graph, so without it a job finishing in - // another tab writes its result onto the same-numbered node in front. - // Spying on the writer rather than reading nodeOutputs, because the - // fixture graph has no nodes and the locator would never resolve. - const workflowA = markLoaded( - new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) - ) - const workflowB = markLoaded( - new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) - ) - workflowA.changeTracker.activeState = workflowGraphData(workflowAId) - workflowB.changeTracker.activeState = workflowGraphData(workflowBId) - useWorkflowStore().activeWorkflow = workflowA - - const executionStore = useExecutionStore() - executionStore.storeJob({ - nodes: ['1'], - id: 'job-b', - promptOutput: {}, - workflow: workflowB, - mode: 'graph' - }) - - const setOutputs = vi.spyOn( - useNodeOutputStore(), - 'setNodeOutputsByExecutionId' - ) - const addEventListener = vi.spyOn(api, 'addEventListener') - Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) - const executedHandler = addEventListener.mock.calls.find( - ([event]) => event === 'executed' - )?.[1] as EventListener | undefined - expect(executedHandler).toBeTypeOf('function') + it.for([ + { jobId: 'job-a', owner: 'active', applied: true }, + { jobId: 'job-b', owner: 'background', applied: false } + ] as const)( + '$owner run output applied: $applied', + async ({ jobId, owner, applied }) => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + Reflect.set(app, 'rootGraphInternal', new LGraph()) - executedHandler?.( - new CustomEvent('executed', { - detail: { - prompt_id: 'job-b', - workflow_id: workflowBId, - node: '1', - display_node: '1', - output: { images: [{ filename: 'from-b.png', type: 'output' }] } - } + const executionStore = useExecutionStore() + const workflow = owner === 'active' ? workflowA : workflowB + const workflowId = owner === 'active' ? workflowAId : workflowBId + executionStore.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: {}, + workflow, + mode: 'graph' }) - ) - expect(setOutputs).not.toHaveBeenCalled() - }) + const setOutputs = vi.spyOn( + useNodeOutputStore(), + 'setNodeOutputsByExecutionId' + ) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const executedHandler = addEventListener.mock.calls.find( + ([event]) => event === 'executed' + )?.[1] as EventListener | undefined + expect(executedHandler).toBeTypeOf('function') + + executedHandler?.( + new CustomEvent('executed', { + detail: { + prompt_id: jobId, + workflow_id: workflowId, + node: '1', + display_node: '1', + output: { images: [{ filename: 'from-b.png', type: 'output' }] } + } + }) + ) + + expect(setOutputs).toHaveBeenCalledTimes(applied ? 1 : 0) + } + ) it('keeps a background preview job-keyed but off the visible graph', () => { const workflowA = markLoaded( diff --git a/src/scripts/app.ts b/src/scripts/app.ts index b3ca6d1c44d..253b159f64a 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -918,9 +918,6 @@ export class ComfyApp { detail.display_node || detail.node ) if (!executionId) return - // Outputs are keyed by a locator resolved against the visible graph, so - // a job finishing in another tab would otherwise write its result onto - // the same-numbered node of the tab in front. if ( !useExecutionStore().belongsToActiveWorkflow( detail.prompt_id, From 0fcf4302acc57de754afe7e5f83cdab18e84de79 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 14:11:20 -0700 Subject: [PATCH 43/52] fix(linear): do not append a second output copy when a reconnect replays CI went red on the strengthened reconnect spec with 2 image outputs where 1 was expected, which is a real duplication rather than a test problem. The first executed frame consumes the skeleton item and clears currentSkeletonId, so the replayed frame finds no skeleton, falls through to the create-directly branch and appends the same output again. onNodeExecuted recorded the node in executedNodeIds but never checked it, where onLatentPreview does. Deduped on jobId:nodeId rather than nodeId alone, because two jobs can be in flight and share node ids, which is the case this whole PR exists for. Keying by node alone turns four other tests red. Four existing tests meant 'another output node in the same job' but passed the default node id, which is indistinguishable from a replay. They name the node now, since the node id is exactly what separates the two cases. Three new store tests: the replay is deduped, a second output node still appends, and the same node id across two concurrent jobs is not deduped. I could not run the browser spec locally, as the e2e backend is not up on this box and is not this session's to start, so CI is the check on it. --- .../linearMode/linearOutputStore.test.ts | 122 ++++++++++++++---- .../linearMode/linearOutputStore.ts | 31 +++++ 2 files changed, 129 insertions(+), 24 deletions(-) diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index b3783e3e2de..1ca0f0ae406 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -266,6 +266,64 @@ describe('linearOutputStore', () => { expect(store.inProgressItems).toHaveLength(2) }) + // A reconnect replays the run's frames. The first `executed` consumes the + // skeleton and clears currentSkeletonId, so the replay found no skeleton and + // fell through to the create-directly branch, appending the same output a + // second time. Caught in CI by the browser spec at 2 image outputs; this pins + // it at the store level too. + it('does not append a second copy when a reconnect replays executed', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(1) + + // Same job, same node, same output: a catch-up replay, not a new node. + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image'), + 'the replayed frame must not add a second item' + ).toHaveLength(1) + }) + + it('still appends a second output node in the same job', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + // A different output node, which is what distinguishes this from a replay. + store.onNodeExecuted( + 'job-1', + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) + ) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + + it('does not dedupe the same node id across two concurrent jobs', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + + // Two jobs in flight share node ids, which is the reason the key is scoped + // by job and not by node alone. + store.onJobStart('job-2') + store.onNodeExecuted('job-2', makeExecutedDetail('job-2')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + it('handles execute without prior skeleton (no latent preview)', () => { const store = useLinearOutputStore() store.onJobStart('job-1') @@ -277,9 +335,11 @@ describe('linearOutputStore', () => { // Second node executes directly (no latent, no skeleton) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) const images = store.inProgressItems.filter((i) => i.state === 'image') @@ -356,15 +416,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'c.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'c.png', subfolder: '', type: 'output' }], + '3' + ) ) const run1Images = store.inProgressItems.filter((i) => i.state === 'image') @@ -383,15 +447,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-2', makeExecutedDetail('job-2')) store.onNodeExecuted( 'job-2', - makeExecutedDetail('job-2', [ - { filename: 'e.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-2', + [{ filename: 'e.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-2', - makeExecutedDetail('job-2', [ - { filename: 'f.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-2', + [{ filename: 'f.png', subfolder: '', type: 'output' }], + '3' + ) ) const run2Images = store.inProgressItems.filter((i) => i.state === 'image') @@ -413,9 +481,11 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onJobComplete('job-1') @@ -440,15 +510,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'c.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'c.png', subfolder: '', type: 'output' }], + '3' + ) ) store.onJobComplete('job-1') diff --git a/src/renderer/extensions/linearMode/linearOutputStore.ts b/src/renderer/extensions/linearMode/linearOutputStore.ts index 28964da64f2..392653d6f7c 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.ts @@ -29,6 +29,18 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { const trackedJobId = ref(null) const pendingResolve = ref(new Set()) const executedNodeIds = new Set() + /** + * `jobId:nodeId` of every output already appended to the history. + * + * A reconnect replays the run's frames, and the first `executed` consumes the + * skeleton item and clears `currentSkeletonId`, so the replayed frame finds no + * skeleton and falls through to the create-directly branch below, appending a + * second copy of the same output. Keyed by job as well as node, because two + * jobs can be in flight at once and share node ids, which is the whole reason + * this PR exists. + */ + const appendedOutputKeys = new Set() + const MAX_APPENDED_OUTPUT_KEYS = 500 const activeWorkflowInProgressItems = computed(() => { const path = workflowStore.activeWorkflow?.path @@ -60,6 +72,10 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onJobStart(jobId: JobId) { executedNodeIds.clear() + // This job's keys only: a concurrent job's outputs must stay deduped. + for (const key of appendedOutputKeys) { + if (key.startsWith(`${jobId}:`)) appendedOutputKeys.delete(key) + } const item: InProgressItem = { id: makeItemId(jobId), @@ -112,6 +128,8 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onNodeExecuted(jobId: JobId, detail: ExecutedWsMessage) { const nodeId = String(detail.display_node || detail.node) + const outputKey = `${jobId}:${nodeId}` + if (appendedOutputKeys.has(outputKey)) return executedNodeIds.add(nodeId) if (raf) { cancelAnimationFrame(raf) @@ -153,6 +171,7 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { arr.splice(idx, 1, imageItem, ...extras) currentSkeletonId.value = null inProgressItems.value = arr + rememberAppendedOutput(outputKey) return } @@ -167,6 +186,18 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { })) autoSelect(`slot:${newItems[0].id}`, jobId) inProgressItems.value = [...newItems, ...inProgressItems.value] + rememberAppendedOutput(outputKey) + } + + function rememberAppendedOutput(key: string) { + appendedOutputKeys.add(key) + // Bounded: far above nodes-per-job times concurrent jobs, and Set keeps + // insertion order, so the oldest key goes first. + while (appendedOutputKeys.size > MAX_APPENDED_OUTPUT_KEYS) { + const oldest = appendedOutputKeys.values().next().value + if (oldest === undefined) break + appendedOutputKeys.delete(oldest) + } } function onJobComplete(jobId: JobId) { From ea05fb743e3d44eaa23bcd488a816d406cc1d475 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 14:14:24 -0700 Subject: [PATCH 44/52] test(e2e): keep the 3D reload idle budget, and drop a no-op assignment openPersistedWorkflow takes an idleTimeout now, so the 3D reload path keeps the 30s it was given rather than inheriting the 5s default. The preview3d spec did pass on CI at the previous head, which answers the reviewer's open question, but a 3D reparse and re-render is the wrong place to quietly cut the budget by six. Kept openPersistedWorkflowFromSidebar rather than inlining it: the spec has only the pipeline fixture in scope, and the wrapper now carries the budget instead of being a pure pass-through. Also removed a second activeWorkflow assignment that was already the current value. Verified the test still falsifies: making the adoption predicate accept any job turns it red. --- .../fixtures/helpers/Preview3DPipelineFixture.ts | 10 +++++++++- browser_tests/fixtures/helpers/WorkflowHelper.ts | 12 ++++++++++-- src/stores/executionStore.workflowGating.test.ts | 3 ++- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts index 6a1f64f495e..e5ee44d7079 100644 --- a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts +++ b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts @@ -214,8 +214,16 @@ export class Preview3DPipelineContext { await this.comfyPage.waitForAppReady() } + /** + * Reopening after a reload has to re-parse and re-render the model, which was + * given 30s before this went through `openPersistedWorkflow` and its 5s + * default. Kept here rather than in the spec so the budget lives with the + * pipeline that needs it. + */ async openPersistedWorkflowFromSidebar(workflowName: string): Promise { - await this.comfyPage.workflow.openPersistedWorkflow(workflowName) + await this.comfyPage.workflow.openPersistedWorkflow(workflowName, { + idleTimeout: 30_000 + }) } async assertPreview3dModelPathAndLastTime(path: string): Promise { diff --git a/browser_tests/fixtures/helpers/WorkflowHelper.ts b/browser_tests/fixtures/helpers/WorkflowHelper.ts index 3907d23e1be..f94a1fc838d 100644 --- a/browser_tests/fixtures/helpers/WorkflowHelper.ts +++ b/browser_tests/fixtures/helpers/WorkflowHelper.ts @@ -230,13 +230,21 @@ export class WorkflowHelper { }) } - async openPersistedWorkflow(workflowName: string): Promise { + /** + * @param idleTimeout How long to wait for the workflow to settle. The default + * suits a normal graph; a 3D reload was given 30s before this helper existed + * and passes its own. + */ + async openPersistedWorkflow( + workflowName: string, + { idleTimeout }: { idleTimeout?: number } = {} + ): Promise { const sidebar = this.comfyPage.menu.workflowsTab const wasOpen = await sidebar.selectedTabButton.isVisible() await sidebar.open() await sidebar.getPersistedItem(workflowName, { exact: true }).click() if (!wasOpen) await sidebar.close() - await this.waitForWorkflowIdle() + await this.waitForWorkflowIdle(idleTimeout) await expect .poll(async () => (await this.getActiveWorkflowPath())?.endsWith(`${workflowName}.json`) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 1ce70407cb3..eaa832f9b6a 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1207,7 +1207,8 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-b', '1', 'running', 3) } }) - useWorkflowStore().activeWorkflow = workflowA + // activeWorkflow is already A, so the watcher fires off that first + // assignment once nextTick flushes. await nextTick() expect(store.activeJobId).toBeNull() From 443d96e7ea2d851f8ce997736f5609ae3171477c Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 14:33:51 -0700 Subject: [PATCH 45/52] test(execution): pin the service-level error release separately handleExecutionError and handleServiceLevelError each release a finished background job's records, and only the node-error branch was covered: deleting the release from handleServiceLevelError left the suite green. The foreign-error test is now a two-row table over a node error and a service-level error (no node_id, which is how 'Job has stagnated' arrives), each asserting nodeProgressStatesByJob and queuedJobs are released and that the visible run is untouched. Verified: the service-level row goes red with that release removed and the node-error row still passes, so the branches are pinned separately. Cloud validation still has no row; it needs isCloud. --- .../executionStore.workflowGating.test.ts | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index eaa832f9b6a..77e4fbef270 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -416,26 +416,48 @@ describe('executionStore workflow gating', () => { ).toBeUndefined() }) - it('execution_error from another workflow leaves active state alone but clears its initializing flag', () => { - queueJobFrom('job-b', workflowB) - fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) - expect(store.isJobInitializing('job-b')).toBe(true) - - fire('execution_error', { - prompt_id: 'job-b', - workflow_id: WORKFLOW_B_ID, - timestamp: 2, - node_id: '1', - node_type: 'TestNode', - exception_message: 'fail', - exception_type: 'Error', - traceback: [], - executed: [] - }) + // Two branches: a node error goes through handleExecutionError, a + // service-level error (no node_id, e.g. "Job has stagnated") through + // handleServiceLevelError. Each releases the background job's records + // separately, and the service-level release was previously untested, so + // deleting it left the whole suite green. + it.for([ + { kind: 'a node error', nodeId: '1' as string | null }, + { kind: 'a service-level error', nodeId: null } + ])( + '$kind from another workflow releases that job and leaves the visible one alone', + ({ nodeId }) => { + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 2) } + }) + expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() + + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: nodeId, + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [], + executed: [] + }) - expect(store.activeJobId).toBe('job-a') - expect(store.isJobInitializing('job-b')).toBe(false) - }) + expect(store.activeJobId, 'the visible run is untouched').toBe('job-a') + expect(store.isJobInitializing('job-b')).toBe(false) + expect( + store.nodeProgressStatesByJob['job-b'], + "the failed background job's own records are released" + ).toBeUndefined() + expect(store.queuedJobs['job-b']).toBeUndefined() + } + ) it('does not reset the active workflow when a stamped background error arrives before storeJob', () => { fire('execution_error', { From aa9680ee394f3ae9c47a24564eef28c0fc6f8d0c Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 14:36:38 -0700 Subject: [PATCH 46/52] fix(linear): key the output dedupe on node, not display_node Node expansion gives each expanded copy its own `node` while they all share one `display_node` (core sends display_node: dynprompt.get_display_node_id(unique_id)), so keying on the display id collapsed every loop iteration's output into the first. A replay repeats the same `node`, so it still dedupes. That was a regression in my own previous commit, caught in review before it could ship. New test: two expanded copies with node 9.0.1 and 9.0.2 sharing display_node 9 keep one item each, and it goes red on the old key. Also dropped comments that narrate history rather than explain code, in the dedupe docblock, the store test, the 3D fixture and the openPersistedWorkflow param, and the leftover comment where a no-op assignment was meant to just go. --- .../helpers/Preview3DPipelineFixture.ts | 6 ---- .../fixtures/helpers/WorkflowHelper.ts | 5 ---- .../linearMode/linearOutputStore.test.ts | 30 +++++++++++++++---- .../linearMode/linearOutputStore.ts | 13 ++------ .../executionStore.workflowGating.test.ts | 2 -- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts index e5ee44d7079..bef3e0e4c44 100644 --- a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts +++ b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts @@ -214,12 +214,6 @@ export class Preview3DPipelineContext { await this.comfyPage.waitForAppReady() } - /** - * Reopening after a reload has to re-parse and re-render the model, which was - * given 30s before this went through `openPersistedWorkflow` and its 5s - * default. Kept here rather than in the spec so the budget lives with the - * pipeline that needs it. - */ async openPersistedWorkflowFromSidebar(workflowName: string): Promise { await this.comfyPage.workflow.openPersistedWorkflow(workflowName, { idleTimeout: 30_000 diff --git a/browser_tests/fixtures/helpers/WorkflowHelper.ts b/browser_tests/fixtures/helpers/WorkflowHelper.ts index f94a1fc838d..fca31e98dc8 100644 --- a/browser_tests/fixtures/helpers/WorkflowHelper.ts +++ b/browser_tests/fixtures/helpers/WorkflowHelper.ts @@ -230,11 +230,6 @@ export class WorkflowHelper { }) } - /** - * @param idleTimeout How long to wait for the workflow to settle. The default - * suits a normal graph; a 3D reload was given 30s before this helper existed - * and passes its own. - */ async openPersistedWorkflow( workflowName: string, { idleTimeout }: { idleTimeout?: number } = {} diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index 1ca0f0ae406..fc8c869b9e9 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -266,11 +266,31 @@ describe('linearOutputStore', () => { expect(store.inProgressItems).toHaveLength(2) }) - // A reconnect replays the run's frames. The first `executed` consumes the - // skeleton and clears currentSkeletonId, so the replay found no skeleton and - // fell through to the create-directly branch, appending the same output a - // second time. Caught in CI by the browser spec at 2 image outputs; this pins - // it at the store level too. + // Node expansion (loop nodes and anything else expanding a subgraph) gives + // each expanded copy its own `node` while they all share one `display_node`, + // so keying the dedupe on the display id collapsed every iteration's output + // into the first. Core: execution.py sends + // "display_node": dynprompt.get_display_node_id(unique_id). + it('keeps one item per expanded copy sharing a display node', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + for (const [node, file] of [ + ['9.0.1', 'iter1.png'], + ['9.0.2', 'iter2.png'] + ] as const) { + store.onNodeExecuted('job-1', { + prompt_id: 'job-1', + node, + display_node: '9', + output: { images: [{ filename: file, subfolder: '', type: 'output' }] } + }) + } + + const images = store.inProgressItems.filter((i) => i.state === 'image') + expect(images, 'each loop iteration keeps its own output').toHaveLength(2) + }) + it('does not append a second copy when a reconnect replays executed', () => { const store = useLinearOutputStore() store.onJobStart('job-1') diff --git a/src/renderer/extensions/linearMode/linearOutputStore.ts b/src/renderer/extensions/linearMode/linearOutputStore.ts index 392653d6f7c..580954f36df 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.ts @@ -29,16 +29,7 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { const trackedJobId = ref(null) const pendingResolve = ref(new Set()) const executedNodeIds = new Set() - /** - * `jobId:nodeId` of every output already appended to the history. - * - * A reconnect replays the run's frames, and the first `executed` consumes the - * skeleton item and clears `currentSkeletonId`, so the replayed frame finds no - * skeleton and falls through to the create-directly branch below, appending a - * second copy of the same output. Keyed by job as well as node, because two - * jobs can be in flight at once and share node ids, which is the whole reason - * this PR exists. - */ + /** `jobId:node` of outputs already appended, so a replayed `executed` is skipped. */ const appendedOutputKeys = new Set() const MAX_APPENDED_OUTPUT_KEYS = 500 @@ -128,7 +119,7 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onNodeExecuted(jobId: JobId, detail: ExecutedWsMessage) { const nodeId = String(detail.display_node || detail.node) - const outputKey = `${jobId}:${nodeId}` + const outputKey = `${jobId}:${detail.node}` if (appendedOutputKeys.has(outputKey)) return executedNodeIds.add(nodeId) if (raf) { diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 77e4fbef270..3915cc7e020 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -1229,8 +1229,6 @@ describe('executionStore workflow gating', () => { nodes: { '1': nodeState('job-b', '1', 'running', 3) } }) - // activeWorkflow is already A, so the watcher fires off that first - // assignment once nextTick flushes. await nextTick() expect(store.activeJobId).toBeNull() From 89f13baa2eac7bf1845231a3e0eaac7160db288f Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 15:05:38 -0700 Subject: [PATCH 47/52] fix(linear): identify a replayed output by the frame, not just its node My node-only dedupe key broke three pre-existing outputHistory specs, which fire the same job and node twice with different files and expect two items. That is a second output, not a replay, so the key was wrong rather than the specs. What identifies a replay is that the frame is identical. The key is now jobId, node and the sorted subfolder/filename list, which separates all three cases: a reconnect replays the same files and is skipped, a node reporting different files is kept, and an expanded copy in a loop differs by node as well. Added the case that broke CI as a store test, so it cannot regress to a node-only or display-node key again without going red. --- .../linearMode/linearOutputStore.test.ts | 21 +++++++++++++++++++ .../linearMode/linearOutputStore.ts | 19 +++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index fc8c869b9e9..0e849dd9a39 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -291,6 +291,27 @@ describe('linearOutputStore', () => { expect(images, 'each loop iteration keeps its own output').toHaveLength(2) }) + // Broke three outputHistory specs when the key was node-only: the same node + // reporting different files is a second output, not a replay. What identifies + // a replay is that the frame is identical, so the key carries the files. + it('keeps both when one node reports two different outputs', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + store.onNodeExecuted( + 'job-1', + makeExecutedDetail('job-1', [ + { filename: 'second.png', subfolder: '', type: 'output' } + ]) + ) + + expect( + store.inProgressItems.filter((i) => i.state === 'image'), + 'same node, different files, so both are real outputs' + ).toHaveLength(2) + }) + it('does not append a second copy when a reconnect replays executed', () => { const store = useLinearOutputStore() store.onJobStart('job-1') diff --git a/src/renderer/extensions/linearMode/linearOutputStore.ts b/src/renderer/extensions/linearMode/linearOutputStore.ts index 580954f36df..b1a34b5b8bd 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.ts @@ -29,7 +29,14 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { const trackedJobId = ref(null) const pendingResolve = ref(new Set()) const executedNodeIds = new Set() - /** `jobId:node` of outputs already appended, so a replayed `executed` is skipped. */ + /** + * Outputs already appended, so a replayed `executed` is skipped. + * + * Keyed by what the frame carries, not just where it came from. A reconnect + * replays the identical frame, while a node that legitimately reports again + * carries different files, and an expanded copy in a loop shares its + * `display_node` with its siblings but not its `node`. + */ const appendedOutputKeys = new Set() const MAX_APPENDED_OUTPUT_KEYS = 500 @@ -119,7 +126,7 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onNodeExecuted(jobId: JobId, detail: ExecutedWsMessage) { const nodeId = String(detail.display_node || detail.node) - const outputKey = `${jobId}:${detail.node}` + const outputKey = appendedOutputKey(jobId, detail) if (appendedOutputKeys.has(outputKey)) return executedNodeIds.add(nodeId) if (raf) { @@ -180,6 +187,14 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { rememberAppendedOutput(outputKey) } + function appendedOutputKey(jobId: JobId, detail: ExecutedWsMessage): string { + const files = flattenNodeOutput([String(detail.node), detail.output]) + .map((item) => `${item.subfolder}/${item.filename}`) + .sort() + .join(',') + return `${jobId}:${detail.node}:${files}` + } + function rememberAppendedOutput(key: string) { appendedOutputKeys.add(key) // Bounded: far above nodes-per-job times concurrent jobs, and Set keeps From 0171c0b7c817eb244dcff8e46ce298f10aadfbd1 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 15:29:12 -0700 Subject: [PATCH 48/52] test(linear): make every part of the dedupe key falsifiable Dropped a loop from a test body and a comment carrying review history, both per the repo's own conventions. While checking the first, I found the node component of the dedupe key was not load-bearing: removing it left every test green, because the two expanded copies in the loop test also differ by filename. Added the case that needs it, two preview nodes fed from one upstream image reporting the same file, where each is its own output. Removing either the node or the file list from the key now turns a test red. --- .../linearMode/linearOutputStore.test.ts | 42 ++++++++++++++----- .../executionStore.workflowGating.test.ts | 5 --- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index 0e849dd9a39..530dc57ab2e 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -275,17 +275,22 @@ describe('linearOutputStore', () => { const store = useLinearOutputStore() store.onJobStart('job-1') - for (const [node, file] of [ - ['9.0.1', 'iter1.png'], - ['9.0.2', 'iter2.png'] - ] as const) { - store.onNodeExecuted('job-1', { - prompt_id: 'job-1', - node, - display_node: '9', - output: { images: [{ filename: file, subfolder: '', type: 'output' }] } - }) - } + store.onNodeExecuted('job-1', { + prompt_id: 'job-1', + node: '9.0.1', + display_node: '9', + output: { + images: [{ filename: 'iter1.png', subfolder: '', type: 'output' }] + } + }) + store.onNodeExecuted('job-1', { + prompt_id: 'job-1', + node: '9.0.2', + display_node: '9', + output: { + images: [{ filename: 'iter2.png', subfolder: '', type: 'output' }] + } + }) const images = store.inProgressItems.filter((i) => i.state === 'image') expect(images, 'each loop iteration keeps its own output').toHaveLength(2) @@ -312,6 +317,21 @@ describe('linearOutputStore', () => { ).toHaveLength(2) }) + it('keeps both when two nodes report the same file', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + // Two PreviewImage nodes fed from one upstream image report the same + // filename, and each is its own output. Without the node in the key the + // files alone would make these look like one frame replayed. + store.onNodeExecuted('job-1', makeExecutedDetail('job-1', undefined, '7')) + store.onNodeExecuted('job-1', makeExecutedDetail('job-1', undefined, '8')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + it('does not append a second copy when a reconnect replays executed', () => { const store = useLinearOutputStore() store.onJobStart('job-1') diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts index 3915cc7e020..5af7ac8dbf9 100644 --- a/src/stores/executionStore.workflowGating.test.ts +++ b/src/stores/executionStore.workflowGating.test.ts @@ -416,11 +416,6 @@ describe('executionStore workflow gating', () => { ).toBeUndefined() }) - // Two branches: a node error goes through handleExecutionError, a - // service-level error (no node_id, e.g. "Job has stagnated") through - // handleServiceLevelError. Each releases the background job's records - // separately, and the service-level release was previously untested, so - // deleting it left the whole suite green. it.for([ { kind: 'a node error', nodeId: '1' as string | null }, { kind: 'a service-level error', nodeId: null } From 4441b348ff65f153b0f069f20b57fb898a956b23 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 11 Oct 2026 15:50:15 -0700 Subject: [PATCH 49/52] test(linear): drop CI history from a test comment The test name and its assertion message carry what matters; the history is in the commit that introduced the fix. --- src/renderer/extensions/linearMode/linearOutputStore.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index 530dc57ab2e..fa76151a5d6 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -296,9 +296,6 @@ describe('linearOutputStore', () => { expect(images, 'each loop iteration keeps its own output').toHaveLength(2) }) - // Broke three outputHistory specs when the key was node-only: the same node - // reporting different files is a second output, not a replay. What identifies - // a replay is that the frame is identical, so the key carries the files. it('keeps both when one node reports two different outputs', () => { const store = useLinearOutputStore() store.onJobStart('job-1') From d63625b06eabe5b013ad06bfe4e2b1cc81386c03 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 17 Oct 2026 10:21:50 -0700 Subject: [PATCH 50/52] docs(adr): record the execution-frame ownership decision and its D2 exception Review raised that this PR contradicts ADR-GRAPH-DOCUMENT-0026 D2 and that no ADR records the ownership decision. D2 says an unresolvable frame goes to an explicit unattributed bucket and that misdelivery is not the fallback; this change applies it to the active tab instead. ADR-EXECUTION-ROUTING-0043 records the resolution order and why the graph id is the weakest signal rather than the strongest, the all-zero UUID being read as absent on both sides, which writes are gated and which are not and why that is keyed on the consumer rather than on foreignness, and the D2 departure as a stated exception with the condition for removing it. Also the alternatives that were rejected: quarantining unresolvable frames, which would stop a single-tab user's own run updating after a reload; gating on the graph id alone, which shipped and was wrong; and a contextvars owner on the backend, which cannot work because prompt_worker is a daemon thread and send_json runs on the event loop. --- ...workflow-ownership-for-websocket-frames.md | 53 +++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 54 insertions(+) create mode 100644 docs/adr/EXECUTION-ROUTING-0043-workflow-ownership-for-websocket-frames.md diff --git a/docs/adr/EXECUTION-ROUTING-0043-workflow-ownership-for-websocket-frames.md b/docs/adr/EXECUTION-ROUTING-0043-workflow-ownership-for-websocket-frames.md new file mode 100644 index 00000000000..8262993919a --- /dev/null +++ b/docs/adr/EXECUTION-ROUTING-0043-workflow-ownership-for-websocket-frames.md @@ -0,0 +1,53 @@ +# ADR-EXECUTION-ROUTING-0043: Workflow ownership for execution websocket frames + +Date: 2026-10-17 + +## Status + +Proposed + +## Context + +One `LGraph` and one `LGraphCanvas` serve every open workflow tab, so execution state written by node id is shared state. A run started in one tab painted its progress, outputs and run status onto whichever tab was in front. Reported on #1293, #2660, #6730 and Comfy-Org/ComfyUI#6581, under the umbrella #3377, and reproduced by manual QA with three pasted copies of one graph. + +[ADR-GRAPH-DOCUMENT-0026](GRAPH-DOCUMENT-0026-frontend-document-model.md) D2 already states the rule: a frame arriving on a push channel resolves its target document at ingress via the job map, never via the active pointer. It also states what to do at the boundary — "events with no job→uid resolution (page reloaded mid-run, jobs queued by another client) go to an explicit _unattributed_ bucket rather than being applied to the active document by default … misdelivery is not the fallback." + +This change implements D2's addressing rule and **departs from its boundary rule**. That departure is the reason this ADR exists; the reviewer on #20045 was right that nothing recorded it. + +## Decision + +**Resolution order, strongest first.** A frame's owner is resolved by the open tab's `instanceId`, then the session path, then the workflow's graph id, then permissive. + +The graph id is deliberately the weakest signal, not the strongest. `ensureWorkflowId` keeps the id already present in a pasted graph, so copy a workflow, open a tab and paste gives two tabs carrying one id; loading a single workflow has also been measured leaving two open entries holding it. Keying on the graph id first is the bug QA found, where every tab sharing an id matched every job. `instanceId` is the only identifier unique per open tab, and it already existed. + +**The all-zero UUID is not an id.** `LGraph._id` defaults to it and the codebase reads it as "no id yet", replacing it on configure and on clear. It is not sent as a routing key, and an incoming one is discarded rather than compared. It still reaches the wire, because core falls back to the id inside `extra_data.extra_pnginfo.workflow`, and comparing it as a real id rejected a fresh graph's own frames. + +**Unresolvable frames are applied to the active tab.** This is the documented exception to D2's boundary rule. When no mapping and no usable id exist, the frame is delivered rather than quarantined. + +**Keys decide gating, not foreignness.** A frame being from another tab is not sufficient reason to drop work. What matters is the key the consumer reads: + +- Node-keyed writes (node progress, node outputs, node previews, text previews) are gated, because the visible graph may hold the same node ids. +- Job-keyed writes are not. `useJobPreviewStore().setPreviewUrl` feeds the queue panel's thumbnail and `linearOutputStore`'s re-apply on return, both for background jobs. +- Account-level signals are not. A sign-in, subscription or credit precondition belongs to the account, and `handleAccountPreconditionError` returns before recording the error, so its dialog is the only signal a background run produces. + +**Finished background jobs release their own records.** A job whose tab is not in front still ends, so each terminal path releases its per-job state rather than returning early at the gate. + +## Alternatives considered + +**Quarantine unresolvable frames, as D2 prescribes.** Rejected for this change. The common unresolvable case is a page reloaded mid-run, where the mappings are gone and the only open tab is the one the user is looking at. Quarantining would make a single-tab user's own run stop updating after a refresh — a regression for the overwhelmingly common case, in service of a rule written for the multi-tab case. D2's boundary also assumes a surfacing UI for the unattributed bucket, which does not exist. + +**Gate on the graph id alone.** This shipped and was wrong; see above. It is recorded here because the id looks like the obvious key and the mistake is easy to repeat. + +**A `contextvars`-based owner on the backend.** Rejected on a verified mechanism error: `prompt_worker` is a daemon thread and `send_json` runs on the event loop, so a ContextVar set in the worker is unset at send time. The attribute set on `PromptServer` and merged in the body of `send_sync` is what ships (Comfy-Org/ComfyUI#16763, #16809). + +**Derive the shared node-progress mirror instead of maintaining it.** Deferred to #20604, not rejected. It is read by the canvas sync path on every frame, so making it a computed changes when that work happens relative to painting, which is the interaction the original bug lived in. + +## Consequences + +**The permissive fallback is load-bearing and is also the weakest point.** Single-tab behaviour after a reload is preserved, and a job from another browser session is delivered to whatever is in front. If two tabs are open and a frame is unresolvable, it can still land on the wrong one. The blast radius is bounded by how rarely both conditions hold at once, not by the design. + +**`status` and `progress_text` frames stay unattributable.** `status` carries no `prompt_id` because one server-wide socket serves every workflow. `progress_text` is a packed binary frame that `send_sync` cannot stamp; the frontend already reads a `supports_progress_text_metadata` server flag that no core version sets. Tracked as Comfy-Org/ComfyUI#16887. Until it lands, the text-preview widget gate goes permissive. + +**The two backends disagree on merge order.** Core merges `{**workflow_metadata, **data}`, so a frame's own fields win. The Cloud gateway assigns `prompt_id` and `workflow_id` after the fact, so the backend wins. Neither order should be relied on; the routing ids that arrive are authoritative whichever side set them. + +**Revisit if** an unattributed-frame surface is built, at which point D2's boundary rule becomes implementable and this exception should be removed rather than inherited. diff --git a/docs/adr/README.md b/docs/adr/README.md index 5ff7ce4fdbb..9473fec07b8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -58,6 +58,7 @@ An Architecture Decision Record captures an important architectural decision mad | [ECS-LINK-PRESENTATION-0028](ECS-LINK-PRESENTATION-0028-link-presentation-store-ownership.md) | Link Presentation Store Owns Hidden/Label State | Proposed | 2026-08-29 | | [ECS-SLOTS-0017](ECS-SLOTS-0017-slot-records-as-the-source-of-truth.md) | Slot Records as the Source of Truth | Accepted | 2026-08-24 | | [ECS-WIDGETS-0023](ECS-WIDGETS-0023-widget-entities-with-a-legacy-layer.md) | Widget Entities with a Legacy Layer | Proposed | 2026-08-26 | +| [EXECUTION-ROUTING-0043](EXECUTION-ROUTING-0043-workflow-ownership-for-websocket-frames.md) | Workflow Ownership for Execution WebSocket Frames | Proposed | 2026-10-17 | | [EXTENSIONS-PUBLIC-API-0005](EXTENSIONS-PUBLIC-API-0005-bundle-vue-dependencies-in-extensions.md) | Bundle Vue Dependencies in Extensions | Accepted | 2025-12-13 | | [GRAPH-DOCUMENT-0024](GRAPH-DOCUMENT-0024-graph-activation-and-document-objects-for-in-app-agent-targets.md) | Graph Activation and Document Objects for In-App Agent Targets | Proposed | 2026-08-28 | | [GRAPH-DOCUMENT-0026](GRAPH-DOCUMENT-0026-frontend-document-model.md) | Frontend Document Model | Proposed | 2026-08-31 | From db6e22c6adccdb5fdeb0808736cb1a1b8602d06d Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 17 Oct 2026 10:23:50 -0700 Subject: [PATCH 51/52] test(e2e): clean up the two-tab suite's saved workflows, and name its phases The devtools endpoint behind setupWorkflowsDirectory only creates and overwrites; it never deletes. The user id is playwright-test-${parallelIndex}, so it is per worker rather than per test, and these three files were accumulating in the worker's user directory and outliving the suite. browser_tests/README.md asks the owning fixture to clean up saved workflows, so an afterEach deletes them through UserDataHelper, whose delete already tolerates a 404. Also wrapped the three phases of the longest test in test.step, so a CI failure names the phase rather than a line number. I cannot run these specs here, as the e2e backend is not up on this box and is not this session's to start, so CI is the check on both. --- .../execution/twoTabExecutionLeak.spec.ts | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts index 4430d743ae4..9c34111833b 100644 --- a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts @@ -1,6 +1,7 @@ import { comfyExpect as expect } from '@e2e/fixtures/ComfyPage' import { executionFixture as test } from '@e2e/fixtures/execution' import { imageOutput } from '@e2e/fixtures/helpers/ExecutionHelper' +import { UserDataHelper } from '@e2e/fixtures/helpers/UserDataHelper' const KSAMPLER_NODE = '3' const SAVE_IMAGE_NODE = '9' @@ -20,6 +21,22 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) }) + // The devtools endpoint behind setupWorkflowsDirectory only creates and + // overwrites, and the user id is `playwright-test-${parallelIndex}`, so these + // files would otherwise accumulate in the worker's user directory and outlive + // the suite. browser_tests/README.md asks the owning fixture to clean up + // saved workflows. + test.afterEach(async ({ comfyPage }) => { + const userData = new UserDataHelper( + comfyPage.request, + comfyPage.id, + comfyPage.url + ) + for (const name of [WORKFLOW_A, WORKFLOW_B, WORKFLOW_A_COPY]) { + await userData.delete(`workflows/${name}.json`) + } + }) + test('switching back shows the finished run, not a stuck one', async ({ comfyPage, execution: exec, @@ -68,25 +85,26 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) .toEqual({ progress: 0.25, outlined: true }) - await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) - - expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false + await test.step('B shows nothing of A run', async () => { + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual( + { progress: null, outlined: false } + ) }) - simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) - - await comfyPage.workflow.switchToTab(WORKFLOW_A) - - await expect - .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) - .toEqual({ progress: 0.75, outlined: true }) + await test.step('A keeps advancing while B is in front', async () => { + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) + }) - await comfyPage.workflow.switchToTab(WORKFLOW_B) - expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ - progress: null, - outlined: false + await test.step('returning to B is still clean', async () => { + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual( + { progress: null, outlined: false } + ) }) }) From ccebbfa43a44139289c92934f5d941a6bd784a46 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 18 Oct 2026 14:00:09 -0700 Subject: [PATCH 52/52] docs(adr): correct the fallback description, which was wrong Review caught that the ADR called the last resolution step permissive. It is not. When ownership is 'unresolved', belongsToActiveWorkflow returns `!activeJobId.value || jobId === activeJobId.value`, so a foreign job's unresolvable frame is dropped while another job is active. Only a frame with no prompt_id is delivered unconditionally. That mattered more than a wording nit: the consequences section claimed a job from another browser session reaches whatever tab is in front, which overstated the exposure. Corrected in both places, along with two sentences of review history that a future reader of the ADR does not need. Also a possessive in a test.step label, which shows up in the failure report, and a comment whose justification the README already carries. --- browser_tests/tests/execution/twoTabExecutionLeak.spec.ts | 7 +------ ...OUTING-0045-workflow-ownership-for-websocket-frames.md | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts index 9c34111833b..ce27fc72cc3 100644 --- a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts +++ b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts @@ -21,11 +21,6 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) }) - // The devtools endpoint behind setupWorkflowsDirectory only creates and - // overwrites, and the user id is `playwright-test-${parallelIndex}`, so these - // files would otherwise accumulate in the worker's user directory and outlive - // the suite. browser_tests/README.md asks the owning fixture to clean up - // saved workflows. test.afterEach(async ({ comfyPage }) => { const userData = new UserDataHelper( comfyPage.request, @@ -85,7 +80,7 @@ test.describe('cross-tab execution leak', { tag: '@ui' }, () => { .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) .toEqual({ progress: 0.25, outlined: true }) - await test.step('B shows nothing of A run', async () => { + await test.step("B shows nothing of A's run", async () => { await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual( { progress: null, outlined: false } diff --git a/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md b/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md index eeeeb163dc8..c2bdd8635c7 100644 --- a/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md +++ b/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md @@ -12,17 +12,17 @@ One `LGraph` and one `LGraphCanvas` serve every open workflow tab, so execution [ADR-GRAPH-DOCUMENT-0026](GRAPH-DOCUMENT-0026-frontend-document-model.md) D2 already states the rule: a frame arriving on a push channel resolves its target document at ingress via the job map, never via the active pointer. It also states what to do at the boundary — "events with no job→uid resolution (page reloaded mid-run, jobs queued by another client) go to an explicit _unattributed_ bucket rather than being applied to the active document by default … misdelivery is not the fallback." -This change implements D2's addressing rule and **departs from its boundary rule**. That departure is the reason this ADR exists; the reviewer on #20045 was right that nothing recorded it. +This change implements D2's addressing rule and **departs from its boundary rule**. ## Decision -**Resolution order, strongest first.** A frame's owner is resolved by the open tab's `instanceId`, then the session path, then the workflow's graph id, then permissive. +**Resolution order, strongest first.** A frame's owner is resolved by the open tab's `instanceId`, then the session path, then the workflow's graph id. When none of these resolves, the `activeJobId` guard decides. The graph id is deliberately the weakest signal, not the strongest. `ensureWorkflowId` keeps the id already present in a pasted graph, so copy a workflow, open a tab and paste gives two tabs carrying one id; loading a single workflow has also been measured leaving two open entries holding it. Keying on the graph id first is the bug QA found, where every tab sharing an id matched every job. `instanceId` is the only identifier unique per open tab, and it already existed. **The all-zero UUID is not an id.** `LGraph._id` defaults to it and the codebase reads it as "no id yet", replacing it on configure and on clear. It is not sent as a routing key, and an incoming one is discarded rather than compared. It still reaches the wire, because core falls back to the id inside `extra_data.extra_pnginfo.workflow`, and comparing it as a real id rejected a fresh graph's own frames. -**Unresolvable frames are applied to the active tab.** This is the documented exception to D2's boundary rule. When no mapping and no usable id exist, the frame is delivered rather than quarantined. +**Unresolvable frames fall back to the active job.** This is the documented exception to D2's boundary rule. When no mapping and no usable id exist, the frame is delivered to the tab in front if no job is active or the frame belongs to `activeJobId`, rather than quarantined. A frame with no `prompt_id` is always delivered. **Keys decide gating, not foreignness.** A frame being from another tab is not sufficient reason to drop work. What matters is the key the consumer reads: @@ -44,7 +44,7 @@ The graph id is deliberately the weakest signal, not the strongest. `ensureWorkf ## Consequences -**The permissive fallback is load-bearing and is also the weakest point.** Single-tab behaviour after a reload is preserved, and a job from another browser session is delivered to whatever is in front. If two tabs are open and a frame is unresolvable, it can still land on the wrong one. The blast radius is bounded by how rarely both conditions hold at once, not by the design. +**The `activeJobId` fallback is load-bearing and is also the weakest point.** Single-tab behaviour after a reload is preserved. A job from another browser session reaches the tab in front only when nothing else is running or it is itself the active job, so it is narrower than a blanket permissive rule. What remains is that with two tabs open and a frame unresolvable, it can still land on the wrong one. The blast radius is bounded by how rarely those conditions coincide, not by the design. **`status` and `progress_text` frames stay unattributable.** `status` carries no `prompt_id` because one server-wide socket serves every workflow. `progress_text` is a packed binary frame that `send_sync` cannot stamp; the frontend already reads a `supports_progress_text_metadata` server flag that no core version sets. Tracked as Comfy-Org/ComfyUI#16887. Until it lands, the text-preview widget gate goes permissive.