diff --git a/browser_tests/assets/execution/workflow_with_id.json b/browser_tests/assets/execution/workflow_with_id.json new file mode 100644 index 00000000000..669efb97138 --- /dev/null +++ b/browser_tests/assets/execution/workflow_with_id.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 9, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [413, 389], + "size": [425.27801513671875, 180.6060791015625], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [6], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": ["text, watermark"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [415, 186], + "size": [422.84503173828125, 164.31304931640625], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [4], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [473, 609], + "size": [315, 106], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + { + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [7], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1209, 188], + "size": [210, 46], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [9], + "slot_index": 0 + } + ], + "properties": {} + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1451, 189], + "size": [210, 26], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {} + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [26, 474], + "size": [315, 98], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [1], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [3, 5], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [8], + "slot_index": 2 + } + ], + "properties": { + "models": [ + { + "name": "v1-5-pruned-emaonly-fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + "directory": "checkpoints" + } + ] + }, + "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] + } + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 5, 0, 3, 3, "LATENT"], + [3, 4, 1, 6, 0, "CLIP"], + [4, 6, 0, 3, 1, "CONDITIONING"], + [5, 4, 1, 7, 0, "CLIP"], + [6, 7, 0, 3, 2, "CONDITIONING"], + [7, 3, 0, 8, 0, "LATENT"], + [8, 4, 2, 8, 1, "VAE"], + [9, 8, 0, 9, 0, "IMAGE"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "offset": [0, 0], + "scale": 1 + } + }, + "version": 0.4, + "id": "0199e3a3-6c01-7000-8000-aaaaaaaaaaaa" +} diff --git a/browser_tests/assets/execution/workflow_with_id_b.json b/browser_tests/assets/execution/workflow_with_id_b.json new file mode 100644 index 00000000000..a6804a22c21 --- /dev/null +++ b/browser_tests/assets/execution/workflow_with_id_b.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 9, + "last_link_id": 9, + "nodes": [ + { + "id": 7, + "type": "CLIPTextEncode", + "pos": [413, 389], + "size": [425.27801513671875, 180.6060791015625], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 5 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [6], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": ["text, watermark"] + }, + { + "id": 6, + "type": "CLIPTextEncode", + "pos": [415, 186], + "size": [422.84503173828125, 164.31304931640625], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 3 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [4], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [ + "beautiful scenery nature glass bottle landscape, , purple galaxy bottle," + ] + }, + { + "id": 5, + "type": "EmptyLatentImage", + "pos": [473, 609], + "size": [315, 106], + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [2], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [512, 512, 1] + }, + { + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 1 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 4 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 2 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [7], + "slot_index": 0 + } + ], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [1209, 188], + "size": [210, 46], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [9], + "slot_index": 0 + } + ], + "properties": {} + }, + { + "id": 9, + "type": "SaveImage", + "pos": [1451, 189], + "size": [210, 26], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 9 + } + ], + "properties": {} + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [26, 474], + "size": [315, 98], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [1], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [3, 5], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [8], + "slot_index": 2 + } + ], + "properties": { + "models": [ + { + "name": "v1-5-pruned-emaonly-fp16.safetensors", + "url": "https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/resolve/main/v1-5-pruned-emaonly-fp16.safetensors", + "directory": "checkpoints" + } + ] + }, + "widgets_values": ["v1-5-pruned-emaonly-fp16.safetensors"] + } + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 5, 0, 3, 3, "LATENT"], + [3, 4, 1, 6, 0, "CLIP"], + [4, 6, 0, 3, 1, "CONDITIONING"], + [5, 4, 1, 7, 0, "CLIP"], + [6, 7, 0, 3, 2, "CONDITIONING"], + [7, 3, 0, 8, 0, "LATENT"], + [8, 4, 2, 8, 1, "VAE"], + [9, 8, 0, 9, 0, "IMAGE"] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "offset": [0, 0], + "scale": 1 + } + }, + "version": 0.4, + "id": "0199e3a3-6c01-7000-8000-bbbbbbbbbbbb" +} diff --git a/browser_tests/fixtures/components/SidebarTab.ts b/browser_tests/fixtures/components/SidebarTab.ts index 08e5b049b5d..90def6486bc 100644 --- a/browser_tests/fixtures/components/SidebarTab.ts +++ b/browser_tests/fixtures/components/SidebarTab.ts @@ -221,10 +221,11 @@ export class WorkflowsSidebarTab extends SidebarTab { }) } - getPersistedItem(name: string) { - return this.root.locator('.comfyui-workflows-browse .node-label', { - hasText: name - }) + getPersistedItem(name: string, options: { exact?: boolean } = {}) { + const browse = this.root.locator('.comfyui-workflows-browse') + return options.exact + ? browse.getByText(name, { exact: true }) + : browse.locator('.node-label', { hasText: name }) } async renameWorkflow(locator: Locator, newName: string) { diff --git a/browser_tests/fixtures/execution.ts b/browser_tests/fixtures/execution.ts new file mode 100644 index 00000000000..a456954b8ad --- /dev/null +++ b/browser_tests/fixtures/execution.ts @@ -0,0 +1,20 @@ +import { mergeTests } from '@playwright/test' + +import { comfyPageFixture } from '@e2e/fixtures/ComfyPage' +import { BackendSimulator } from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' +import { webSocketFixture } from '@e2e/fixtures/ws' + +const base = mergeTests(comfyPageFixture, webSocketFixture) + +export const executionFixture = base.extend<{ + execution: ExecutionHelper + backendSimulator: BackendSimulator +}>({ + execution: async ({ comfyPage, getWebSocket }, use) => { + await use(new ExecutionHelper(comfyPage, await getWebSocket())) + }, + backendSimulator: async ({ execution }, use) => { + await use(new BackendSimulator(execution)) + } +}) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.test.ts b/browser_tests/fixtures/helpers/BackendSimulator.test.ts new file mode 100644 index 00000000000..ca542a1cce5 --- /dev/null +++ b/browser_tests/fixtures/helpers/BackendSimulator.test.ts @@ -0,0 +1,398 @@ +import type { WebSocketRoute } from '@playwright/test' +import { fromPartial } from '@total-typescript/shoehorn' +import { describe, expect, it } from 'vitest' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { Frame } from '@e2e/fixtures/helpers/BackendSimulator' +import { + BackendSimulator, + dropFrames, + duplicateFrames, + interleave, + swapFrames +} from '@e2e/fixtures/helpers/BackendSimulator' +import { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' + +interface SentFrame { + type: string + data: Record +} + +const OVERLAP_SEEDS = [0, 3] + +function harness() { + const sent: SentFrame[] = [] + const binary: Buffer[] = [] + const ws = fromPartial({ + send: (message: string | Buffer) => { + if (typeof message === 'string') sent.push(JSON.parse(message)) + else binary.push(message) + } + }) + const execution = new ExecutionHelper(fromPartial({}), ws) + return { sent, binary, simulator: new BackendSimulator(execution), execution } +} + +function positionsOf(sent: SentFrame[], workflowId: string): number[] { + return sent.flatMap((frame, index) => + frame.data.workflow_id === workflowId ? [index] : [] + ) +} + +describe('BackendSimulator frame scripting', () => { + it('stamps each prompt with its own workflow id', () => { + const { sent, simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + + simulator.play([a.start(), b.start(), a.success(), b.success()]) + + expect( + sent.map((frame) => [ + frame.type, + frame.data.prompt_id, + frame.data.workflow_id + ]) + ).toEqual([ + ['execution_start', 'job-a', 'wf-a'], + ['execution_start', 'job-b', 'wf-b'], + ['execution_success', 'job-a', 'wf-a'], + ['execution_success', 'job-b', 'wf-b'] + ]) + }) + + it('omits workflow_id for a prompt with no metadata (legacy backend)', () => { + const { sent, simulator } = harness() + const legacy = simulator.prompt('job-a') + + simulator.play([legacy.start(), ...legacy.nodeRunning('1', 1, 4)]) + + expect(sent).toHaveLength(3) + expect(sent.every((frame) => !('workflow_id' in frame.data))).toBe(true) + }) + + it('leaves the metadata scope clean after each frame', () => { + const { sent, simulator, execution } = harness() + const scoped = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + simulator.play([scoped.start()]) + // Straight through `execution`, outside any scope. Going through a second + // prompt masked the defect, because that prompt opened its own scope and + // set the metadata itself, so a scope that never restored still passed. + execution.executionStart('job-b') + + expect(sent[0].data.workflow_id).toBe('wf-a') + expect(sent[1].data).not.toHaveProperty('workflow_id') + }) + + it('never lets metadata overwrite a frame field', () => { + const { sent, execution } = harness() + execution.withMetadata( + { workflow_id: 'wf-a', prompt_id: 'spoofed', node: 'spoofed' }, + () => execution.executing('job-a', '7') + ) + + expect(sent[0].data).toEqual({ + workflow_id: 'wf-a', + prompt_id: 'job-a', + node: '7', + display_node: '7' + }) + }) + + it('does not stamp status frames, even inside an active scope', () => { + const { sent, execution } = harness() + + execution.withWorkflowId('wf-a', () => { + execution.executionStart('job-a') + execution.status(2) + }) + + expect(sent[0].data.workflow_id).toBe('wf-a') + expect(sent[1].type).toBe('status') + expect(sent[1].data).not.toHaveProperty('workflow_id') + }) + + it('stamps validation errors and binary previews with workflow ownership', () => { + const { sent, binary, execution } = harness() + + execution.withWorkflowId('wf-a', () => { + execution.validationError('job-a', '1', {}) + execution.latentPreview('job-a', '1') + }) + + expect(sent[0].data.workflow_id).toBe('wf-a') + const metadataLength = binary[0].readUInt32BE(4) + const metadata = JSON.parse( + binary[0].subarray(8, 8 + metadataLength).toString() + ) + expect(metadata).toMatchObject({ + prompt_id: 'job-a', + workflow_id: 'wf-a' + }) + }) + + it('duplicates a frame', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + + simulator.play( + duplicateFrames( + [prompt.start(), prompt.success()], + 'wf-a:execution_success' + ) + ) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'execution_success', + 'execution_success' + ]) + }) + + it('swaps two frames to produce out-of-order arrival', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start(), prompt.executed('1', {}), prompt.success()] + + simulator.play( + swapFrames(script, 'wf-a:executed', 'wf-a:execution_success') + ) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'execution_success', + 'executed' + ]) + }) + + it.for([ + { + name: 'an unknown swap target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => + swapFrames([prompt.start()], 'wf-a:execution_start', 'wf-a:nope') + }, + message: /not in the script/ + }, + { + name: 'an ambiguous swap target', + makeCall: () => { + const { simulator } = harness() + const first = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const second = simulator.prompt('job-b', { workflowId: 'wf-a' }) + return () => + swapFrames( + [first.start(), second.start(), first.success()], + 'wf-a:execution_start', + 'wf-a:execution_success' + ) + }, + message: /matches 2 frames/ + }, + { + name: 'the same frame twice', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + const script = [prompt.start(), prompt.success()] + return () => + swapFrames(script, 'wf-a:execution_start', 'wf-a:execution_start') + }, + message: /same frame/ + }, + { + name: 'an unknown drop target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => dropFrames([prompt.start()], 'wf-a:execution_success') + }, + message: /not in the script/ + }, + { + name: 'an unknown duplicate target', + makeCall: () => { + const prompt = harness().simulator.prompt('job-a', { + workflowId: 'wf-a' + }) + return () => duplicateFrames([prompt.start()], 'wf-a:progress') + }, + message: /not in the script/ + } + ])('throws for $name', ({ makeCall, message }) => { + expect(makeCall()).toThrow(message) + }) +}) + +describe('BackendSimulator frame granularity', () => { + function everyFrame(simulator: BackendSimulator): Frame[] { + const p = simulator.prompt('job-a', { workflowId: 'wf-a' }) + return [ + p.start(), + p.executing('1'), + ...p.nodeRunning('1', 1, 4), + p.executed('1', {}), + p.success(), + p.latentPreview('1') + ] + } + + it('sends exactly one WebSocket frame per scheduled Frame', () => { + let sends = 0 + const ws = fromPartial({ + send: () => { + sends++ + } + }) + const execution = new ExecutionHelper(fromPartial({}), ws) + const simulator = new BackendSimulator(execution) + const script = everyFrame(simulator) + + simulator.play(script) + + expect(sends).toBe(script.length) + }) + + it('gives a frame builder a label matching the event it sends', () => { + const { sent, simulator } = harness() + const script = everyFrame(simulator) + + simulator.play(script) + + expect(sent.map((frame) => frame.type)).toEqual( + script.slice(0, -1).map((frame) => frame.label.split(':')[1]) + ) + }) + + it('labels the binary frame with the event it dispatches', () => { + const { simulator } = harness() + const script = everyFrame(simulator) + + expect(script.at(-1)?.label).toBe('wf-a:b_preview_with_metadata') + }) + + it('scripts the two halves of a running node independently', () => { + const { sent, simulator } = harness() + const prompt = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [prompt.start(), ...prompt.nodeRunning('1', 1, 4)] + + simulator.play(dropFrames(script, 'wf-a:progress')) + + expect(sent.map((frame) => frame.type)).toEqual([ + 'execution_start', + 'progress_state' + ]) + }) +}) + +describe('BackendSimulator interleaving', () => { + function twoPrompts(seed: number) { + const h = harness() + const a = h.simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = h.simulator.prompt('job-b', { workflowId: 'wf-b' }) + h.simulator.play( + interleave( + [a.start(), a.executed('1', {}), a.success()], + [b.start(), b.executed('2', {}), b.success()], + seed + ) + ) + return h.sent + } + + it('produces the documented overlapping order for seed 7', () => { + expect( + twoPrompts(7).map((frame) => `${frame.data.workflow_id}:${frame.type}`) + ).toEqual([ + 'wf-a:execution_start', + 'wf-a:executed', + 'wf-b:execution_start', + 'wf-a:execution_success', + 'wf-b:executed', + 'wf-b:execution_success' + ]) + }) + + it.for(OVERLAP_SEEDS)( + 'keeps both prompts in flight at once for seed %i', + (seed) => { + const sent = twoPrompts(seed) + const a = positionsOf(sent, 'wf-a') + const b = positionsOf(sent, 'wf-b') + + expect(a[0]).toBeLessThan(b[b.length - 1]) + expect(b[0]).toBeLessThan(a[a.length - 1]) + } + ) + + it.for(OVERLAP_SEEDS)( + "preserves each prompt's own dependency order for seed %i", + (seed) => { + const sent = twoPrompts(seed) + function typesOf(workflowId: string) { + return sent + .filter((frame) => frame.data.workflow_id === workflowId) + .map((frame) => frame.type) + } + + expect(typesOf('wf-a')).toEqual([ + 'execution_start', + 'executed', + 'execution_success' + ]) + expect(typesOf('wf-b')).toEqual([ + 'execution_start', + 'executed', + 'execution_success' + ]) + } + ) + + it('interleaves deterministically for a given seed', () => { + const first = twoPrompts(7).map((frame) => frame.data.workflow_id) + const second = twoPrompts(7).map((frame) => frame.data.workflow_id) + + expect(second).toEqual(first) + expect(first).toHaveLength(6) + }) + + it('gives different seeds different schedules', () => { + const schedules = new Set( + OVERLAP_SEEDS.map((seed) => + twoPrompts(seed) + .map((frame) => frame.data.workflow_id) + .join(',') + ) + ) + + expect(schedules.size).toBe(2) + }) + + it('throws rather than returning a schedule that cannot overlap', () => { + const { simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const script = [a.start(), a.success()] + + expect(() => interleave(script, [], 7)).toThrow(/both sides/) + expect(() => interleave([], script, 7)).toThrow(/both sides/) + expect(() => interleave([script[0]], [script[1]], 7)).toThrow( + /at least three/ + ) + }) + + it.for([11.9, 2 ** 32 + 11])('rejects unsupported seed %s', (seed) => { + const { simulator } = harness() + const a = simulator.prompt('job-a', { workflowId: 'wf-a' }) + const b = simulator.prompt('job-b', { workflowId: 'wf-b' }) + + expect(() => + interleave([a.start(), a.success()], [b.start()], seed) + ).toThrow(/signed 32-bit integer/) + }) +}) diff --git a/browser_tests/fixtures/helpers/BackendSimulator.ts b/browser_tests/fixtures/helpers/BackendSimulator.ts new file mode 100644 index 00000000000..e9004c78c66 --- /dev/null +++ b/browser_tests/fixtures/helpers/BackendSimulator.ts @@ -0,0 +1,174 @@ +import type { ExecutionHelper } from '@e2e/fixtures/helpers/ExecutionHelper' + +export interface Frame { + readonly label: string + readonly send: () => void +} + +export type FrameSelector = string + +function indicesOf(frames: Frame[], selector: FrameSelector): number[] { + return frames.flatMap((frame, index) => + frame.label === selector ? [index] : [] + ) +} + +function requireMatches( + frames: Frame[], + selector: FrameSelector, + helper: string +): number[] { + const found = indicesOf(frames, selector) + if (found.length === 0) { + throw new Error(`${helper}: ${selector} is not in the script`) + } + return found +} + +export class SimulatedPrompt { + constructor( + private readonly execution: ExecutionHelper, + readonly jobId: string, + readonly workflowId?: string + ) {} + + private frame(type: string, send: () => void): Frame { + return { + label: `${this.workflowId ?? this.jobId}:${type}`, + send: () => this.execution.withWorkflowId(this.workflowId, send) + } + } + + start(): Frame { + return this.frame('execution_start', () => + this.execution.executionStart(this.jobId) + ) + } + + executing(nodeId: string | null): Frame { + return this.frame('executing', () => + this.execution.executing(this.jobId, nodeId) + ) + } + + nodeRunning(nodeId: string, value: number, max: number): [Frame, Frame] { + return [ + this.frame('progress_state', () => + this.execution.progressState( + this.jobId, + this.execution.runningNodeState(this.jobId, nodeId, value, max) + ) + ), + this.frame('progress', () => + this.execution.progress(this.jobId, nodeId, value, max) + ) + ] + } + + executed( + nodeId: string, + output: Record | null | undefined + ): Frame { + return this.frame('executed', () => + this.execution.executed(this.jobId, nodeId, output) + ) + } + + success(): Frame { + return this.frame('execution_success', () => + this.execution.executionSuccess(this.jobId) + ) + } + + latentPreview(nodeId: string): Frame { + return this.frame('b_preview_with_metadata', () => + this.execution.latentPreview(this.jobId, nodeId) + ) + } +} + +export class BackendSimulator { + constructor(private readonly execution: ExecutionHelper) {} + + prompt( + jobId: string, + options: { workflowId?: string } = {} + ): SimulatedPrompt { + return new SimulatedPrompt(this.execution, jobId, options.workflowId) + } + + play(frames: Frame[]): void { + for (const frame of frames) frame.send() + } +} + +export function dropFrames(frames: Frame[], selector: FrameSelector): Frame[] { + const drop = new Set(requireMatches(frames, selector, 'dropFrames')) + return frames.filter((_, index) => !drop.has(index)) +} + +export function duplicateFrames( + frames: Frame[], + selector: FrameSelector +): Frame[] { + const duplicate = new Set(requireMatches(frames, selector, 'duplicateFrames')) + return frames.flatMap((frame, index) => + duplicate.has(index) ? [frame, frame] : [frame] + ) +} + +export function swapFrames( + frames: Frame[], + first: FrameSelector, + second: FrameSelector +): Frame[] { + const a = requireMatches(frames, first, 'swapFrames') + const b = requireMatches(frames, second, 'swapFrames') + for (const [selector, found] of [ + [first, a], + [second, b] + ] as const) { + if (found.length > 1) { + throw new Error( + `swapFrames: ${selector} matches ${found.length} frames; ` + + 'pick a label that matches exactly one frame' + ) + } + } + if (a[0] === b[0]) { + throw new Error( + `swapFrames: ${first} and ${second} ` + + 'select the same frame; swapping it with itself applies no fault' + ) + } + const swapped = [...frames] + ;[swapped[a[0]], swapped[b[0]]] = [swapped[b[0]], swapped[a[0]]] + return swapped +} + +export function interleave(a: Frame[], b: Frame[], seed: number): Frame[] { + if (!Number.isInteger(seed) || seed !== (seed | 0)) { + throw new Error( + `interleave: seed must be a signed 32-bit integer, got ${seed}` + ) + } + if (a.length === 0 || b.length === 0 || a.length + b.length < 3) { + throw new Error( + `interleave: needs frames on both sides and at least three in total to overlap, got ${a.length} and ${b.length}` + ) + } + const out: Frame[] = [] + let state = seed | 0 + let i = 0 + let j = 0 + while (i < a.length || j < b.length) { + // Math.imul: the plain product exceeds 2^53 and rounds the low bits away. + state = (Math.imul(state, 1103515245) + 12345) | 0 + let takeA = j >= b.length || (i < a.length && ((state >>> 16) & 1) === 0) + if (takeA && b.length > 0 && j === 0 && i === a.length - 1) takeA = false + else if (!takeA && a.length > 0 && i === 0 && j === b.length - 1) + takeA = true + out.push(takeA ? a[i++] : b[j++]) + } + return out +} diff --git a/browser_tests/fixtures/helpers/ExecutionHelper.ts b/browser_tests/fixtures/helpers/ExecutionHelper.ts index ec6e056bb66..2606c29374f 100644 --- a/browser_tests/fixtures/helpers/ExecutionHelper.ts +++ b/browser_tests/fixtures/helpers/ExecutionHelper.ts @@ -17,6 +17,10 @@ type RunOptions = { beforePromptResponse?: (jobId: string) => void | Promise } +export function imageOutput(filename: string) { + return { images: [{ filename, subfolder: '', type: 'output' }] } +} + /** * Build a `NodeError` describing a single failed input on a KSampler node. * Shared between specs that surface validation rings via 400 responses. @@ -45,6 +49,7 @@ export function buildKSamplerError( */ export class ExecutionHelper { private jobCounter = 0 + private metadata: Record | undefined private readonly completedJobs: RawJobListItem[] = [] private readonly page: ComfyPage['page'] private readonly command: ComfyPage['command'] @@ -69,6 +74,34 @@ export class ExecutionHelper { return this.ws } + withWorkflowId(workflowId: string | undefined, send: () => void): void { + this.withMetadata( + workflowId === undefined ? undefined : { workflow_id: workflowId }, + send + ) + } + + withMetadata( + metadata: Record | undefined, + send: () => void + ): void { + const previous = this.metadata + this.metadata = metadata + try { + send() + } finally { + this.metadata = previous + } + } + + private emit(type: string, data: Record): void { + const payload = + this.metadata && 'prompt_id' in data + ? { ...this.metadata, ...data } + : data + this.requireWs().send(JSON.stringify({ type, data: payload })) + } + /** * Intercept POST /api/prompt, execute Comfy.QueuePrompt, and return * the synthetic job ID. @@ -139,12 +172,14 @@ export class ExecutionHelper { * Encodes the metadata and a tiny 1x1 PNG so the app creates a blob URL. */ latentPreview(jobId: string, nodeId: string): void { + const workflowId = this.metadata?.workflow_id const metadata = JSON.stringify({ node_id: nodeId, display_node_id: nodeId, parent_node_id: nodeId, real_node_id: nodeId, prompt_id: jobId, + ...(typeof workflowId === 'string' && { workflow_id: workflowId }), image_type: 'image/png' }) const metadataBytes = new TextEncoder().encode(metadata) @@ -168,22 +203,16 @@ export class ExecutionHelper { /** Send `execution_start` WS event. */ executionStart(jobId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_start', - data: { prompt_id: jobId, timestamp: Date.now() } - }) - ) + this.emit('execution_start', { prompt_id: jobId, timestamp: Date.now() }) } /** Send `executing` WS event to signal which node is currently running. */ executing(jobId: string, nodeId: string | null): void { - this.requireWs().send( - JSON.stringify({ - type: 'executing', - data: { prompt_id: jobId, node: nodeId } - }) - ) + this.emit('executing', { + prompt_id: jobId, + node: nodeId, + ...(nodeId !== null && { display_node: nodeId }) + }) } /** Send `executed` WS event with node output. */ @@ -192,45 +221,30 @@ export class ExecutionHelper { nodeId: string, output: Record | null | undefined ): void { - this.requireWs().send( - JSON.stringify({ - type: 'executed', - data: { - prompt_id: jobId, - node: nodeId, - display_node: nodeId, - output - } - }) - ) + this.emit('executed', { + prompt_id: jobId, + node: nodeId, + display_node: nodeId, + output + }) } /** Send `execution_success` WS event. */ executionSuccess(jobId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_success', - data: { prompt_id: jobId, timestamp: Date.now() } - }) - ) + this.emit('execution_success', { prompt_id: jobId, timestamp: Date.now() }) } /** Send `execution_error` WS event. */ executionError(jobId: string, nodeId: string, message: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_error', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - exception_message: message, - exception_type: 'RuntimeError', - traceback: [] - } - }) - ) + this.emit('execution_error', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + exception_message: message, + exception_type: 'RuntimeError', + traceback: [] + }) } /** Send `execution_error` WS event carrying cloud validation node errors. */ @@ -239,46 +253,50 @@ export class ExecutionHelper { nodeId: string, nodeErrors: Record ): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_error', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - exception_message: JSON.stringify({ node_errors: nodeErrors }), - exception_type: 'prompt_outputs_failed_validation', - traceback: [] - } - }) - ) + this.emit('execution_error', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + exception_message: JSON.stringify({ node_errors: nodeErrors }), + exception_type: 'prompt_outputs_failed_validation', + traceback: [] + }) } /** Send `execution_interrupted` WS event (user-initiated stop). */ executionInterrupted(jobId: string, nodeId: string): void { - this.requireWs().send( - JSON.stringify({ - type: 'execution_interrupted', - data: { - prompt_id: jobId, - timestamp: Date.now(), - node_id: nodeId, - node_type: 'Unknown', - executed: [] - } - }) - ) + this.emit('execution_interrupted', { + prompt_id: jobId, + timestamp: Date.now(), + node_id: nodeId, + node_type: 'Unknown', + executed: [] + }) } /** Send `progress` WS event. */ progress(jobId: string, nodeId: string, value: number, max: number): void { - this.requireWs().send( - JSON.stringify({ - type: 'progress', - data: { prompt_id: jobId, node: nodeId, value, max } - }) - ) + this.emit('progress', { prompt_id: jobId, node: nodeId, value, max }) + } + + runningNodeState( + jobId: string, + nodeId: string, + value: number, + max: number + ): Record { + return { + [nodeId]: { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state: 'running', + value, + max + } + } } /** @@ -286,27 +304,13 @@ export class ExecutionHelper { * emitting both the `progress_state` and `progress` events the backend sends. */ nodeRunning(jobId: string, nodeId: string, value: number, max: number): void { - const state: NodeProgressState = { - node_id: nodeId, - display_node_id: nodeId, - real_node_id: nodeId, - prompt_id: jobId, - state: 'running', - value, - max - } - this.progressState(jobId, { [nodeId]: state }) + this.progressState(jobId, this.runningNodeState(jobId, nodeId, value, max)) this.progress(jobId, nodeId, value, max) } /** Send `progress_state` WS event with per-node execution state. */ progressState(jobId: string, nodes: Record): void { - this.requireWs().send( - JSON.stringify({ - type: 'progress_state', - data: { prompt_id: jobId, nodes } - }) - ) + this.emit('progress_state', { prompt_id: jobId, nodes }) } /** @@ -341,11 +345,8 @@ export class ExecutionHelper { /** Send `status` WS event to update queue count. */ status(queueRemaining: number): void { - this.requireWs().send( - JSON.stringify({ - type: 'status', - data: { status: { exec_info: { queue_remaining: queueRemaining } } } - }) - ) + this.emit('status', { + status: { exec_info: { queue_remaining: queueRemaining } } + }) } } diff --git a/browser_tests/fixtures/helpers/NodeOperationsHelper.ts b/browser_tests/fixtures/helpers/NodeOperationsHelper.ts index 976b43e91f9..5fd3acf9855 100644 --- a/browser_tests/fixtures/helpers/NodeOperationsHelper.ts +++ b/browser_tests/fixtures/helpers/NodeOperationsHelper.ts @@ -51,6 +51,15 @@ export class NodeOperationsHelper { return selectedNodeIds.map(toNodeId) } + async getExecutionRender(nodeId: string) { + return this.page.evaluate((id) => { + const node = window.app!.graph.getNodeById(id) + if (!node) return null + const stroke = node.strokeStyles['running']?.call(node) + return { progress: node.progress ?? null, outlined: stroke !== undefined } + }, toNodeId(nodeId)) + } + /** * Add a node to the graph by type. * @param type - The node type (e.g. 'KSampler', 'VAEDecode') diff --git a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts index 272ae43b0f5..bef3e0e4c44 100644 --- a/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts +++ b/browser_tests/fixtures/helpers/Preview3DPipelineFixture.ts @@ -215,11 +215,9 @@ export class Preview3DPipelineContext { } async openPersistedWorkflowFromSidebar(workflowName: string): Promise { - const tab = this.comfyPage.menu.workflowsTab - await tab.open() - await tab.getPersistedItem(workflowName).click() - await this.comfyPage.workflow.waitForWorkflowIdle(30_000) - await this.comfyPage.vueNodes.waitForNodes() + await this.comfyPage.workflow.openPersistedWorkflow(workflowName, { + idleTimeout: 30_000 + }) } async assertPreview3dModelPathAndLastTime(path: string): Promise { diff --git a/browser_tests/fixtures/helpers/WorkflowHelper.ts b/browser_tests/fixtures/helpers/WorkflowHelper.ts index 3035e61864e..35c460e14d0 100644 --- a/browser_tests/fixtures/helpers/WorkflowHelper.ts +++ b/browser_tests/fixtures/helpers/WorkflowHelper.ts @@ -219,6 +219,14 @@ export class WorkflowHelper { }) } + async getActiveWorkflowId(): Promise { + return this.comfyPage.page.evaluate(() => { + const workflow = (window.app!.extensionManager as WorkspaceStore).workflow + .activeWorkflow + return workflow?.activeState.id ?? workflow?.initialState.id ?? undefined + }) + } + async getActiveWorkflowActiveAppMode(): Promise { return this.comfyPage.page.evaluate(() => { return (window.app!.extensionManager as WorkspaceStore).workflow @@ -253,23 +261,21 @@ export class WorkflowHelper { }) } - async openPersistedWorkflow(workflowName: string): Promise { - await this.comfyPage.page.evaluate(async (name) => { - const store = (window.app!.extensionManager as WorkspaceStore).workflow - await store.syncWorkflows() - const workflow = - store.getWorkflowByPath(`workflows/${name}.json`) ?? - store.persistedWorkflows.find( - (candidate) => - candidate.filename === name || - candidate.path.endsWith(`${name}.json`) - ) - if (!workflow) { - throw new Error(`Persisted workflow not found: ${name}`) - } - await store.openWorkflow(workflow) - }, workflowName) - await this.waitForWorkflowIdle() + async openPersistedWorkflow( + workflowName: string, + { idleTimeout }: { idleTimeout?: number } = {} + ): Promise { + const sidebar = this.comfyPage.menu.workflowsTab + const wasOpen = await sidebar.selectedTabButton.isVisible() + await sidebar.open() + await sidebar.getPersistedItem(workflowName, { exact: true }).click() + if (!wasOpen) await sidebar.close() + await this.waitForWorkflowIdle(idleTimeout) + await expect + .poll(async () => + (await this.getActiveWorkflowPath())?.endsWith(`${workflowName}.json`) + ) + .toBe(true) await this.comfyPage.vueNodes.waitForNodes() } diff --git a/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts new file mode 100644 index 00000000000..ce27fc72cc3 --- /dev/null +++ b/browser_tests/tests/execution/twoTabExecutionLeak.spec.ts @@ -0,0 +1,212 @@ +import { comfyExpect as expect } from '@e2e/fixtures/ComfyPage' +import { executionFixture as test } from '@e2e/fixtures/execution' +import { imageOutput } from '@e2e/fixtures/helpers/ExecutionHelper' +import { UserDataHelper } from '@e2e/fixtures/helpers/UserDataHelper' + +const KSAMPLER_NODE = '3' +const SAVE_IMAGE_NODE = '9' +const WORKFLOW_A = 'exec_leak_a' +const WORKFLOW_B = 'exec_leak_b' +const WORKFLOW_A_COPY = 'pasted_twin' + +test.describe('cross-tab execution leak', { tag: '@ui' }, () => { + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.workflow.setupWorkflowsDirectory({ + [`${WORKFLOW_A}.json`]: 'execution/workflow_with_id.json', + [`${WORKFLOW_B}.json`]: 'execution/workflow_with_id_b.json', + // The same asset under a second name: two files carrying one workflow + // id, which is what a copy-paste into a new tab produces. + [`${WORKFLOW_A_COPY}.json`]: 'execution/workflow_with_id.json' + }) + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A) + }) + + test.afterEach(async ({ comfyPage }) => { + const userData = new UserDataHelper( + comfyPage.request, + comfyPage.id, + comfyPage.url + ) + for (const name of [WORKFLOW_A, WORKFLOW_B, WORKFLOW_A_COPY]) { + await userData.delete(`workflows/${name}.json`) + } + }) + + test('switching back shows the finished run, not a stuck one', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + const tabA = comfyPage.menu.topbar.getWorkflowTab(WORKFLOW_A) + + simulator.play([ + running.executed(SAVE_IMAGE_NODE, imageOutput('a.png')), + running.success() + ]) + await expect(tabA.getByRole('img', { name: 'Completed' })).toBeVisible() + await expect(tabA.getByRole('img', { name: 'Running' })).toHaveCount(0) + + await comfyPage.workflow.switchToTab(WORKFLOW_A) + + await expect( + comfyPage.menu.topbar.workflowTabs.getByRole('img', { name: 'Running' }) + ).toHaveCount(0) + }) + + test('the node the user is looking at does not take over a foreign run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.25, outlined: true }) + + await test.step("B shows nothing of A's run", async () => { + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual( + { progress: null, outlined: false } + ) + }) + + await test.step('A keeps advancing while B is in front', async () => { + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) + }) + + await test.step('returning to B is still clean', async () => { + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual( + { progress: null, outlined: false } + ) + }) + }) + + test('stays clean across repeated switches during one run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: workflowAId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + + async function switchAndAssert(step: number, arrival: number) { + await comfyPage.workflow.switchToTab(WORKFLOW_A) + simulator.play([...running.nodeRunning(KSAMPLER_NODE, step, 10)]) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: step / 10, outlined: true }) + + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect( + await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE), + `arrival ${arrival} in B must not inherit A` + ).toEqual({ progress: null, outlined: false }) + } + + await switchAndAssert(3, 1) + await switchAndAssert(7, 2) + }) + + test('stays clean while the visible workflow is queued behind the running one', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const workflowAId = await comfyPage.workflow.getActiveWorkflowId() + + const jobA = await exec.run() + const running = simulator.prompt(jobA, { workflowId: workflowAId }) + simulator.play([ + running.start(), + ...running.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_B) + + await exec.run() + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + + simulator.play([...running.nodeRunning(KSAMPLER_NODE, 3, 4)]) + await comfyPage.workflow.switchToTab(WORKFLOW_A) + await expect + .poll(() => comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)) + .toEqual({ progress: 0.75, outlined: true }) + await comfyPage.workflow.switchToTab(WORKFLOW_B) + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) + + test('a second open workflow sharing the workflow id does not receive the output', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const sharedId = await comfyPage.workflow.getActiveWorkflowId() + + const jobId = await exec.run() + const running = simulator.prompt(jobId, { workflowId: sharedId }) + simulator.play([running.start()]) + + await comfyPage.workflow.openPersistedWorkflow(WORKFLOW_A_COPY) + expect( + await comfyPage.workflow.getActiveWorkflowId(), + 'the copy must carry the same id, which is what makes this reachable' + ).toBe(sharedId) + + simulator.play([ + ...running.nodeRunning(KSAMPLER_NODE, 2, 4), + running.executed(SAVE_IMAGE_NODE, imageOutput('from-the-original.png')), + running.success() + ]) + + await expect( + comfyPage.menu.topbar + .getWorkflowTab(WORKFLOW_A) + .getByRole('img', { name: 'Completed' }) + ).toBeVisible() + + const leaked = await comfyPage.page.evaluate(() => + JSON.stringify(window.app!.nodeOutputs) + ) + expect( + leaked, + 'the original output must not land on the copy' + ).not.toContain('from-the-original.png') + expect(await comfyPage.nodeOps.getExecutionRender(KSAMPLER_NODE)).toEqual({ + progress: null, + outlined: false + }) + }) +}) diff --git a/browser_tests/tests/execution/workflowScopedExecution.spec.ts b/browser_tests/tests/execution/workflowScopedExecution.spec.ts new file mode 100644 index 00000000000..042c67ca3ac --- /dev/null +++ b/browser_tests/tests/execution/workflowScopedExecution.spec.ts @@ -0,0 +1,282 @@ +import { comfyExpect as expect } from '@e2e/fixtures/ComfyPage' +import { executionFixture as test } from '@e2e/fixtures/execution' +import { + BackendSimulator, + duplicateFrames, + interleave, + swapFrames +} from '@e2e/fixtures/helpers/BackendSimulator' +import { + ExecutionHelper, + imageOutput +} from '@e2e/fixtures/helpers/ExecutionHelper' + +const KSAMPLER_NODE = '3' +const SAVE_IMAGE_NODE = '9' +const FOREIGN_WORKFLOW_ID = '0199e3a3-6c01-7000-8000-ffffffffffff' +const WORKFLOW_WITH_ID = 'execution/workflow_with_id' + +test.describe('workflow-scoped execution', { tag: '@ui' }, () => { + let workflowId: string + + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.workflow.loadWorkflow(WORKFLOW_WITH_ID) + await comfyPage.appMode.enterAppModeWithInputs([[KSAMPLER_NODE, 'seed']]) + await expect(comfyPage.appMode.linearWidgets).toBeVisible() + const loadedWorkflowId = await comfyPage.workflow.getActiveWorkflowId() + if (!loadedWorkflowId) throw new Error('loaded workflow has no id') + workflowId = loadedWorkflowId + }) + + test('renders the visible workflow own frames', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + + const mine = simulator.prompt(jobId, { workflowId }) + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('an output from another workflow does not land on the visible tab', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([ + mine.start(), + foreign.start(), + foreign.executed(SAVE_IMAGE_NODE, imageOutput('theirs.png')), + foreign.success(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('interleaved prompts keep the visible tab on its own run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play( + interleave( + [ + mine.start(), + ...mine.nodeRunning(KSAMPLER_NODE, 1, 4), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ], + [ + foreign.start(), + ...foreign.nodeRunning(KSAMPLER_NODE, 3, 4), + foreign.executed(SAVE_IMAGE_NODE, imageOutput('theirs.png')), + foreign.success() + ], + 2 + ) + ) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + await expect( + comfyPage.appMode.outputHistory.imageOutputs.first() + ).toHaveAttribute('src', /mine\.png/) + }) + + test('an output that arrives after the terminal frame is ignored', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + + const mine = simulator.prompt(jobId, { workflowId }) + const script = [ + mine.start(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ] + const label = workflowId + + simulator.play( + swapFrames(script, `${label}:executed`, `${label}:execution_success`) + ) + await comfyPage.nextFrame() + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(0) + await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) + }) + + test('a repeated terminal frame does not duplicate the output', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + + const mine = simulator.prompt(jobId, { workflowId }) + const script = [ + mine.start(), + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ] + + simulator.play(duplicateFrames(script, `${workflowId}:execution_success`)) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a backend that sends no workflow id behaves as before', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + + const legacy = simulator.prompt(jobId) + const foreign = simulator.prompt('foreign-job') + simulator.play([ + legacy.start(), + foreign.start(), + ...foreign.nodeRunning(KSAMPLER_NODE, 3, 4), + foreign.success(), + ...legacy.nodeRunning(KSAMPLER_NODE, 1, 4) + ]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + legacy.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + legacy.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + await expect(comfyPage.appMode.outputHistory.skeletons).toHaveCount(0) + }) + + test('another workflow finishing does not end the visible run', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + foreign.start(), + foreign.executing(null), + foreign.success() + ]) + + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([ + mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')), + mine.success() + ]) + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a reconnect mid-run does not duplicate the output', async ({ + comfyPage, + getWebSocket, + nextWebSocket, + execution: exec, + backendSimulator: simulator + }) => { + const ws = await getWebSocket() + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + await expect( + comfyPage.appMode.outputHistory.inProgressItems.first() + ).toBeVisible() + + simulator.play([mine.executed(SAVE_IMAGE_NODE, imageOutput('mine.png'))]) + + const reconnected = nextWebSocket() + await ws.close() + const nextRoute = await reconnected + const afterReconnect = new BackendSimulator( + new ExecutionHelper(comfyPage, nextRoute) + ).prompt(jobId, { workflowId }) + + afterReconnect.executed(SAVE_IMAGE_NODE, imageOutput('mine.png')).send() + afterReconnect.success().send() + + await expect(comfyPage.appMode.outputHistory.imageOutputs).toHaveCount(1) + }) + + test('a binary preview for another workflow does not reach the visible tab', async ({ + comfyPage, + execution: exec, + backendSimulator: simulator + }) => { + const jobId = await exec.run() + await comfyPage.nextFrame() + const mine = simulator.prompt(jobId, { workflowId }) + const foreign = simulator.prompt('foreign-job', { + workflowId: FOREIGN_WORKFLOW_ID + }) + + simulator.play([mine.start(), ...mine.nodeRunning(KSAMPLER_NODE, 1, 4)]) + + simulator.play([foreign.latentPreview(KSAMPLER_NODE)]) + await comfyPage.nextFrame() + + simulator.play([mine.latentPreview(KSAMPLER_NODE)]) + await comfyPage.nextFrame() + await expect(comfyPage.appMode.outputHistory.latentPreviews).toHaveCount(1) + expect( + await comfyPage.page.evaluate( + () => Object.values(window.app!.nodePreviewImages).flat().length + ) + ).toBeGreaterThan(0) + }) +}) diff --git a/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md b/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md new file mode 100644 index 00000000000..c2bdd8635c7 --- /dev/null +++ b/docs/adr/EXECUTION-ROUTING-0045-workflow-ownership-for-websocket-frames.md @@ -0,0 +1,53 @@ +# ADR-EXECUTION-ROUTING-0045: Workflow ownership for execution websocket frames + +Date: 2026-10-17 + +## Status + +Proposed + +## Context + +One `LGraph` and one `LGraphCanvas` serve every open workflow tab, so execution state written by node id is shared state. A run started in one tab painted its progress, outputs and run status onto whichever tab was in front. Reported on #1293, #2660, #6730 and Comfy-Org/ComfyUI#6581, under the umbrella #3377, and reproduced by manual QA with three pasted copies of one graph. + +[ADR-GRAPH-DOCUMENT-0026](GRAPH-DOCUMENT-0026-frontend-document-model.md) D2 already states the rule: a frame arriving on a push channel resolves its target document at ingress via the job map, never via the active pointer. It also states what to do at the boundary — "events with no job→uid resolution (page reloaded mid-run, jobs queued by another client) go to an explicit _unattributed_ bucket rather than being applied to the active document by default … misdelivery is not the fallback." + +This change implements D2's addressing rule and **departs from its boundary rule**. + +## Decision + +**Resolution order, strongest first.** A frame's owner is resolved by the open tab's `instanceId`, then the session path, then the workflow's graph id. When none of these resolves, the `activeJobId` guard decides. + +The graph id is deliberately the weakest signal, not the strongest. `ensureWorkflowId` keeps the id already present in a pasted graph, so copy a workflow, open a tab and paste gives two tabs carrying one id; loading a single workflow has also been measured leaving two open entries holding it. Keying on the graph id first is the bug QA found, where every tab sharing an id matched every job. `instanceId` is the only identifier unique per open tab, and it already existed. + +**The all-zero UUID is not an id.** `LGraph._id` defaults to it and the codebase reads it as "no id yet", replacing it on configure and on clear. It is not sent as a routing key, and an incoming one is discarded rather than compared. It still reaches the wire, because core falls back to the id inside `extra_data.extra_pnginfo.workflow`, and comparing it as a real id rejected a fresh graph's own frames. + +**Unresolvable frames fall back to the active job.** This is the documented exception to D2's boundary rule. When no mapping and no usable id exist, the frame is delivered to the tab in front if no job is active or the frame belongs to `activeJobId`, rather than quarantined. A frame with no `prompt_id` is always delivered. + +**Keys decide gating, not foreignness.** A frame being from another tab is not sufficient reason to drop work. What matters is the key the consumer reads: + +- Node-keyed writes (node progress, node outputs, node previews, text previews) are gated, because the visible graph may hold the same node ids. +- Job-keyed writes are not. `useJobPreviewStore().setPreviewUrl` feeds the queue panel's thumbnail and `linearOutputStore`'s re-apply on return, both for background jobs. +- Account-level signals are not. A sign-in, subscription or credit precondition belongs to the account, and `handleAccountPreconditionError` returns before recording the error, so its dialog is the only signal a background run produces. + +**Finished background jobs release their own records.** A job whose tab is not in front still ends, so each terminal path releases its per-job state rather than returning early at the gate. + +## Alternatives considered + +**Quarantine unresolvable frames, as D2 prescribes.** Rejected for this change. The common unresolvable case is a page reloaded mid-run, where the mappings are gone and the only open tab is the one the user is looking at. Quarantining would make a single-tab user's own run stop updating after a refresh — a regression for the overwhelmingly common case, in service of a rule written for the multi-tab case. D2's boundary also assumes a surfacing UI for the unattributed bucket, which does not exist. + +**Gate on the graph id alone.** This shipped and was wrong; see above. It is recorded here because the id looks like the obvious key and the mistake is easy to repeat. + +**A `contextvars`-based owner on the backend.** Rejected on a verified mechanism error: `prompt_worker` is a daemon thread and `send_json` runs on the event loop, so a ContextVar set in the worker is unset at send time. The attribute set on `PromptServer` and merged in the body of `send_sync` is what ships (Comfy-Org/ComfyUI#16763, #16809). + +**Derive the shared node-progress mirror instead of maintaining it.** Deferred to #20604, not rejected. It is read by the canvas sync path on every frame, so making it a computed changes when that work happens relative to painting, which is the interaction the original bug lived in. + +## Consequences + +**The `activeJobId` fallback is load-bearing and is also the weakest point.** Single-tab behaviour after a reload is preserved. A job from another browser session reaches the tab in front only when nothing else is running or it is itself the active job, so it is narrower than a blanket permissive rule. What remains is that with two tabs open and a frame unresolvable, it can still land on the wrong one. The blast radius is bounded by how rarely those conditions coincide, not by the design. + +**`status` and `progress_text` frames stay unattributable.** `status` carries no `prompt_id` because one server-wide socket serves every workflow. `progress_text` is a packed binary frame that `send_sync` cannot stamp; the frontend already reads a `supports_progress_text_metadata` server flag that no core version sets. Tracked as Comfy-Org/ComfyUI#16887. Until it lands, the text-preview widget gate goes permissive. + +**The two backends disagree on merge order.** Core merges `{**workflow_metadata, **data}`, so a frame's own fields win. The Cloud gateway assigns `prompt_id` and `workflow_id` after the fact, so the backend wins. Neither order should be relied on; the routing ids that arrive are authoritative whichever side set them. + +**Revisit if** an unattributed-frame surface is built, at which point D2's boundary rule becomes implementable and this exception should be removed rather than inherited. diff --git a/src/composables/node/useProgressTextPreviews.test.ts b/src/composables/node/useProgressTextPreviews.test.ts index 4449a3ebeeb..629361c2bb7 100644 --- a/src/composables/node/useProgressTextPreviews.test.ts +++ b/src/composables/node/useProgressTextPreviews.test.ts @@ -42,11 +42,14 @@ function showCanvasNode() { return node } -function createWorkflow(path = 'workflows/test.json') { +function createWorkflow(path = 'workflows/test.json', id = 'workflow-id') { return fromPartial({ - activeState: { id: 'workflow-id' }, - initialState: { id: 'workflow-id' }, - path + activeState: { id }, + initialState: { id }, + path, + // The only identifier unique per open tab, and the strongest leg of the + // ownership gate this suite now goes through. + instanceId: `instance-${path}` }) } @@ -64,6 +67,22 @@ function fireProgressText(detail: ProgressTextWsMessage) { progressTextListener()(new CustomEvent('progress_text', { detail })) } +/** + * Queue a job from `workflow` without making it the active job, which is what a + * run started while another tab was in front looks like. + */ +function storeBackgroundJob(jobId: string, workflow: LoadedComfyWorkflow) { + useExecutionStore().storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { + '1': { inputs: {}, class_type: 'Node', _meta: { title: 'Node' } } + }, + workflow, + mode: 'graph' + }) +} + function storeActiveJob(workflow: LoadedComfyWorkflow) { const executionStore = useExecutionStore() executionStore.storeJob({ @@ -124,6 +143,44 @@ describe('useProgressTextPreviews', () => { expect(showTextPreview).not.toHaveBeenCalled() }) + // Ported from executionStore.workflowGating.test.ts when handleProgressText + // moved into this composable. The gate is ownership-based, not a bare + // activeJobId comparison, because activeJobId can name another tab's job. + it('drops text from a job belonging to another open workflow', () => { + showCanvasNode() + const visible = createWorkflow('workflows/a.json', 'workflow-a') + const other = createWorkflow('workflows/b.json', 'workflow-b') + useWorkflowStore().activeWorkflow = visible + storeBackgroundJob('job-b', other) + mountPreviews() + + fireProgressText({ + nodeId: toNodeId('1'), + text: 'from the other tab', + prompt_id: 'job-b' + }) + + expect(showTextPreview).not.toHaveBeenCalled() + }) + + it('shows text from the visible workflow own background job', () => { + const node = showCanvasNode() + const visible = createWorkflow('workflows/a.json', 'workflow-a') + useWorkflowStore().activeWorkflow = visible + // Owned by the tab in front but not the active job, which is what the + // old activeJobId-only guard dropped. + storeBackgroundJob('job-a', visible) + mountPreviews() + + fireProgressText({ + nodeId: toNodeId('1'), + text: 'mine', + prompt_id: 'job-a' + }) + + expect(showTextPreview).toHaveBeenCalledExactlyOnceWith(node, 'mine') + }) + it('ignores events for a prompt other than the active job', () => { showCanvasNode() storeActiveJob(createWorkflow()) diff --git a/src/composables/node/useProgressTextPreviews.ts b/src/composables/node/useProgressTextPreviews.ts index acbae94e8bd..01753e51286 100644 --- a/src/composables/node/useProgressTextPreviews.ts +++ b/src/composables/node/useProgressTextPreviews.ts @@ -26,8 +26,13 @@ export function useProgressTextPreviews() { const { nodeId, text, prompt_id } = e.detail if (!text || !nodeId) return - const activeJobId = executionStore.activeJobId - if (prompt_id && activeJobId && prompt_id !== activeJobId) return + // The ownership gate, not a bare activeJobId comparison: activeJobId can + // name a job belonging to another open workflow, which both drops frames + // for the tab in front and lets a background job's text land on it. The + // gate falls back to the activeJobId check only when ownership cannot be + // resolved at all. No workflow id to pass: core packs progress_text as a + // binary frame that carries none, tracked in Comfy-Org/ComfyUI#16887. + if (!executionStore.belongsToActiveWorkflow(prompt_id)) return const executionId = String(nodeId) const currentId = executionId.includes(':') diff --git a/src/platform/remote/comfyui/execution/types.ts b/src/platform/remote/comfyui/execution/types.ts index 89c2177b6e1..7a1f2ab2ec3 100644 --- a/src/platform/remote/comfyui/execution/types.ts +++ b/src/platform/remote/comfyui/execution/types.ts @@ -3,6 +3,7 @@ import { z } from 'zod' import { zNodeId } from '@/platform/workflow/validation/schemas/workflowSchema' import type { TaskStatus } from '@/platform/tasks/services/taskSchema' import { resultItemType } from '@/schemas/resultItemTypeSchema' +import type { WorkflowId } from '@/platform/workflow/validation/schemas/workflowSchema' export type JobId = string @@ -43,6 +44,7 @@ export interface ProgressWsMessage { value: number max: number prompt_id: JobId + workflow_id?: WorkflowId node: NodeId } export interface NodeProgressState { @@ -51,18 +53,21 @@ export interface NodeProgressState { state: 'pending' | 'running' | 'finished' | 'error' node_id: NodeId prompt_id: JobId + workflow_id?: WorkflowId display_node_id?: NodeId parent_node_id?: NodeId real_node_id?: NodeId } export interface ProgressStateWsMessage { prompt_id: JobId + workflow_id?: WorkflowId nodes: Record } export interface ExecutingWsMessage { node: NodeId | null display_node?: NodeId prompt_id: JobId + workflow_id?: WorkflowId } export interface ExecutedWsMessage extends ExecutingWsMessage { node: NodeId @@ -72,6 +77,7 @@ export interface ExecutedWsMessage extends ExecutingWsMessage { } interface ExecutionWsMessageBase { prompt_id: JobId + workflow_id?: WorkflowId timestamp: number } export type ExecutionStartWsMessage = ExecutionWsMessageBase @@ -80,9 +86,18 @@ export interface ExecutionCachedWsMessage extends ExecutionWsMessageBase { nodes: NodeId[] } export interface ExecutionInterruptedWsMessage extends ExecutionWsMessageBase { - node_id: NodeId - node_type: string - executed: NodeId[] + /** + * Optional because a user cancel on Cloud is synthesised by the gateway + * rather than forwarded from the worker, and carries only `synthetic`, + * `reason` and the routing ids. Core always sends all three. + */ + node_id?: NodeId + node_type?: string + executed?: NodeId[] + /** Cloud only, set on a gateway-synthesised cancel. */ + synthetic?: boolean + /** Cloud only, the human-readable cancel reason. */ + reason?: string } export interface ExecutionErrorWsMessage extends ExecutionWsMessageBase { node_id?: NodeId | null diff --git a/src/platform/remote/comfyui/useQueuePolling.test.ts b/src/platform/remote/comfyui/useQueuePolling.test.ts index 6f2635ce501..b7f22586b85 100644 --- a/src/platform/remote/comfyui/useQueuePolling.test.ts +++ b/src/platform/remote/comfyui/useQueuePolling.test.ts @@ -3,7 +3,9 @@ import { nextTick } from 'vue' import { beforeEach, describe, expect, it, vi } from 'vitest' import { useQueuePolling } from '@/platform/remote/comfyui/useQueuePolling' +import { useExecutionStore } from '@/stores/executionStore' import { useQueueStore } from '@/stores/queueStore' +import { toNodeId } from '@/types/nodeId' function mountUseQueuePolling() { render({ @@ -30,32 +32,45 @@ describe('useQueuePolling', () => { expect(store.update).not.toHaveBeenCalled() }) - it('polls when activeJobsCount is exactly 1', async () => { + // The 2 row is what keeps this from being narrowed back to `=== 1`: polling + // with several jobs in flight is how per-job progress recovers when a + // terminal WebSocket frame is dropped. + it.for([1, 2])('polls when activeJobsCount is %i', async (count) => { mountUseQueuePolling() - Object.assign(store, { activeJobsCount: 1 }) + Object.assign(store, { activeJobsCount: count }) await vi.advanceTimersByTimeAsync(8_000) expect(store.update).toHaveBeenCalledOnce() }) - it('does not poll when activeJobsCount > 1', async () => { + it('stops polling when activeJobsCount drops to 0', async () => { + Object.assign(store, { activeJobsCount: 1 }) mountUseQueuePolling() - Object.assign(store, { activeJobsCount: 2 }) + Object.assign(store, { activeJobsCount: 0 }) await vi.advanceTimersByTimeAsync(16_000) expect(store.update).not.toHaveBeenCalled() }) - it('stops polling when activeJobsCount drops to 0', async () => { - Object.assign(store, { activeJobsCount: 1 }) + it('keeps polling while execution state awaits terminal reconciliation', async () => { + useExecutionStore().nodeProgressStatesByJob = { + 'stuck-job': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'stuck-job' + } + } + } mountUseQueuePolling() - Object.assign(store, { activeJobsCount: 0 }) - await vi.advanceTimersByTimeAsync(16_000) + await vi.advanceTimersByTimeAsync(8_000) - expect(store.update).not.toHaveBeenCalled() + expect(store.update).toHaveBeenCalledOnce() }) it('resets timer when loading completes', async () => { diff --git a/src/platform/remote/comfyui/useQueuePolling.ts b/src/platform/remote/comfyui/useQueuePolling.ts index c66c14a25a6..3359c8ab20e 100644 --- a/src/platform/remote/comfyui/useQueuePolling.ts +++ b/src/platform/remote/comfyui/useQueuePolling.ts @@ -2,6 +2,7 @@ import { useTimeoutFn } from '@vueuse/core' import { ref, watch } from 'vue' import { useQueueStore } from '@/stores/queueStore' +import { useExecutionStore } from '@/stores/executionStore' const BASE_INTERVAL_MS = 8_000 const MAX_INTERVAL_MS = 32_000 @@ -9,11 +10,17 @@ const BACKOFF_MULTIPLIER = 1.5 export function useQueuePolling() { const queueStore = useQueueStore() + const executionStore = useExecutionStore() const delay = ref(BASE_INTERVAL_MS) const { start, stop } = useTimeoutFn( () => { - if (queueStore.activeJobsCount !== 1 || queueStore.isLoading) return + if ( + (queueStore.activeJobsCount < 1 && + executionStore.trackedJobIds.length < 1) || + queueStore.isLoading + ) + return delay.value = Math.min(delay.value * BACKOFF_MULTIPLIER, MAX_INTERVAL_MS) void queueStore.update() }, @@ -22,12 +29,17 @@ export function useQueuePolling() { ) function scheduleNextPoll() { - if (queueStore.activeJobsCount === 1 && !queueStore.isLoading) start() + if ( + (queueStore.activeJobsCount >= 1 || + executionStore.trackedJobIds.length >= 1) && + !queueStore.isLoading + ) + start() else stop() } watch( - () => queueStore.activeJobsCount, + () => [queueStore.activeJobsCount, executionStore.trackedJobIds.length], () => { delay.value = BASE_INTERVAL_MS scheduleNextPoll() diff --git a/src/renderer/extensions/linearMode/linearOutputStore.test.ts b/src/renderer/extensions/linearMode/linearOutputStore.test.ts index 17de7a78c23..13c1305fceb 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.test.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.test.ts @@ -264,6 +264,122 @@ describe('linearOutputStore', () => { expect(store.inProgressItems).toHaveLength(2) }) + // Node expansion (loop nodes and anything else expanding a subgraph) gives + // each expanded copy its own `node` while they all share one `display_node`, + // so keying the dedupe on the display id collapsed every iteration's output + // into the first. Core: execution.py sends + // "display_node": dynprompt.get_display_node_id(unique_id). + it('keeps one item per expanded copy sharing a display node', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', { + prompt_id: 'job-1', + node: '9.0.1', + display_node: '9', + output: { + images: [{ filename: 'iter1.png', subfolder: '', type: 'output' }] + } + }) + store.onNodeExecuted('job-1', { + prompt_id: 'job-1', + node: '9.0.2', + display_node: '9', + output: { + images: [{ filename: 'iter2.png', subfolder: '', type: 'output' }] + } + }) + + const images = store.inProgressItems.filter((i) => i.state === 'image') + expect(images, 'each loop iteration keeps its own output').toHaveLength(2) + }) + + it('keeps both when one node reports two different outputs', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + store.onNodeExecuted( + 'job-1', + makeExecutedDetail('job-1', [ + { filename: 'second.png', subfolder: '', type: 'output' } + ]) + ) + + expect( + store.inProgressItems.filter((i) => i.state === 'image'), + 'same node, different files, so both are real outputs' + ).toHaveLength(2) + }) + + it('keeps both when two nodes report the same file', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + // Two PreviewImage nodes fed from one upstream image report the same + // filename, and each is its own output. Without the node in the key the + // files alone would make these look like one frame replayed. + store.onNodeExecuted('job-1', makeExecutedDetail('job-1', undefined, '7')) + store.onNodeExecuted('job-1', makeExecutedDetail('job-1', undefined, '8')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + + it('does not append a second copy when a reconnect replays executed', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(1) + + // Same job, same node, same output: a catch-up replay, not a new node. + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image'), + 'the replayed frame must not add a second item' + ).toHaveLength(1) + }) + + it('still appends a second output node in the same job', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + // A different output node, which is what distinguishes this from a replay. + store.onNodeExecuted( + 'job-1', + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) + ) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + + it('does not dedupe the same node id across two concurrent jobs', () => { + const store = useLinearOutputStore() + store.onJobStart('job-1') + store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) + + // Two jobs in flight share node ids, which is the reason the key is scoped + // by job and not by node alone. + store.onJobStart('job-2') + store.onNodeExecuted('job-2', makeExecutedDetail('job-2')) + + expect( + store.inProgressItems.filter((i) => i.state === 'image') + ).toHaveLength(2) + }) + it('handles execute without prior skeleton (no latent preview)', () => { const store = useLinearOutputStore() store.onJobStart('job-1') @@ -275,9 +391,11 @@ describe('linearOutputStore', () => { // Second node executes directly (no latent, no skeleton) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) const images = store.inProgressItems.filter((i) => i.state === 'image') @@ -354,15 +472,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'c.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'c.png', subfolder: '', type: 'output' }], + '3' + ) ) const run1Images = store.inProgressItems.filter((i) => i.state === 'image') @@ -381,15 +503,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-2', makeExecutedDetail('job-2')) store.onNodeExecuted( 'job-2', - makeExecutedDetail('job-2', [ - { filename: 'e.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-2', + [{ filename: 'e.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-2', - makeExecutedDetail('job-2', [ - { filename: 'f.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-2', + [{ filename: 'f.png', subfolder: '', type: 'output' }], + '3' + ) ) const run2Images = store.inProgressItems.filter((i) => i.state === 'image') @@ -411,9 +537,11 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onJobComplete('job-1') @@ -438,15 +566,19 @@ describe('linearOutputStore', () => { store.onNodeExecuted('job-1', makeExecutedDetail('job-1')) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'b.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'b.png', subfolder: '', type: 'output' }], + '2' + ) ) store.onNodeExecuted( 'job-1', - makeExecutedDetail('job-1', [ - { filename: 'c.png', subfolder: '', type: 'output' } - ]) + makeExecutedDetail( + 'job-1', + [{ filename: 'c.png', subfolder: '', type: 'output' }], + '3' + ) ) store.onJobComplete('job-1') diff --git a/src/renderer/extensions/linearMode/linearOutputStore.ts b/src/renderer/extensions/linearMode/linearOutputStore.ts index 28964da64f2..b1a34b5b8bd 100644 --- a/src/renderer/extensions/linearMode/linearOutputStore.ts +++ b/src/renderer/extensions/linearMode/linearOutputStore.ts @@ -29,6 +29,16 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { const trackedJobId = ref(null) const pendingResolve = ref(new Set()) const executedNodeIds = new Set() + /** + * Outputs already appended, so a replayed `executed` is skipped. + * + * Keyed by what the frame carries, not just where it came from. A reconnect + * replays the identical frame, while a node that legitimately reports again + * carries different files, and an expanded copy in a loop shares its + * `display_node` with its siblings but not its `node`. + */ + const appendedOutputKeys = new Set() + const MAX_APPENDED_OUTPUT_KEYS = 500 const activeWorkflowInProgressItems = computed(() => { const path = workflowStore.activeWorkflow?.path @@ -60,6 +70,10 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onJobStart(jobId: JobId) { executedNodeIds.clear() + // This job's keys only: a concurrent job's outputs must stay deduped. + for (const key of appendedOutputKeys) { + if (key.startsWith(`${jobId}:`)) appendedOutputKeys.delete(key) + } const item: InProgressItem = { id: makeItemId(jobId), @@ -112,6 +126,8 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { function onNodeExecuted(jobId: JobId, detail: ExecutedWsMessage) { const nodeId = String(detail.display_node || detail.node) + const outputKey = appendedOutputKey(jobId, detail) + if (appendedOutputKeys.has(outputKey)) return executedNodeIds.add(nodeId) if (raf) { cancelAnimationFrame(raf) @@ -153,6 +169,7 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { arr.splice(idx, 1, imageItem, ...extras) currentSkeletonId.value = null inProgressItems.value = arr + rememberAppendedOutput(outputKey) return } @@ -167,6 +184,26 @@ export const useLinearOutputStore = defineStore('linearOutput', () => { })) autoSelect(`slot:${newItems[0].id}`, jobId) inProgressItems.value = [...newItems, ...inProgressItems.value] + rememberAppendedOutput(outputKey) + } + + function appendedOutputKey(jobId: JobId, detail: ExecutedWsMessage): string { + const files = flattenNodeOutput([String(detail.node), detail.output]) + .map((item) => `${item.subfolder}/${item.filename}`) + .sort() + .join(',') + return `${jobId}:${detail.node}:${files}` + } + + function rememberAppendedOutput(key: string) { + appendedOutputKeys.add(key) + // Bounded: far above nodes-per-job times concurrent jobs, and Set keeps + // insertion order, so the oldest key goes first. + while (appendedOutputKeys.size > MAX_APPENDED_OUTPUT_KEYS) { + const oldest = appendedOutputKeys.values().next().value + if (oldest === undefined) break + appendedOutputKeys.delete(oldest) + } } function onJobComplete(jobId: JobId) { diff --git a/src/scripts/api.queuePrompt.test.ts b/src/scripts/api.queuePrompt.test.ts new file mode 100644 index 00000000000..a73aeb102b7 --- /dev/null +++ b/src/scripts/api.queuePrompt.test.ts @@ -0,0 +1,85 @@ +import type { MockInstance } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ComfyApiWorkflow, + ComfyWorkflowJSON +} from '@/platform/workflow/validation/schemas/workflowSchema' +import { api } from '@/scripts/api' +import { zeroUuid } from '@/utils/uuid' + +const WORKFLOW_ID = '0199e3a3-6c01-7000-8000-c1a1f7a8d9b2' +const EMPTY_PROMPT: ComfyApiWorkflow = {} + +function promptResponse(): Response { + return Response.json( + { prompt_id: 'p1', number: 1, node_errors: {} }, + { status: 200 } + ) +} + +function workflow(id?: string): ComfyWorkflowJSON { + return { + id, + revision: 0, + last_node_id: 0, + last_link_id: 0, + nodes: [], + links: [], + groups: [], + config: {}, + extra: {}, + version: 1 + } +} + +function sentBody(fetchApiSpy: MockInstance) { + const [, init] = fetchApiSpy.mock.calls[0] + return JSON.parse(String(init?.body)) as Record +} + +describe('api.queuePrompt workflow_metadata', () => { + let fetchApiSpy: MockInstance + + beforeEach(() => { + fetchApiSpy = vi.spyOn(api, 'fetchApi').mockResolvedValue(promptResponse()) + }) + + it('sends the workflow id as opaque metadata', async () => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(WORKFLOW_ID) + }) + + expect(sentBody(fetchApiSpy).workflow_metadata).toEqual({ + workflow_id: WORKFLOW_ID + }) + }) + + // The sentinel row earns its place: `LGraph._id` defaults to `zeroUuid` and + // `LGraph.serialize()` returns it unfiltered, so a bare truthiness check + // ships it as a routing key. The rest of the codebase already reads it as + // "no id yet" (`ensureNonZeroUuid`). + it.for([ + ['no id', undefined], + ['the all-zero id sentinel', zeroUuid] + ] as const)('omits the field for %s', async ([, id]) => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(id) + }) + + expect(sentBody(fetchApiSpy)).not.toHaveProperty('workflow_metadata') + }) + + it('still sends the id inside extra_pnginfo for older backends', async () => { + await api.queuePrompt(0, { + output: EMPTY_PROMPT, + workflow: workflow(WORKFLOW_ID) + }) + + expect(sentBody(fetchApiSpy)).toMatchObject({ + extra_data: { extra_pnginfo: { workflow: { id: WORKFLOW_ID } } } + }) + }) +}) diff --git a/src/scripts/api.socketReset.test.ts b/src/scripts/api.socketReset.test.ts index 8df6918a6a8..8d54dc1121a 100644 --- a/src/scripts/api.socketReset.test.ts +++ b/src/scripts/api.socketReset.test.ts @@ -52,6 +52,16 @@ class FakeWebSocket { const flush = () => new Promise((resolve) => setTimeout(resolve, 0)) +function previewWithMetadata(metadata: Record): ArrayBuffer { + const metadataBytes = new TextEncoder().encode(JSON.stringify(metadata)) + const buffer = new ArrayBuffer(8 + metadataBytes.length + 1) + const view = new DataView(buffer) + view.setUint32(0, 4) + view.setUint32(4, metadataBytes.length) + new Uint8Array(buffer, 8, metadataBytes.length).set(metadataBytes) + return buffer +} + describe('ComfyApi realtime socket reset', () => { beforeEach(() => { FakeWebSocket.instances = [] @@ -155,4 +165,67 @@ describe('ComfyApi realtime socket reset', () => { expect(window.name).toBe('') expect(sessionStorage.getItem('clientId')).toBeNull() }) + + it('exposes the executing ids during dispatch and clears them after', async () => { + await api.resetSocket() + const socket = FakeWebSocket.instances[0] + socket.simulateOpen() + + let seenDuringDispatch: unknown = 'not called' + const listener = () => { + seenDuringDispatch = api.lastExecutingMessage + } + api.addEventListener('executing', listener) + + socket.handlers['message']?.({ + data: JSON.stringify({ + type: 'executing', + data: { + node: '3', + display_node: '3', + prompt_id: 'job-a', + workflow_id: 'wf-a' + } + }) + }) + + api.removeEventListener('executing', listener) + + expect(seenDuringDispatch).toMatchObject({ + prompt_id: 'job-a', + workflow_id: 'wf-a', + node: '3' + }) + expect(api.lastExecutingMessage).toBeNull() + }) + + it('preserves workflow ownership from binary preview metadata', async () => { + await api.resetSocket() + const socket = FakeWebSocket.instances[0] + socket.simulateOpen() + + let received: unknown + const listener = (event: Event) => { + received = (event as CustomEvent).detail + } + api.addEventListener('b_preview_with_metadata', listener) + + socket.handlers['message']?.({ + data: previewWithMetadata({ + node_id: '3', + display_node_id: '3', + parent_node_id: '3', + real_node_id: '3', + prompt_id: 'job-b', + workflow_id: 'workflow-b', + image_type: 'image/png' + }) + }) + + api.removeEventListener('b_preview_with_metadata', listener) + expect(received).toMatchObject({ + jobId: 'job-b', + workflowId: 'workflow-b' + }) + }) }) diff --git a/src/scripts/api.ts b/src/scripts/api.ts index 0db427c7ab6..4b7a4b510b2 100644 --- a/src/scripts/api.ts +++ b/src/scripts/api.ts @@ -26,6 +26,7 @@ import { webSessionRequests } from '@/platform/auth/session/webSessionFetch' import { FETCH_RESPONSE_HEADERS_TIMEOUT_MS } from '@/scripts/apiTimeouts' import { getDevOverride } from '@/utils/devFeatureFlagOverride' import { getSessionOverride } from '@/utils/sessionFeatureFlagOverride' +import { zeroUuid } from '@/utils/uuid' import type { ModelFile, ModelFolderInfo @@ -73,7 +74,8 @@ import type { } from '@/platform/workflow/templates/types/template' import type { ComfyApiWorkflow, - ComfyWorkflowJSON + ComfyWorkflowJSON, + WorkflowId } from '@/platform/workflow/validation/schemas/workflowSchema' import type { SerializedNodeId } from '@/types/nodeId' import type { @@ -97,6 +99,29 @@ interface QueuePromptRequestBody { client_id: string prompt: ComfyApiWorkflow partial_execution_targets?: NodeExecutionId[] + /** + * Echoed back by the server on every WebSocket frame this prompt produces + * that carries a `prompt_id` (ComfyUI#16763 for JSON frames, #16809 for the + * metadata header on binary previews). An older server ignores the field + * rather than rejecting the prompt. + * + * Cloud does not read this field: its gateway stamps `workflow_id` from + * `extra_pnginfo.workflow.id` and assigns the routing ids after the frame's + * own fields, where core lets the frame's fields win. Treat the ids that + * arrive as authoritative either way. + * + * Two limits worth knowing, since the contract is the server's and not ours: + * + * - `status` and `logs` frames have no `prompt_id`, so they are never + * stamped and a consumer still needs the `prompt_id`-to-workflow mapping. + * - A value the server cannot use is dropped rather than rejected: over 256 + * characters of its own JSON serialisation, not 256 bytes of content, with + * non-ASCII escaped to `\\uXXXX` first. The server then falls back to + * `extra_data.extra_pnginfo.workflow.id`, so omitting this field does not + * keep frames unstamped. + */ + workflow_metadata?: { workflow_id: string } + extra_data: { extra_pnginfo: { workflow: ComfyWorkflowJSON @@ -265,6 +290,7 @@ interface BackendApiCalls { displayNodeId: string realNodeId: string jobId: string + workflowId?: WorkflowId } progress_text: ProgressTextWsMessage progress_state: ProgressStateWsMessage @@ -444,6 +470,12 @@ export class ComfyApi extends EventTarget { * The current client id from websocket status updates. */ clientId?: string + /** + * The `executing` message being dispatched, including `prompt_id` and + * `workflow_id`. Non-null only while `executing` listeners run, so read it + * synchronously inside the listener. + */ + lastExecutingMessage: ExecutingWsMessage | null = null /** * The current user id. */ @@ -1062,7 +1094,8 @@ export class ComfyApi extends EventTarget { displayNodeId: metadata.display_node_id, parentNodeId: metadata.parent_node_id, realNodeId: metadata.real_node_id, - jobId: metadata.prompt_id + jobId: metadata.prompt_id, + workflowId: metadata.workflow_id }) // Also dispatch legacy b_preview for backward compatibility @@ -1088,10 +1121,15 @@ export class ComfyApi extends EventTarget { this.dispatchCustomEvent('status', msg.data.status ?? null) break case 'executing': - this.dispatchCustomEvent( - 'executing', - msg.data.display_node || msg.data.node - ) + this.lastExecutingMessage = msg.data + try { + this.dispatchCustomEvent( + 'executing', + msg.data.display_node || msg.data.node + ) + } finally { + this.lastExecutingMessage = null + } break case 'execution_start': case 'execution_error': @@ -1286,6 +1324,10 @@ export class ComfyApi extends EventTarget { ...(options?.partialExecutionTargets && { partial_execution_targets: options.partialExecutionTargets }), + ...(workflow.id && + workflow.id !== zeroUuid && { + workflow_metadata: { workflow_id: workflow.id } + }), extra_data: { auth_token_comfy_org: this.authToken, api_key_comfy_org: this.apiKey, diff --git a/src/scripts/app.test.ts b/src/scripts/app.test.ts index fc4a5c61ecd..321dd5730c4 100644 --- a/src/scripts/app.test.ts +++ b/src/scripts/app.test.ts @@ -4,6 +4,7 @@ import { useToast } from '@/components/ui/toast/toastStore' import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore' import { useSubgraphNavigationStore } from '@/stores/subgraphNavigationStore' import { useNodeOutputStore } from '@/stores/nodeOutputStore' +import { useJobPreviewStore } from '@/stores/jobPreviewStore' import { useSettingStore } from '@/platform/settings/settingStore' import { useAuthStore } from '@/stores/authStore' @@ -97,6 +98,7 @@ import { setTelemetryRegistry } from '@/platform/telemetry' import { TelemetryRegistry } from '@/platform/telemetry/TelemetryRegistry' import * as executionContextUtils from '@/platform/telemetry/utils/getExecutionContext' import { isCloud } from '@/platform/distribution/types' +import * as accountPreconditionDialogModule from '@/platform/cloud/subscription/composables/useAccountPreconditionDialog' import { PromptExecutionError, api } from '@/scripts/api' import { useApp } from '@/scripts/appRegistry' @@ -3530,6 +3532,184 @@ describe('ComfyApp', () => { expect(executionErrorStore.isErrorOverlayOpen).toBe(false) }) + it('opens the account dialog even for a background run error', () => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + useExecutionStore().storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: {}, + workflow: workflowB, + mode: 'graph' + }) + + const open = vi.fn() + vi.spyOn( + accountPreconditionDialogModule, + 'useAccountPreconditionDialog' + ).mockReturnValue({ open }) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const executionErrorHandler = addEventListener.mock.calls.find( + ([event]) => event === 'execution_error' + )?.[1] as EventListener | undefined + expect(executionErrorHandler).toBeTypeOf('function') + + executionErrorHandler?.( + new CustomEvent('execution_error', { + detail: { + prompt_id: 'job-b', + workflow_id: workflowBId, + node_id: '1', + node_type: 'PartnerApiNode', + exception_message: + 'Payment Required: Please add credits to your account to use this node.', + exception_type: 'InsufficientFundsError', + traceback: [] + } + }) + ) + + expect(open).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ nodeType: 'PartnerApiNode' }) + ) + }) + + it.for([ + { jobId: 'job-a', owner: 'active', applied: true }, + { jobId: 'job-b', owner: 'background', applied: false } + ] as const)( + '$owner run output applied: $applied', + async ({ jobId, owner, applied }) => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + Reflect.set(app, 'rootGraphInternal', new LGraph()) + + const executionStore = useExecutionStore() + const workflow = owner === 'active' ? workflowA : workflowB + const workflowId = owner === 'active' ? workflowAId : workflowBId + executionStore.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: {}, + workflow, + mode: 'graph' + }) + + const setOutputs = vi.spyOn( + useNodeOutputStore(), + 'setNodeOutputsByExecutionId' + ) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const executedHandler = addEventListener.mock.calls.find( + ([event]) => event === 'executed' + )?.[1] as EventListener | undefined + expect(executedHandler).toBeTypeOf('function') + + executedHandler?.( + new CustomEvent('executed', { + detail: { + prompt_id: jobId, + workflow_id: workflowId, + node: '1', + display_node: '1', + output: { images: [{ filename: 'from-b.png', type: 'output' }] } + } + }) + ) + + expect(setOutputs).toHaveBeenCalledTimes(applied ? 1 : 0) + } + ) + + it('keeps a background preview job-keyed but off the visible graph', () => { + const workflowA = markLoaded( + new ComfyWorkflow({ path: 'workflows/a.json', modified: 0, size: 0 }) + ) + const workflowB = markLoaded( + new ComfyWorkflow({ path: 'workflows/b.json', modified: 0, size: 0 }) + ) + workflowA.changeTracker.activeState = workflowGraphData(workflowAId) + workflowB.changeTracker.activeState = workflowGraphData(workflowBId) + useWorkflowStore().activeWorkflow = workflowA + + useExecutionStore().storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: {}, + workflow: workflowB, + mode: 'graph' + }) + + const setPreviews = vi.spyOn( + useNodeOutputStore(), + 'setNodePreviewsByExecutionId' + ) + const addEventListener = vi.spyOn(api, 'addEventListener') + Reflect.apply(Reflect.get(app, 'addApiUpdateHandlers'), app, []) + const previewHandler = addEventListener.mock.calls.find( + ([event]) => event === 'b_preview_with_metadata' + )?.[1] as EventListener | undefined + expect(previewHandler).toBeTypeOf('function') + + previewHandler?.( + new CustomEvent('b_preview_with_metadata', { + detail: { + blob: new Blob(), + nodeId: '1', + displayNodeId: '1', + parentNodeId: '1', + realNodeId: '1', + jobId: 'job-b', + workflowId: workflowBId + } + }) + ) + + expect( + setPreviews, + 'the node-keyed write is gated: the visible graph has the same node ids' + ).not.toHaveBeenCalled() + useWorkflowStore().activeWorkflow = workflowB + previewHandler?.( + new CustomEvent('b_preview_with_metadata', { + detail: { + blob: new Blob(), + nodeId: '1', + displayNodeId: '1', + parentNodeId: '1', + realNodeId: '1', + jobId: 'job-b', + workflowId: workflowBId + } + }) + ) + expect( + setPreviews, + "and applied once that job's own tab is in front" + ).toHaveBeenCalled() + expect( + useJobPreviewStore().previewsByPromptId['job-b'], + 'the background job keeps its own thumbnail' + ).toBeTruthy() + }) + it('restores the failed run state when returning to a workflow tab', async () => { const workflowService = await useRealWorkflowService() const graph = new LGraph() diff --git a/src/scripts/app.ts b/src/scripts/app.ts index 4338510c9be..d8326f02b4b 100644 --- a/src/scripts/app.ts +++ b/src/scripts/app.ts @@ -788,6 +788,13 @@ export class ComfyApp implements IComfyApp { detail.display_node || detail.node ) if (!executionId) return + if ( + !useExecutionStore().belongsToActiveWorkflow( + detail.prompt_id, + detail.workflow_id + ) + ) + return nodeOutputStore.setNodeOutputsByExecutionId(executionId, detail.output, { merge: detail.merge @@ -804,6 +811,7 @@ export class ComfyApp implements IComfyApp { }) api.addEventListener('execution_error', ({ detail }) => { + // Account-scoped, so not gated by the visible workflow. const precondition = resolveAccountPrecondition({ exceptionType: detail.exception_type, exceptionMessage: detail.exception_message @@ -818,13 +826,17 @@ export class ComfyApp implements IComfyApp { api.addEventListener('b_preview_with_metadata', ({ detail }) => { // Enhanced preview with explicit node context - const { blob, displayNodeId, jobId } = detail + const { blob, displayNodeId, jobId, workflowId } = detail const { setNodePreviewsByExecutionId, revokePreviewsByExecutionId } = useNodeOutputStore() const displayNodeExecutionId = tryNormalizeNodeExecutionId(displayNodeId) if (!displayNodeExecutionId) return const blobUrl = createSharedObjectUrl(blob) useJobPreviewStore().setPreviewUrl(jobId, blobUrl, displayNodeId) + if (!useExecutionStore().belongsToActiveWorkflow(jobId, workflowId)) { + releaseSharedObjectUrl(blobUrl) + return + } // Ensure clean up if `executing` event is missed. revokePreviewsByExecutionId(displayNodeExecutionId) // Preview cleanup is handled in progress_state event to support multiple concurrent previews diff --git a/src/stores/executionStore.terminalRecovery.test.ts b/src/stores/executionStore.terminalRecovery.test.ts new file mode 100644 index 00000000000..bae369a0342 --- /dev/null +++ b/src/stores/executionStore.terminalRecovery.test.ts @@ -0,0 +1,380 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ExecutingWsMessage, + ExecutionErrorWsMessage, + ExecutionStartWsMessage, + NodeProgressState, + NotificationWsMessage, + ProgressStateWsMessage +} from '@/platform/remote/comfyui/execution/types' +import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' +import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' +import { api } from '@/scripts/api' +import { useExecutionStore } from '@/stores/executionStore' + +const { mockRemoveTextPreview } = await vi.hoisted(async () => ({ + mockRemoveTextPreview: vi.fn() +})) + +vi.mock(import('@/composables/useAppMode')) +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) + +vi.mock(import('@/composables/node/useNodeProgressText'), () => ({ + useNodeProgressText: () => ({ + showTextPreview: vi.fn(), + removeTextPreview: mockRemoveTextPreview + }) +})) + +type EventHandler = (event: CustomEvent) => void +interface EventPayloads { + execution_error: ExecutionErrorWsMessage + execution_start: ExecutionStartWsMessage + notification: NotificationWsMessage + progress_state: ProgressStateWsMessage +} +const apiEventHandlers = new Map() + +vi.mock(import('@/scripts/api'), () => ({ + api: { + addEventListener: vi.fn((event: string, handler: EventHandler) => { + apiEventHandlers.set(event, handler) + }), + removeEventListener: vi.fn((event: string) => { + apiEventHandlers.delete(event) + }), + clientId: 'test-client', + apiURL: vi.fn((path: string) => `/api${path}`), + lastExecutingMessage: null + } +})) + +vi.mock(import('@/scripts/app'), () => { + const rootGraph = { getNodeById: vi.fn(), nodes: [] } + return { + app: { + rootGraph, + rootGraphOrUndefined: rootGraph, + revokePreviews: vi.fn(), + nodePreviewImages: {} + } + } +}) + +const WORKFLOW_A_ID = 'workflow-a-id' +const WORKFLOW_B_ID = 'workflow-b-id' + +function workflow(id: string, path: string): LoadedComfyWorkflow { + return fromPartial({ + activeState: { id }, + initialState: { id }, + path + }) +} + +const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') +const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') + +function fireExecuting( + node: string | null, + ids: Pick +) { + api.lastExecutingMessage = { node, ...ids } + const handler = apiEventHandlers.get('executing') + if (!handler) throw new Error('executing handler not bound') + handler(new CustomEvent('executing', { detail: node })) +} + +function fire( + event: K, + detail: EventPayloads[K] +) { + const handler = apiEventHandlers.get(event) + if (!handler) throw new Error(`${event} handler not bound`) + handler(new CustomEvent(event, { detail })) + // Some frames are RAF-coalesced in the store; flush unconditionally so the + // assertion sees the applied state, and so this does not have to track which + // handlers are coalesced. + vi.advanceTimersToNextFrame() +} + +function runningNode(jobId: string, nodeId: string): NodeProgressState { + return { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state: 'running', + value: 3, + max: 10 + } +} + +/** + * Open these tabs through the real store, so `isOpen` and `openWorkflows` are + * the production implementations rather than a copy of them. A copy would keep + * passing if `isOpen` started keying by instance instead of path, which is + * exactly the identity mix-up this suite exists to catch. + * + * Additive, and a no-op for a tab already open. Pinia is rebuilt per test, so + * nothing accumulates across tests. + */ +function openTabs(...workflows: LoadedComfyWorkflow[]) { + const workflowStore = useWorkflowStore() + for (const workflow of workflows) { + if (workflowStore.isOpen(workflow)) continue + workflowStore.attachWorkflow(workflow, workflowStore.openWorkflows.length) + } +} + +describe('executionStore terminal-job recovery', () => { + let store: ReturnType + + beforeEach(() => { + apiEventHandlers.clear() + useWorkflowStore().activeWorkflow = workflowA + openTabs(workflowA, workflowB) + store = useExecutionStore() + store.bindExecutionEvents() + }) + + function stuckActiveJob(jobId = 'job-a') { + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fireExecuting('1', { prompt_id: jobId, workflow_id: WORKFLOW_A_ID }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + return jobId + } + + it('clears progress for a job the backend reports as finished', () => { + const jobId = stuckActiveJob() + expect(store.nodeProgressStates['1']?.state).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob[jobId]).toBeUndefined() + expect(store.activeJobId).toBeNull() + }) + + it('leaves a job the backend still reports as running alone', () => { + const jobId = stuckActiveJob() + + store.reconcileTerminalJobs(new Set([jobId]), new Set([jobId])) + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe(jobId) + }) + + it('ignores a job that is neither running nor in history yet', () => { + const jobId = stuckActiveJob() + + store.reconcileTerminalJobs(new Set(), new Set()) + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe(jobId) + }) + + it('does not blank a live run when evicting an older job', () => { + const stale = 'job-stale' + store.registerJobWorkflowIdMapping(stale, WORKFLOW_A_ID) + fire('progress_state', { + prompt_id: stale, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(stale, '1') } + }) + const live = stuckActiveJob('job-live') + + store.reconcileTerminalJobs(new Set([live]), new Set([stale])) + + expect(store.nodeProgressStatesByJob[stale]).toBeUndefined() + expect(store.nodeProgressStates['1']?.prompt_id).toBe(live) + expect(store.activeJobId).toBe(live) + }) + + it('clears a stuck job belonging to another workflow', () => { + store.registerJobWorkflowIdMapping('job-b', WORKFLOW_B_ID) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': runningNode('job-b', '1') } + }) + expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() + + store.reconcileTerminalJobs(new Set(), new Set(['job-b'])) + + expect(store.nodeProgressStatesByJob['job-b']).toBeUndefined() + }) + + it('clears an initializing job that reached history without a frame', () => { + fire('notification', { id: 'job-init', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-init')).toBe(true) + + store.reconcileTerminalJobs(new Set(), new Set(['job-init'])) + + expect(store.isJobInitializing('job-init')).toBe(false) + }) + + it('leaves a second workflow alone on a repeat pass', () => { + const jobId = stuckActiveJob() + store.registerJobWorkflowIdMapping('job-b', WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: 'job-b', + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect( + store.getWorkflowStatus(workflowB), + 'B never went terminal, so neither pass may release it' + ).toBe('running') + }) + + it('still recovers when the terminal frame was dropped for a legacy backend', () => { + const jobId = 'job-legacy' + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + fire('execution_start', { prompt_id: jobId, timestamp: 1 }) + fire('progress_state', { + prompt_id: jobId, + nodes: { '1': runningNode(jobId, '1') } + }) + expect(store.nodeProgressStates['1']?.state).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + function stuckQueuedJob(jobId = 'job-queued') { + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_A_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowA, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_A_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + return jobId + } + + it('releases the workflow status when a terminal frame was dropped', () => { + const jobId = stuckQueuedJob() + expect(store.getWorkflowStatus(workflowA)).toBe('running') + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.getWorkflowStatus(workflowA)).toBeUndefined() + }) + + it('fires onJobReset while the job record its consumer reads still exists', () => { + const jobId = stuckQueuedJob() + const seen: string[][] = [] + store.onJobReset((job) => { + seen.push(Object.keys(job.nodes)) + }) + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect( + seen, + 'fired exactly once, with the node list still populated' + ).toEqual([['1']]) + }) + + it.for([ + { name: 'node error', nodeId: '1' }, + { name: 'service-level error', nodeId: null } + ])('releases background job records after a $name', ({ nodeId }) => { + const jobId = `job-bg-error-${nodeId ?? 'service'}` + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + nodes: { '1': runningNode(jobId, '1') } + }) + fire('execution_error', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: nodeId, + node_type: 'TestNode', + exception_type: 'RuntimeError', + exception_message: 'boom', + traceback: [], + executed: [], + current_inputs: {}, + current_outputs: {} + }) + + expect(store.nodeProgressStatesByJob[jobId]).toBeUndefined() + expect(store.queuedJobs[jobId]).toBeUndefined() + }) + + it('unsticks a tab whose job never reported progress', () => { + // The gap the tracked set had: execution_start sets the tab to Running, so + // a job whose terminal frame was dropped before any progress_state arrived + // was in no tracked collection and its badge stayed Running for the + // session. + const jobId = 'job-no-progress' + store.registerJobWorkflowIdMapping(jobId, WORKFLOW_B_ID) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + workflow: workflowB, + mode: 'graph' + }) + fire('execution_start', { + prompt_id: jobId, + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + + store.reconcileTerminalJobs(new Set(), new Set([jobId])) + + expect(store.getWorkflowStatus(workflowB)).toBeUndefined() + }) +}) diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index e78a75c475a..cd770bc7305 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -36,12 +36,14 @@ import { useNodeOutputStore } from '@/stores/nodeOutputStore' import { useJobPreviewStore } from '@/stores/jobPreviewStore' import { useExecutionErrorStore } from '@/stores/executionErrorStore' import { tryNormalizeNodeExecutionId } from '@/types/nodeIdentification' +import type { SerializedNodeId } from '@/types/nodeId' import type { NodeLocatorId } from '@/types/nodeIdentification' import type { AppMode } from '@/utils/appMode' import { isAppModeValue } from '@/utils/appMode' import { classifyCloudValidationError } from '@/utils/executionErrorUtil' import { executionIdToNodeLocatorId } from '@/utils/graphTraversalUtil' -import { createRafCoalescer } from '@/utils/rafBatch' +import { createRafBatch, createRafCoalescer } from '@/utils/rafBatch' +import { zeroUuid } from '@/utils/uuid' type RuntimeExecutionError = Omit< ExecutionErrorWsMessage, @@ -51,6 +53,8 @@ type RuntimeExecutionError = Omit< traceback?: ExecutionErrorWsMessage['traceback'] | null } +type WorkflowOwnership = 'active' | 'foreign' | 'unresolved' + interface ExecutionNodeInfo { title?: string | null type?: string | null @@ -372,6 +376,7 @@ export const useExecutionStore = defineStore('execution', () => { const states = nodeProgressStates.value // Apparently doing this inside `Object.entries` causes issues for (const state of Object.values(states)) { + if (!belongsToActiveWorkflow(state.prompt_id, state.workflow_id)) continue const parts = String(state.display_node_id).split(':') for (let i = 0; i < parts.length; i++) { const executionId = parts.slice(0, i + 1).join(':') @@ -478,35 +483,35 @@ export const useExecutionStore = defineStore('execution', () => { } function handleExecutionStart(e: CustomEvent) { - executionIdToLocatorCache.clear() - activeJobId.value = e.detail.prompt_id - queuedJobs.value[activeJobId.value] ??= { nodes: {} } - clearInitializationByJobId(activeJobId.value) + const jobId = e.detail.prompt_id + queuedJobs.value[jobId] ??= { nodes: {} } + clearInitializationByJobId(jobId) // Ensure path mapping exists — execution_start can arrive via WebSocket // before the HTTP response from queuePrompt triggers storeJob. - if (!jobIdToSessionWorkflowPath.value.has(activeJobId.value)) { - const workflow = queuedJobs.value[activeJobId.value]?.workflow + if (!jobIdToSessionWorkflowPath.value.has(jobId)) { + const workflow = queuedJobs.value[jobId]?.workflow if (workflow) { - ensureSessionWorkflowPath( - activeJobId.value, - workflow.path, - workflow.instanceId - ) + ensureSessionWorkflowPath(jobId, workflow.path, workflow.instanceId) } } - executionErrorStore.clearExecutionStartErrors( - runErrorKeyForJob(activeJobId.value) - ) - queuedJobs.value[activeJobId.value].executionStartedAt ??= performance.now() - setWorkflowStatus(activeJobId.value, { + queuedJobs.value[jobId].executionStartedAt ??= performance.now() + setWorkflowStatus(jobId, { status: 'running', - executionStartedAt: queuedJobs.value[activeJobId.value].executionStartedAt + executionStartedAt: queuedJobs.value[jobId].executionStartedAt }) + + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) return + + executionIdToLocatorCache.clear() + executionErrorStore.clearExecutionStartErrors(runErrorKeyForJob(jobId)) + activeJobId.value = jobId } function handleExecutionCached(e: CustomEvent) { if (!activeJob.value) return + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + return for (const n of e.detail.nodes) { activeJob.value.nodes[n] = true } @@ -525,18 +530,25 @@ export const useExecutionStore = defineStore('execution', () => { }) const workflow = jobIdToWorkflow.get(jobId) if (workflow) clearWorkflowStatus(workflow) - if (activeJobId.value) clearInitializationByJobId(activeJobId.value) + clearInitializationByJobId(jobId) + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { + releaseFinishedJobRecords(jobId) + return + } resetExecutionState(jobId) } function handleExecuted(e: CustomEvent) { if (!activeJob.value) return + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) + return activeJob.value.nodes[e.detail.node] = true } function handleExecutionSuccess(e: CustomEvent) { const jobId = e.detail.prompt_id pendingExecutionErrorsByJobId.delete(jobId) + clearInitializationByJobId(jobId) setWorkflowStatus(jobId, { status: 'completed', endTime: performance.now() @@ -556,19 +568,31 @@ export const useExecutionStore = defineStore('execution', () => { }) } } + if (!belongsToActiveWorkflow(jobId, e.detail.workflow_id)) { + releaseFinishedJobRecords(jobId) + return + } resetExecutionState(jobId) } - function handleExecuting(e: CustomEvent): void { + function handleExecuting(e: CustomEvent): void { + // Bare node id for extensions; the ids come from api.lastExecutingMessage. + const raw = api.lastExecutingMessage + if (raw && !belongsToActiveWorkflow(raw.prompt_id, raw.workflow_id)) { + return + } + progressCoalescer.cancel() - if (e.detail == null) progressStateCoalescer.cancel() + if (e.detail == null) { + if (raw) pendingProgressStates.delete(raw.prompt_id) + else cancelPendingProgressStateUpdates() + } // Clear the current node progress when a new node starts executing _executingNodeProgress.value = null if (!activeJob.value) return - // Update the executing nodes list if (e.detail == null) { activeJobId.value = null } @@ -602,60 +626,154 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStatesByJob.value = pruned } - const progressStateCoalescer = createRafCoalescer( - applyProgressState, - 'raf:progress_state' - ) + const pendingProgressStates = new Map() + const progressStateBatch = createRafBatch(() => { + const pending = [...pendingProgressStates.values()] + pendingProgressStates.clear() + for (const detail of pending) applyProgressState(detail) + }) function handleProgressState(e: CustomEvent) { - progressStateCoalescer.push(e.detail) + pendingProgressStates.set(e.detail.prompt_id, e.detail) + progressStateBatch.schedule() } - function applyProgressState(detail: ProgressStateWsMessage) { - const { nodes, prompt_id: jobId } = detail + function cancelPendingProgressStateUpdates() { + pendingProgressStates.clear() + progressStateBatch.cancel() + } - // Revoke previews for nodes that are starting to execute - const previousForJob = - jobId in nodeProgressStatesByJob.value - ? nodeProgressStatesByJob.value[jobId] - : {} + function revokeStartedNodePreviews( + nodes: Record, + previousForJob: Record + ) { + const { revokePreviewsByExecutionId } = useNodeOutputStore() for (const nodeId in nodes) { - const nodeState = nodes[nodeId] - if (nodeState.state === 'running' && !(nodeId in previousForJob)) { - // This node just started executing, revoke its previews - // Note that we're doing the *actual* node id instead of the display node id - // here intentionally. That way, we don't clear the preview every time a new node - // within an expanded graph starts executing. - const { revokePreviewsByExecutionId } = useNodeOutputStore() - const executionId = tryNormalizeNodeExecutionId(nodeId) - if (executionId) revokePreviewsByExecutionId(executionId) - } + if (nodes[nodeId].state !== 'running') continue + if (previousForJob[nodeId]?.state === 'running') continue + const executionId = tryNormalizeNodeExecutionId(nodeId) + if (executionId) revokePreviewsByExecutionId(executionId) + } + } + + function mirrorExecutingNodeProgress( + nodes: Record + ) { + const executingId = executingNodeId.value + if (!executingId || !Object.hasOwn(nodes, executingId)) return + const nodeState = nodes[executingId] + _executingNodeProgress.value = { + value: nodeState.value, + max: nodeState.max, + prompt_id: nodeState.prompt_id, + node: nodeState.display_node_id || nodeState.node_id + } + } + + function applyProgressState(detail: ProgressStateWsMessage) { + const { nodes, prompt_id: jobId, workflow_id: messageWorkflowId } = detail + const isActiveWorkflowMessage = belongsToActiveWorkflow( + jobId, + messageWorkflowId + ) + + if (isActiveWorkflowMessage) { + revokeStartedNodePreviews( + nodes, + nodeProgressStatesByJob.value[jobId] ?? {} + ) } - // Update the progress states for all nodes nodeProgressStatesByJob.value = { ...nodeProgressStatesByJob.value, [jobId]: nodes } evictOldProgressJobs() + + if (!isActiveWorkflowMessage) return nodeProgressStates.value = nodes + mirrorExecutingNodeProgress(nodes) + } - // If we have progress for the currently executing node, update it for backwards compatibility - if (executingNodeId.value) { - const nodeState = Object.hasOwn(nodes, executingNodeId.value) - ? nodes[executingNodeId.value] - : undefined - if (!nodeState) return - _executingNodeProgress.value = { - value: nodeState.value, - max: nodeState.max, - prompt_id: nodeState.prompt_id, - node: nodeState.display_node_id || nodeState.node_id - } - } + /** + * Graph id of a workflow: the id carried by the workflow's own JSON, which + * every copy of that content shares, so it identifies the content and not + * the tab holding it. + * + * Both states are nullable on ComfyWorkflow and this is read from a watcher + * and a computed, so an unloaded workflow returns undefined rather than + * throwing. + */ + function graphIdOf( + workflow: ComfyWorkflow | null | undefined + ): string | undefined { + return workflow?.activeState?.id ?? workflow?.initialState?.id + } + + function activeWorkflowGraphId(): string | null { + return graphIdOf(workflowStore.activeWorkflow) ?? null + } + + /** + * The all-zero sentinel is `LGraph._id`'s default, which the codebase reads + * as "no id yet" and replaces on configure and on clear. Treating it as a + * real id would make every workflow still carrying it match every other one + * on the graph-id leg, which is the leak this gate exists to stop. The queue + * side does not send it, but core falls back to the id inside + * `extra_pnginfo.workflow`, so it still arrives on the wire. + */ + function workflowIdOrUndefined(id: string | undefined): string | undefined { + return !id || id === zeroUuid ? undefined : id + } + + function compareWorkflowOwner( + owner: string | undefined, + active: string | undefined + ): WorkflowOwnership { + if (!owner || !active) return 'unresolved' + return owner === active ? 'active' : 'foreign' + } + + function resolveWorkflowOwnership( + jobId: JobId, + rawMessageWorkflowId?: string + ): WorkflowOwnership { + const messageWorkflowId = workflowIdOrUndefined(rawMessageWorkflowId) + const activeWorkflow = workflowStore.activeWorkflow + if (!activeWorkflow) return 'unresolved' + + const instanceOwnership = compareWorkflowOwner( + jobIdToWorkflowInstanceId.get(jobId), + activeWorkflow.instanceId + ) + if (instanceOwnership !== 'unresolved') return instanceOwnership + + const pathOwnership = compareWorkflowOwner( + jobIdToSessionWorkflowPath.value.get(jobId), + activeWorkflow.path + ) + if (pathOwnership !== 'unresolved') return pathOwnership + + const activeId = workflowIdOrUndefined(activeWorkflowGraphId() ?? undefined) + const ownerId = + messageWorkflowId ?? + workflowIdOrUndefined(jobIdToWorkflowId.value.get(jobId)) + return compareWorkflowOwner(ownerId, activeId) + } + + function belongsToActiveWorkflow( + jobId: JobId | undefined, + workflowId?: string + ): boolean { + if (!jobId) return true + const ownership = resolveWorkflowOwnership(jobId, workflowId) + if (ownership !== 'unresolved') return ownership === 'active' + return !activeJobId.value || jobId === activeJobId.value } const progressCoalescer = createRafCoalescer((detail) => { + const { prompt_id: jobId, workflow_id: messageWorkflowId } = detail + if (!belongsToActiveWorkflow(jobId, messageWorkflowId)) return _executingNodeProgress.value = detail }, 'raf:progress') @@ -665,7 +783,7 @@ export const useExecutionStore = defineStore('execution', () => { function cancelPendingProgressUpdates() { progressCoalescer.cancel() - progressStateCoalescer.cancel() + cancelPendingProgressStateUpdates() } function handleStatus() { @@ -686,7 +804,7 @@ export const useExecutionStore = defineStore('execution', () => { */ function openWorkflowPathForGraph(graphId: WorkflowId): string | undefined { const matches = workflowStore.openWorkflows.filter( - (w) => (w.activeState?.id ?? w.initialState?.id) === graphId + (w) => graphIdOf(w) === graphId ) return matches.length === 1 ? matches[0].path : undefined } @@ -705,10 +823,7 @@ export const useExecutionStore = defineStore('execution', () => { */ function runErrorKeyForJob(jobId: string): string | null { const workflow = jobIdToWorkflow.get(jobId) - const graphId = - workflow?.activeState?.id ?? - workflow?.initialState?.id ?? - jobIdToWorkflowId.value.get(jobId) + const graphId = graphIdOf(workflow) ?? jobIdToWorkflowId.value.get(jobId) if (graphId === undefined) return null const path = @@ -727,6 +842,10 @@ export const useExecutionStore = defineStore('execution', () => { const runErrorKey = runErrorKeyForJob(e.detail.prompt_id) if (runErrorKey === null) { bufferPendingExecutionError({ detail: e.detail, endTime }) + if (!belongsToActiveWorkflow(e.detail.prompt_id, e.detail.workflow_id)) { + releaseFinishedJobRecords(e.detail.prompt_id) + return + } resetExecutionState(e.detail.prompt_id) return } @@ -779,6 +898,11 @@ export const useExecutionStore = defineStore('execution', () => { executionErrorStore.recordExecutionError(detail, runErrorKey) executionErrorStore.showExecutionError(detail, runErrorKey) clearInitializationByJobId(detail.prompt_id) + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) + return + } + resetExecutionState(detail.prompt_id) } @@ -794,6 +918,10 @@ export const useExecutionStore = defineStore('execution', () => { const workflow = jobIdToWorkflow.get(detail.prompt_id) if (workflow) clearWorkflowStatus(workflow) clearInitializationByJobId(detail.prompt_id) + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) + return true + } resetExecutionState(detail.prompt_id) return true } @@ -807,6 +935,11 @@ export const useExecutionStore = defineStore('execution', () => { return false clearInitializationByJobId(detail.prompt_id) + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) + return true + } + resetExecutionState(detail.prompt_id) executionErrorStore.recordPromptError( { @@ -829,6 +962,11 @@ export const useExecutionStore = defineStore('execution', () => { if (!result) return false clearInitializationByJobId(detail.prompt_id) + if (!belongsToActiveWorkflow(detail.prompt_id, detail.workflow_id)) { + releaseFinishedJobRecords(detail.prompt_id) + return true + } + resetExecutionState(detail.prompt_id) if (result.kind === 'nodeErrors') { @@ -876,6 +1014,84 @@ export const useExecutionStore = defineStore('execution', () => { initializingJobIds.value = next } + /** + * Returns the prompt_id the global {@link nodeProgressStates} mirror belongs + * to, or null when it is empty. The mirror is replaced wholesale on every + * `progress_state` frame, so all of its entries share one prompt_id. + */ + function mirrorOwnerJobId(): JobId | null { + const entries = Object.values(nodeProgressStates.value) + if (entries.length === 0) return null + return entries[0].prompt_id + } + + function evictTerminalJob(jobId: JobId) { + const isActive = activeJobId.value === jobId + + const evictedWorkflow = jobIdToWorkflow.get(jobId) + if (evictedWorkflow) clearWorkflowStatus(evictedWorkflow) + + releaseFinishedJobRecords(jobId) + clearInitializationByJobId(jobId) + if (isActive || mirrorOwnerJobId() === jobId) { + nodeProgressStates.value = {} + executionIdToLocatorCache.clear() + } + + if (_executingNodeProgress.value?.prompt_id === jobId) { + _executingNodeProgress.value = null + } + + if (isActive) { + activeJobId.value = null + executionErrorStore.clearPromptError(runErrorKeyForJob(jobId)) + } + } + + function reconcileMirrorForActiveWorkflow() { + const activeWorkflow = workflowStore.activeWorkflow + if (!activeWorkflow) return + + const matchedJobId = Object.keys(nodeProgressStatesByJob.value).findLast( + (jobId) => resolveWorkflowOwnership(jobId) === 'active' + ) + + if (matchedJobId) { + nodeProgressStates.value = + nodeProgressStatesByJob.value[matchedJobId] ?? {} + executionIdToLocatorCache.clear() + activeJobId.value = matchedJobId + if (_executingNodeProgress.value?.prompt_id !== matchedJobId) { + _executingNodeProgress.value = null + } + return + } + + if (Object.keys(nodeProgressStates.value).length > 0) { + nodeProgressStates.value = {} + executionIdToLocatorCache.clear() + } + _executingNodeProgress.value = null + } + + watch(() => workflowStore.activeWorkflow, reconcileMirrorForActiveWorkflow) + + /** + * Reconcile tracked per-job state against the backend's authoritative job + * sets. A job the backend reports as terminal but which still holds progress + * state lost its terminal frame, so evict it. + */ + function reconcileTerminalJobs( + activeJobIds: Set, + terminalJobIds: Set + ) { + for (const jobId of trackedJobIds.value) { + if (activeJobIds.has(jobId)) continue + if (!terminalJobIds.has(jobId)) continue + evictTerminalJob(jobId) + } + } + function reconcileInitializingJobs(activeJobIds: Set) { const orphaned = [...initializingJobIds.value].filter( (id) => !activeJobIds.has(id) @@ -898,6 +1114,22 @@ export const useExecutionStore = defineStore('execution', () => { return initializingJobIds.value.has(String(jobId)) } + function releaseFinishedJobRecords(jobId: JobId) { + // Fire before the queuedJobs delete: the onJobReset consumer reads the + // job's node list off that entry to find the widgets to remove, so + // triggering afterwards hands it nothing. It self-gates on the active + // workflow, so firing for a background job is a safe no-op. + if (jobId in queuedJobs.value) + void jobReset.trigger(queuedJobs.value[jobId]) + delete queuedJobs.value[jobId] + const byJob = { ...nodeProgressStatesByJob.value } + delete byJob[jobId] + nodeProgressStatesByJob.value = byJob + jobIdToWorkflow.delete(jobId) + useJobPreviewStore().clearPreview(jobId) + if (activeJobId.value === jobId) activeJobId.value = null + } + /** * Reset execution-related state after a run completes or is stopped. */ @@ -908,16 +1140,7 @@ export const useExecutionStore = defineStore('execution', () => { nodeProgressStates.value = {} const jobId = jobIdParam ?? activeJobId.value ?? null const runErrorKey = jobId ? runErrorKeyForJob(jobId) : undefined - if (jobId) { - const map = { ...nodeProgressStatesByJob.value } - delete map[jobId] - nodeProgressStatesByJob.value = map - useJobPreviewStore().clearPreview(jobId) - jobIdToWorkflow.delete(jobId) - if (jobId in queuedJobs.value) - void jobReset.trigger(queuedJobs.value[jobId]) - } - if (jobId) delete queuedJobs.value[jobId] + if (jobId) releaseFinishedJobRecords(jobId) activeJobId.value = null _executingNodeProgress.value = null executionErrorStore.clearPromptError(runErrorKey) @@ -963,7 +1186,7 @@ export const useExecutionStore = defineStore('execution', () => { queuedJob.shareId = workflow.shareId queuedJob.viewMode = mode queuedJob.isAppMode = isAppModeValue(mode) - const wid = workflow.activeState?.id ?? workflow.initialState?.id + const wid = graphIdOf(workflow) if (wid) { jobIdToWorkflowId.value.set(id, wid) } @@ -1078,6 +1301,16 @@ export const useExecutionStore = defineStore('execution', () => { return result }) + const trackedJobIds = computed(() => { + const tracked = new Set([ + ...Object.keys(nodeProgressStatesByJob.value), + ...Object.keys(queuedJobs.value), + ...initializingJobIds.value + ]) + if (activeJobId.value) tracked.add(activeJobId.value) + return [...tracked] + }) + const runningWorkflowCount = computed( () => runningJobIds.value.length ) @@ -1106,6 +1339,7 @@ export const useExecutionStore = defineStore('execution', () => { nodeLocationProgressStates, nodeProgressStatesByJob, runningJobIds, + trackedJobIds, runningWorkflowCount, initializingJobIds, isActiveWorkflowRunning, @@ -1113,6 +1347,7 @@ export const useExecutionStore = defineStore('execution', () => { clearInitializationByJobId, clearInitializationByJobIds, reconcileInitializingJobs, + reconcileTerminalJobs, clearActiveJobIfStale, bindExecutionEvents, unbindExecutionEvents, @@ -1128,6 +1363,7 @@ export const useExecutionStore = defineStore('execution', () => { jobIdToSessionWorkflowPath, ensureSessionWorkflowPath, getWorkflowStatus, - clearWorkflowStatus + clearWorkflowStatus, + belongsToActiveWorkflow } }) diff --git a/src/stores/executionStore.workflowGating.test.ts b/src/stores/executionStore.workflowGating.test.ts new file mode 100644 index 00000000000..5af7ac8dbf9 --- /dev/null +++ b/src/stores/executionStore.workflowGating.test.ts @@ -0,0 +1,1337 @@ +import { fromPartial } from '@total-typescript/shoehorn' +import { assert, beforeEach, describe, expect, it, vi } from 'vitest' +import { nextTick } from 'vue' + +import type { + ExecutedWsMessage, + ExecutingWsMessage, + ExecutionCachedWsMessage, + ExecutionErrorWsMessage, + ExecutionInterruptedWsMessage, + ExecutionStartWsMessage, + ExecutionSuccessWsMessage, + NodeProgressState, + NotificationWsMessage, + ProgressStateWsMessage, + ProgressWsMessage +} from '@/platform/remote/comfyui/execution/types' +import type { LoadedComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' +import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' +import { api } from '@/scripts/api' +import { useExecutionStore } from '@/stores/executionStore' +import { useNodeOutputStore } from '@/stores/nodeOutputStore' +import { zeroUuid } from '@/utils/uuid' + +vi.mock(import('@/composables/useAppMode')) +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/platform/distribution/types'), () => ({ isCloud: true })) + +type EventHandler = (event: CustomEvent) => void +interface EventPayloads { + executed: ExecutedWsMessage + execution_cached: ExecutionCachedWsMessage + execution_error: ExecutionErrorWsMessage + execution_interrupted: ExecutionInterruptedWsMessage + execution_start: ExecutionStartWsMessage + execution_success: ExecutionSuccessWsMessage + notification: NotificationWsMessage + progress: ProgressWsMessage + progress_state: ProgressStateWsMessage +} +const apiEventHandlers = new Map() + +vi.mock(import('@/scripts/api'), () => ({ + api: { + addEventListener: vi.fn((event: string, handler: EventHandler) => { + apiEventHandlers.set(event, handler) + }), + removeEventListener: vi.fn((event: string) => { + apiEventHandlers.delete(event) + }), + clientId: 'test-client', + apiURL: vi.fn((path: string) => `/api${path}`), + lastExecutingMessage: null + } +})) + +vi.mock(import('@/scripts/app'), () => { + const rootGraph = { getNodeById: vi.fn(), nodes: [] } + return { + app: { + rootGraph, + rootGraphOrUndefined: rootGraph, + revokePreviews: vi.fn(), + nodePreviewImages: {} + } + } +}) + +const WORKFLOW_A_ID = 'workflow-a-id' +const WORKFLOW_B_ID = 'workflow-b-id' + +function workflow(id: string, path: string): LoadedComfyWorkflow { + return fromPartial({ + activeState: { id }, + initialState: { id }, + path, + instanceId: `instance-${path}` + }) +} + +const workflowA = workflow(WORKFLOW_A_ID, 'workflows/a.json') +const workflowB = workflow(WORKFLOW_B_ID, 'workflows/b.json') + +const workflowACopy = workflow(WORKFLOW_A_ID, 'workflows/a copy.json') + +/** + * `executing` is dispatched with a bare node id for extension compatibility, + * so the store reads the ids off the raw message api recorded. Mirror that. + */ +function fireExecuting( + node: string | null, + ids: Pick +) { + api.lastExecutingMessage = { node, ...ids } + const handler = apiEventHandlers.get('executing') + if (!handler) throw new Error('executing handler not bound') + handler(new CustomEvent('executing', { detail: node })) +} + +function fire( + event: K, + detail: EventPayloads[K] +) { + const handler = apiEventHandlers.get(event) + if (!handler) throw new Error(`${event} handler not bound`) + handler(new CustomEvent(event, { detail })) + // Some frames are RAF-coalesced in the store; flush unconditionally so the + // assertion sees the applied state rather than the pending batch, and so + // this does not have to track which handlers are coalesced. + vi.advanceTimersToNextFrame() +} + +function nodeState( + jobId: string, + nodeId: string, + state: NodeProgressState['state'], + value = 0 +): NodeProgressState { + return { + node_id: nodeId, + display_node_id: nodeId, + real_node_id: nodeId, + prompt_id: jobId, + state, + value, + max: 10 + } +} + +/** + * Open these tabs through the real store, so `isOpen` and `openWorkflows` are + * the production implementations rather than a copy of them. A copy would keep + * passing if `isOpen` started keying by instance instead of path, which is + * exactly the identity mix-up this suite exists to catch. + * + * Additive, and a no-op for a tab already open. Pinia is rebuilt per test, so + * nothing accumulates across tests. + */ +function openTabs(...workflows: LoadedComfyWorkflow[]) { + const workflowStore = useWorkflowStore() + for (const workflow of workflows) { + if (workflowStore.isOpen(workflow)) continue + workflowStore.attachWorkflow(workflow, workflowStore.openWorkflows.length) + } +} + +describe('executionStore workflow gating', () => { + let store: ReturnType + beforeEach(() => { + apiEventHandlers.clear() + api.lastExecutingMessage = null + vi.mocked( + useNodeOutputStore().revokePreviewsByExecutionId + ).mockImplementation(() => {}) + useWorkflowStore().activeWorkflow = null + openTabs(workflowA, workflowB) + store = useExecutionStore() + store.bindExecutionEvents() + }) + + function queueLegacyJobFrom(jobId: string, wf: LoadedComfyWorkflow) { + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + startTime: 42, + submissionAcceptedAt: 62, + workflow: wf, + mode: 'graph' + }) + } + + function queueJobFrom(jobId: string, wf: LoadedComfyWorkflow) { + const graphId = wf.activeState.id ?? wf.initialState.id + if (!graphId) throw new Error('workflow graph id missing') + store.registerJobWorkflowIdMapping(jobId, graphId) + store.storeJob({ + nodes: ['1'], + id: jobId, + promptOutput: { '1': { inputs: {}, class_type: 'TestNode' } }, + startTime: 42, + submissionAcceptedAt: 62, + workflow: wf, + mode: 'graph' + }) + } + + describe('execution_start', () => { + it('adopts activeJobId for a frame from the active workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('does not steal activeJobId for a frame from another workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('still records the other workflow as running', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + + expect(store.getWorkflowStatus(workflowB)).toBe('running') + expect(store.activeJobId).toBeNull() + }) + + it('adopts the job when there is no active workflow at all', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-a') + }) + }) + + describe('node completion frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + // Two handlers crossed with own-versus-foreign. `executed` and + // `execution_cached` are separate code paths, so all four rows stay. + it.for([ + { event: 'executed', own: true }, + { event: 'executed', own: false }, + { event: 'execution_cached', own: true }, + { event: 'execution_cached', own: false } + ])('$event marks the active job nodes: $own', ({ event, own }) => { + const ids = own + ? { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID } + : { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID } + if (event === 'executed') { + fire('executed', { ...ids, node: '1', display_node: '1', output: {} }) + } else { + fire('execution_cached', { ...ids, nodes: ['1'], timestamp: 2 }) + } + + expect(store.activeJob?.nodes['1']).toBe(own ? true : undefined) + }) + + it('does not apply an unresolvable frame to another active job', () => { + fire('executed', { + prompt_id: 'job-x', + node: '1', + display_node: '1', + output: {} + }) + + expect(store.activeJob?.nodes['1']).toBeUndefined() + }) + }) + + describe('terminal frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('execution_success from another workflow does not clear activeJobId', () => { + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('unresolvable execution_error does not reset another active job', () => { + fire('execution_error', { + prompt_id: 'job-x', + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [], + executed: [], + current_inputs: {}, + current_outputs: {} + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('still marks another workflow completed on its own tab', () => { + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + expect(store.getWorkflowStatus(workflowB)).toBe('running') + + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.getWorkflowStatus(workflowB)).toBe('completed') + expect(store.activeJobId).toBe('job-a') + }) + + it('records a background completion only on its owning workflow', () => { + queueJobFrom('job-b', workflowB) + + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.getWorkflowStatus(workflowA)).toBeUndefined() + expect(store.getWorkflowStatus(workflowB)).toBe('completed') + }) + + it('execution_success from the active workflow clears it', () => { + fire('execution_success', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBeNull() + }) + + it('clears the initializing flag for the completed job either way', () => { + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) + + fire('execution_success', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect(store.isJobInitializing('job-b')).toBe(false) + }) + + it('execution_interrupted from another workflow does not clear activeJobId', () => { + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + executed: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('handles a gateway-synthesised cancel with no node fields', () => { + queueJobFrom('job-b', workflowB) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 4) } + }) + + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + synthetic: true, + reason: 'Job cancelled by user' + }) + + expect(store.activeJobId, 'the visible run is untouched').toBe('job-a') + expect( + store.nodeProgressStatesByJob['job-b'], + "the cancelled job's own records are still released" + ).toBeUndefined() + }) + + it.for([ + { kind: 'a node error', nodeId: '1' as string | null }, + { kind: 'a service-level error', nodeId: null } + ])( + '$kind from another workflow releases that job and leaves the visible one alone', + ({ nodeId }) => { + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 2) } + }) + expect(store.nodeProgressStatesByJob['job-b']).toBeDefined() + + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: nodeId, + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [], + executed: [] + }) + + expect(store.activeJobId, 'the visible run is untouched').toBe('job-a') + expect(store.isJobInitializing('job-b')).toBe(false) + expect( + store.nodeProgressStatesByJob['job-b'], + "the failed background job's own records are released" + ).toBeUndefined() + expect(store.queuedJobs['job-b']).toBeUndefined() + } + ) + + it('does not reset the active workflow when a stamped background error arrives before storeJob', () => { + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'TestNode', + exception_message: 'fail', + exception_type: 'Error', + traceback: [], + executed: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('does not reset the active workflow for a background account precondition', () => { + queueJobFrom('job-b', workflowB) + + fire('execution_error', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2, + node_id: '1', + node_type: 'PartnerApiNode', + exception_message: + 'Payment Required: Please add credits to your account to use this node.', + exception_type: 'InsufficientFundsError', + traceback: [], + executed: [] + }) + + expect(store.activeJobId).toBe('job-a') + }) + }) + + describe('progress_state', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('updates the global mirror when the frame matches the active workflow', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(3) + }) + + it('skips the global mirror when the frame is from another workflow', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + + it('records per-job progress regardless of which workflow sent it', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(7) + }) + + it('retains the latest progress snapshot for each job in one frame', () => { + const handler = apiEventHandlers.get('progress_state') + assert(handler) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + } + }) + ) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + } + }) + ) + + vi.advanceTimersToNextFrame() + + expect(store.nodeProgressStatesByJob['job-a']?.['1']?.value).toBe(3) + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(7) + }) + + it('updates the mirror when there is no active workflow', () => { + useWorkflowStore().activeWorkflow = null + + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(7) + }) + + it('updates the executing node progress on a workflow_id match', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fireExecuting('1', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 4) } + }) + + expect(store.executingNodeProgress).toBe(0.4) + }) + + it('skips the executing node progress on a workflow_id mismatch', () => { + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fireExecuting('1', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 9) } + }) + + expect(store.executingNodeProgress).toBeNull() + }) + }) + + describe('preview revocation', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('revokes previews for a frame from the active workflow', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running') } + }) + + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalled() + }) + + it('does not revoke previews for a frame from another workflow', () => { + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running') } + }) + + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).not.toHaveBeenCalled() + }) + + it('revokes when a node transitions pending -> running', () => { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'pending') } + }) + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).not.toHaveBeenCalled() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running') } + }) + + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalled() + }) + + it('does not revoke twice while a node stays running', () => { + const running = { '1': nodeState('job-a', '1', 'running') } + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: running + }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: running + }) + + expect( + useNodeOutputStore().revokePreviewsByExecutionId + ).toHaveBeenCalledTimes(1) + }) + }) + + describe('legacy backends that never send workflow_id', () => { + it('falls back to the job -> workflow id mapping', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + fire('execution_start', { prompt_id: 'job-a', timestamp: 1 }) + + fire('progress_state', { + prompt_id: 'job-b', + nodes: { '1': nodeState('job-b', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(5) + }) + + it('uses the mapping to accept a frame for the active workflow', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(5) + }) + + it('stays permissive when ownership cannot be resolved at all', () => { + useWorkflowStore().activeWorkflow = workflowA + + fire('progress_state', { + prompt_id: 'job-from-elsewhere', + nodes: { '1': nodeState('job-from-elsewhere', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(5) + }) + + it('does not steal activeJobId from a mapped foreign job', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + fire('execution_start', { prompt_id: 'job-b', timestamp: 1 }) + + expect(store.activeJobId).toBeNull() + }) + }) + + describe('metadata hygiene', () => { + it('ignores an empty-string workflow_id rather than matching on it', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: '', + nodes: { '1': nodeState('job-b', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + + it('gates on workflow_id even when the job mapping disagrees', () => { + useWorkflowStore().activeWorkflow = workflowA + store.registerJobWorkflowIdMapping('job-x', WORKFLOW_A_ID) + + fire('progress_state', { + prompt_id: 'job-x', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-x', '1', 'running', 5) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + }) + + describe('executing frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('does not let another workflow clear the visible active job', () => { + fireExecuting(null, { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID + }) + + expect(store.activeJobId).toBe('job-a') + }) + + it('clears the active job on the visible workflow own terminal executing', () => { + fireExecuting(null, { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID + }) + + expect(store.activeJobId).toBeNull() + }) + + it('does not clear node progress for another workflow executing frame', () => { + fireExecuting('1', { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 5) } + }) + expect(store.executingNodeProgress).toBe(0.5) + + fireExecuting('9', { prompt_id: 'job-b', workflow_id: WORKFLOW_B_ID }) + + expect(store.executingNodeProgress).toBe(0.5) + }) + + it('still works when the raw message was never recorded', () => { + const handler = apiEventHandlers.get('executing') + assert.exists(handler) + handler(new CustomEvent('executing', { detail: null })) + + expect(store.activeJobId).toBeNull() + }) + }) + + describe('progress frames', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + }) + + it('applies a progress frame from the active workflow', () => { + fire('progress', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + node: '1', + value: 4, + max: 10 + }) + + expect(store.executingNodeProgress).toBe(0.4) + }) + + it('drops a progress frame from another workflow', () => { + fire('progress', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + node: '1', + value: 7, + max: 10 + }) + + expect(store.executingNodeProgress).toBeNull() + }) + }) + + describe('mixed backend, upgraded mid-session', () => { + beforeEach(() => { + useWorkflowStore().activeWorkflow = workflowA + }) + + it('accepts its own frames before and after the field appears', () => { + queueJobFrom('job-a', workflowA) + + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 2) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(2) + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(6) + }) + + it('rejects a foreign job both before and after the field appears', () => { + queueJobFrom('job-b', workflowB) + + fire('progress_state', { + prompt_id: 'job-b', + nodes: { '1': nodeState('job-b', '1', 'running', 2) } + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-b']?.['1']?.value).toBe(6) + }) + }) + + describe('switching tabs mid-run', () => { + it('clears the mirror when switching to a workflow with no running job', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(3) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.executingNodeProgress).toBeNull() + }) + + it('replays the active workflow own progress when switching back', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + expect(store.nodeProgressStates['1']).toBeUndefined() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 8) } + }) + + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + + expect(store.nodeProgressStates['1']?.value).toBe(8) + }) + + it('keeps recording per-job progress for the backgrounded workflow', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 8) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeProgressStatesByJob['job-a']?.['1']?.value).toBe(8) + }) + + it('shows the newly active workflow own job rather than the one it replaced', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 2) } + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 7) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(2) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']?.value).toBe(7) + expect(store.nodeProgressStates['1']?.prompt_id).toBe('job-b') + }) + it('clears the mirror on a backend that sends no workflow id', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueLegacyJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(store.nodeProgressStates['1']?.value).toBe(3) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + it('stays clean across repeated switches during one run', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + async function cycle(step: number) { + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', step) } + }) + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + const inB = store.nodeProgressStates['1'] + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + return { inB, inA: store.nodeProgressStates['1']?.value } + } + + const first = await cycle(3) + expect(first.inB, 'first arrival in B').toBeUndefined() + expect(first.inA).toBe(3) + + const second = await cycle(5) + expect(second.inB, 'second arrival in B').toBeUndefined() + expect(second.inA).toBe(5) + }) + + it('stays clean while the active workflow is queued behind the running one', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + useWorkflowStore().activeWorkflow = workflowB + queueJobFrom('job-b', workflowB) + await nextTick() + expect(store.nodeProgressStates['1']).toBeUndefined() + + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 6) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + }) + it('does not project a foreign entry that is still in the mirror', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + expect(Object.keys(store.nodeLocationProgressStates)).toHaveLength(1) + + useWorkflowStore().activeWorkflow = workflowB + + expect(store.nodeProgressStates['1']?.value).toBe(3) + expect(store.nodeLocationProgressStates).toEqual({}) + }) + + it('projects the active workflow own entry', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + const projected = Object.values(store.nodeLocationProgressStates) + expect(projected).toHaveLength(1) + expect(projected[0]?.value).toBe(3) + }) + }) + + describe('two open workflows that share an id', () => { + beforeEach(() => { + openTabs(workflowA, workflowACopy) + }) + + it('rejects a foreign job output frame for the copy', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-copy', workflowACopy) + + useWorkflowStore().activeWorkflow = workflowACopy + + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) + expect(store.belongsToActiveWorkflow('job-copy', WORKFLOW_A_ID)).toBe( + true + ) + }) + + it('does not project a foreign job progress onto the copy', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-copy', workflowACopy) + + useWorkflowStore().activeWorkflow = workflowACopy + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + + expect(store.nodeProgressStates['1']).toBeUndefined() + expect(store.nodeLocationProgressStates).toEqual({}) + }) + + it('still renders the copy own job', () => { + useWorkflowStore().activeWorkflow = workflowACopy + queueJobFrom('job-copy', workflowACopy) + fire('progress_state', { + prompt_id: 'job-copy', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-copy', '1', 'running', 4) } + }) + + expect(store.nodeProgressStates['1']?.value).toBe(4) + }) + }) + + describe('ownership resolution precedence', () => { + beforeEach(() => { + openTabs(workflowA, workflowACopy) + }) + + it('prefers the tab instance over both weaker legs', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + store.ensureSessionWorkflowPath( + 'job-a', + workflowACopy.path, + workflowA.instanceId + ) + + useWorkflowStore().activeWorkflow = workflowACopy + expect( + store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID), + 'the instance still says A, so the copy does not own it' + ).toBe(false) + useWorkflowStore().activeWorkflow = workflowA + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) + }) + + it('falls back to the session path when no instance is known', () => { + useWorkflowStore().activeWorkflow = workflowA + store.ensureSessionWorkflowPath('job-pathonly', workflowACopy.path) + + expect( + store.belongsToActiveWorkflow('job-pathonly', WORKFLOW_A_ID), + 'the path says the copy owns it, even though the graph id matches A' + ).toBe(false) + useWorkflowStore().activeWorkflow = workflowACopy + expect(store.belongsToActiveWorkflow('job-pathonly', WORKFLOW_A_ID)).toBe( + true + ) + }) + + it('falls back to the graph id when neither instance nor path is known', () => { + useWorkflowStore().activeWorkflow = workflowA + store.registerJobWorkflowIdMapping('job-elsewhere', WORKFLOW_B_ID) + + expect( + store.belongsToActiveWorkflow('job-elsewhere', WORKFLOW_B_ID) + ).toBe(false) + expect( + store.belongsToActiveWorkflow('job-elsewhere', WORKFLOW_A_ID) + ).toBe(true) + }) + + it('stays permissive when ownership cannot be resolved at all', () => { + useWorkflowStore().activeWorkflow = workflowA + + expect(store.belongsToActiveWorkflow('job-unknown', undefined)).toBe(true) + }) + + it('does not give a running job to the copy that Save As creates', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + const savedAs = workflow( + 'a-brand-new-graph-id', + 'workflows/saved as.json' + ) + openTabs(workflowA, savedAs) + useWorkflowStore().activeWorkflow = savedAs + + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(false) + useWorkflowStore().activeWorkflow = workflowA + expect(store.belongsToActiveWorkflow('job-a', WORKFLOW_A_ID)).toBe(true) + }) + + // Core falls back to the id inside extra_pnginfo.workflow, so a graph still + // on LGraph's default id has the sentinel stamped onto its frames. Compared + // as a real id it matches nothing, so the tab's own frames were rejected. + it('does not reject a frame whose only id is the all-zero sentinel', () => { + useWorkflowStore().activeWorkflow = workflowA + + expect( + store.belongsToActiveWorkflow('job-fresh-graph', zeroUuid), + 'the sentinel is not an id, so it cannot be grounds for rejection' + ).toBe(true) + }) + }) + + describe('returning to a tab whose job started in the background', () => { + it('adopts the job so the tab is not both running and idle', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + + useWorkflowStore().activeWorkflow = workflowB + await nextTick() + + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + nodes: { '1': nodeState('job-a', '1', 'running', 3) } + }) + useWorkflowStore().activeWorkflow = workflowA + await nextTick() + + expect(store.nodeProgressStates['1']?.state).toBe('running') + expect(store.activeJobId).toBe('job-a') + expect(store.isIdle).toBe(false) + }) + + it('does not adopt a job belonging to a different tab', async () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 1 + }) + fire('progress_state', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 3) } + }) + + await nextTick() + + expect(store.activeJobId).toBeNull() + expect(store.isIdle).toBe(true) + }) + }) + + describe('per-job state that a neighbouring job must not disturb', () => { + it('keeps a background pending snapshot when the visible job ends', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + // B's frame and A's terminal `executing` land in the same tick. + const handler = apiEventHandlers.get('progress_state') + assert.exists(handler) + handler( + new CustomEvent('progress_state', { + detail: { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + nodes: { '1': nodeState('job-b', '1', 'running', 6) } + } + }) + ) + fireExecuting(null, { prompt_id: 'job-a', workflow_id: WORKFLOW_A_ID }) + vi.advanceTimersToNextFrame() + + expect( + store.nodeProgressStatesByJob['job-b'], + "A ending must not drop B's pending snapshot" + ).toBeDefined() + }) + + it('clears only the interrupted job initializing flag', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + queueJobFrom('job-b', workflowB) + fire('notification', { id: 'job-a', value: 'Waiting for a machine' }) + fire('notification', { id: 'job-b', value: 'Waiting for a machine' }) + expect(store.isJobInitializing('job-b')).toBe(true) + // Set directly rather than via execution_start, which would clear A's own + // initializing flag and make the assertion below vacuous. + store.activeJobId = 'job-a' + + fire('execution_interrupted', { + prompt_id: 'job-b', + workflow_id: WORKFLOW_B_ID, + timestamp: 2 + }) + + expect( + store.isJobInitializing('job-a'), + 'the visible run is still waiting for a machine' + ).toBe(true) + expect(store.isJobInitializing('job-b')).toBe(false) + }) + }) + + describe('the all-zero sentinel, one row per normalization', () => { + it('leaves a sentinel-only frame unresolvable, so activeJobId rejects it', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + + expect(store.belongsToActiveWorkflow('job-x', zeroUuid)).toBe(false) + }) + + it('adopts an unmapped job stamped with the sentinel', () => { + useWorkflowStore().activeWorkflow = workflowA + + fire('execution_start', { + prompt_id: 'job-fresh', + workflow_id: zeroUuid, + timestamp: 1 + }) + + expect(store.activeJobId).toBe('job-fresh') + }) + }) + + describe('a background job that finishes', () => { + it('does not leave activeJobId naming a finished background job', () => { + useWorkflowStore().activeWorkflow = workflowA + queueJobFrom('job-a', workflowA) + fire('execution_start', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 1 + }) + useWorkflowStore().activeWorkflow = workflowB + fire('execution_success', { + prompt_id: 'job-a', + workflow_id: WORKFLOW_A_ID, + timestamp: 2 + }) + + expect(store.activeJobId).toBeNull() + expect(store.isIdle).toBe(true) + }) + }) +}) diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index b62b1858d0b..97de9c45c1c 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -4,6 +4,7 @@ import type { JobListItem } from '@/platform/remote/comfyui/jobs/jobTypes' import type { TaskOutput } from '@/platform/remote/comfyui/execution/types' import { api } from '@/scripts/api' import { useExecutionStore } from '@/stores/executionStore' +import { toNodeId } from '@/types/nodeId' import { TaskItemImpl, useQueuePendingTaskCountStore, @@ -354,6 +355,7 @@ describe('useQueueStore', () => { beforeEach(() => { store = useQueueStore() + vi.mocked(api.getJobDetail).mockResolvedValue(undefined) }) describe('initial state', () => { @@ -760,6 +762,127 @@ describe('useQueueStore', () => { }) }) + describe('terminal-job reconciliation', () => { + it('clears progress for a job the backend reports only in history', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + 'hist-stuck': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'hist-stuck' + } + } + } + + vi.mocked(api.getQueue).mockResolvedValue({ Running: [], Pending: [] }) + vi.mocked(api.getHistory).mockResolvedValue([ + createHistoryJob(1, 'hist-stuck') + ]) + + await store.update() + + expect( + executionStore.nodeProgressStatesByJob['hist-stuck'] + ).toBeUndefined() + }) + + it('looks up a tracked job that fell outside the history page', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + 'older-stuck': { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'older-stuck' + } + } + } + + vi.mocked(api.getQueue).mockResolvedValue({ Running: [], Pending: [] }) + vi.mocked(api.getHistory).mockResolvedValue([ + createHistoryJob(2, 'newer-job') + ]) + vi.mocked(api.getJobDetail).mockResolvedValue( + createHistoryJob(1, 'older-stuck') + ) + + await store.update() + + expect( + executionStore.nodeProgressStatesByJob['older-stuck'] + ).toBeUndefined() + }) + + it('evicts a tracked job only after a second failed lookup', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + ghost: { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'ghost' + } + } + } + + vi.mocked(api.getQueue).mockResolvedValue({ Running: [], Pending: [] }) + vi.mocked(api.getHistory).mockResolvedValue([]) + vi.mocked(api.getJobDetail).mockResolvedValue(undefined) + + await store.update() + expect( + executionStore.nodeProgressStatesByJob['ghost'], + 'one miss could be a network blip, so it is still tracked' + ).toBeDefined() + + await store.update() + expect( + executionStore.nodeProgressStatesByJob['ghost'], + 'two consecutive misses means it is gone' + ).toBeUndefined() + }) + + it('resets the miss count when the tracked job reappears', async () => { + const executionStore = useExecutionStore() + executionStore.nodeProgressStatesByJob = { + ghost: { + '1': { + value: 3, + max: 10, + state: 'running', + node_id: toNodeId('1'), + prompt_id: 'ghost' + } + } + } + + vi.mocked(api.getQueue) + .mockResolvedValueOnce({ Running: [], Pending: [] }) + .mockResolvedValueOnce({ + Running: [createRunningJob(1, 'ghost')], + Pending: [] + }) + .mockResolvedValueOnce({ Running: [], Pending: [] }) + vi.mocked(api.getHistory).mockResolvedValue([]) + vi.mocked(api.getJobDetail) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined) + + await store.update() + await store.update() + await store.update() + + expect(executionStore.trackedJobIds).toContain('ghost') + }) + }) + describe('update() - maxHistoryItems limit', () => { it('should enforce maxHistoryItems limit', async () => { store.maxHistoryItems = 3 diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index bae24435ff6..e53597362ae 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -5,9 +5,11 @@ import { extractWorkflow } from '@/platform/remote/comfyui/jobs/fetchJobs' import type { APITaskType, JobListItem, + JobStatus, TaskType } from '@/platform/remote/comfyui/jobs/jobTypes' import type { + JobId, StatusWsMessageStatus, TaskOutput } from '@/platform/remote/comfyui/execution/types' @@ -315,6 +317,109 @@ export const useQueueStore = defineStore('queue', () => { () => pendingTasks.value.length + runningTasks.value.length ) + function applyQueueSnapshot(queue: Awaited>) { + // API returns pre-sorted data (sort_by=create_time&order=desc) + runningTasks.value = queue.Running.map((job) => new TaskItemImpl(job)) + pendingTasks.value = queue.Pending.map((job) => new TaskItemImpl(job)) + + const appearedTasks = [...pendingTasks.value, ...runningTasks.value] + const executionStore = useExecutionStore() + appearedTasks.forEach((task) => { + const jobIdString = task.jobId + const workflowId = task.workflowId + if (workflowId && jobIdString) { + executionStore.registerJobWorkflowIdMapping(jobIdString, workflowId) + } + }) + } + + function applyHistorySnapshot( + history: Awaited> + ) { + const currentHistory = toValue(historyTasks) + + // Sort by create_time descending and limit to maxItems + const sortedHistory = [...history] + .sort((a, b) => b.create_time - a.create_time) + .slice(0, toValue(maxHistoryItems)) + + // Reuse existing TaskItemImpl instances or create new + // Must recreate if outputs_count changed (e.g., API started returning it) + const existingByJobId = new Map( + currentHistory.map((impl) => [impl.jobId, impl]) + ) + + const nextHistoryTasks = sortedHistory.map((job) => { + const existing = existingByJobId.get(job.id) + if (!existing) return new TaskItemImpl(job) + // Recreate if outputs_count changed to ensure lazy loading works + if ( + existing.outputsCount !== (job.outputs_count ?? undefined) || + existing.previewableOutputsCount !== + (job.previewable_outputs_count ?? undefined) + ) { + return new TaskItemImpl(job) + } + return existing + }) + + const isHistoryUnchanged = + nextHistoryTasks.length === currentHistory.length && + nextHistoryTasks.every((task, index) => task === currentHistory[index]) + + if (!isHistoryUnchanged) { + historyTasks.value = nextHistoryTasks + } + hasFetchedHistorySnapshot.value = true + } + + const terminalStatuses = new Set([ + 'completed', + 'failed', + 'cancelled' + ]) + + /** + * Consecutive failed `/jobs/{id}` lookups per tracked job, for the eviction + * rule in {@link reconcileTrackedJobs}. + */ + const missedJobLookups = new Map() + + async function reconcileTrackedJobs( + activeJobIds: Set, + history: JobListItem[] + ) { + const executionStore = useExecutionStore() + const terminalJobIds = new Set(history.map((job) => job.id)) + const unresolvedJobIds = executionStore.trackedJobIds.filter( + (jobId) => !activeJobIds.has(jobId) && !terminalJobIds.has(jobId) + ) + for (const jobId of missedJobLookups.keys()) { + if (!unresolvedJobIds.includes(jobId)) missedJobLookups.delete(jobId) + } + const details = await Promise.all( + unresolvedJobIds.map((jobId) => api.getJobDetail(jobId)) + ) + unresolvedJobIds.forEach((jobId, index) => { + const detail = details[index] + if (detail) { + missedJobLookups.delete(jobId) + if (terminalStatuses.has(detail.status)) terminalJobIds.add(detail.id) + return + } + // fetchJobDetail returns undefined for a network error too, so wait for a + // second miss. + const misses = (missedJobLookups.get(jobId) ?? 0) + 1 + if (misses >= 2) { + missedJobLookups.delete(jobId) + terminalJobIds.add(jobId) + } else { + missedJobLookups.set(jobId, misses) + } + }) + executionStore.reconcileTerminalJobs(activeJobIds, terminalJobIds) + } + const update = async () => { if (updateState.inFlight) { updateState.dirty = true @@ -330,70 +435,32 @@ export const useQueueStore = defineStore('queue', () => { api.getHistory(maxHistoryItems.value) ]) - if (queueResult.status === 'fulfilled') { - const queue = queueResult.value - // API returns pre-sorted data (sort_by=create_time&order=desc) - runningTasks.value = queue.Running.map((job) => new TaskItemImpl(job)) - pendingTasks.value = queue.Pending.map((job) => new TaskItemImpl(job)) - - const appearedTasks = [...pendingTasks.value, ...runningTasks.value] - const executionStore = useExecutionStore() - appearedTasks.forEach((task) => { - const jobIdString = task.jobId - const workflowId = task.workflowId - if (workflowId && jobIdString) { - executionStore.registerJobWorkflowIdMapping(jobIdString, workflowId) - } - }) + // Shared by the queue and history branches below. `null` means the queue + // fetch failed, which is not the same as an empty queue. + const activeJobIds: Set | null = + queueResult.status === 'fulfilled' + ? new Set([ + ...queueResult.value.Running.map((j) => j.id), + ...queueResult.value.Pending.map((j) => j.id) + ]) + : null - const activeJobIds = new Set([ - ...queue.Running.map((j) => j.id), - ...queue.Pending.map((j) => j.id) - ]) - executionStore.reconcileInitializingJobs(activeJobIds) + if (queueResult.status === 'fulfilled') { + applyQueueSnapshot(queueResult.value) } else { console.error('Failed to fetch queue:', queueResult.reason) } - if (historyResult.status === 'fulfilled') { - const history = historyResult.value - const currentHistory = toValue(historyTasks) - - // Sort by create_time descending and limit to maxItems - const sortedHistory = [...history] - .sort((a, b) => b.create_time - a.create_time) - .slice(0, toValue(maxHistoryItems)) - - // Reuse existing TaskItemImpl instances or create new - // Must recreate if outputs_count changed (e.g., API started returning it) - const existingByJobId = new Map( - currentHistory.map((impl) => [impl.jobId, impl]) + if (activeJobIds) { + useExecutionStore().reconcileInitializingJobs(activeJobIds) + await reconcileTrackedJobs( + activeJobIds, + historyResult.status === 'fulfilled' ? historyResult.value : [] ) + } - const nextHistoryTasks = sortedHistory.map((job) => { - const existing = existingByJobId.get(job.id) - if (!existing) return new TaskItemImpl(job) - // Recreate if outputs_count changed to ensure lazy loading works - if ( - existing.outputsCount !== (job.outputs_count ?? undefined) || - existing.previewableOutputsCount !== - (job.previewable_outputs_count ?? undefined) - ) { - return new TaskItemImpl(job) - } - return existing - }) - - const isHistoryUnchanged = - nextHistoryTasks.length === currentHistory.length && - nextHistoryTasks.every( - (task, index) => task === currentHistory[index] - ) - - if (!isHistoryUnchanged) { - historyTasks.value = nextHistoryTasks - } - hasFetchedHistorySnapshot.value = true + if (historyResult.status === 'fulfilled') { + applyHistorySnapshot(historyResult.value) } else { console.error('Failed to fetch history:', historyResult.reason) }