Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
52cc35e
test(execution): cover the workflow ownership gate
christian-byrne Oct 26, 2026
896c87d
feat(execution): send the workflow id and use it to place unmapped jobs
christian-byrne Oct 26, 2026
2ab043d
feat(execution): forward the stamped workflow id to the gate
christian-byrne Oct 26, 2026
60010c6
fix(execution): release the tab badge when a job is forgotten
christian-byrne Oct 26, 2026
a3c1a29
docs(adr): record workflow ownership for execution frames
christian-byrne Oct 26, 2026
70305d2
Merge origin/main into cbyrne/ws-workflow-metadata-contract (automati…
christian-byrne Oct 29, 2026
3c1c42b
fix(execution): complete the main merge and implement the review find…
christian-byrne Oct 29, 2026
d7c0105
fix: preserve running status for overlapping jobs
christian-byrne Oct 29, 2026
e9270bd
fix(execution): keep a newer terminal badge through stale-job cleanup
christian-byrne Oct 29, 2026
a8e2594
test(execution): implement the review findings on the ownership tests
christian-byrne Oct 29, 2026
24b64d6
Merge origin/main into cbyrne/ws-workflow-metadata-contract
christian-byrne Oct 31, 2026
0cf7350
chore(architecture): refresh the domain catalog for this PR's new tes…
christian-byrne Oct 31, 2026
5b5400b
Merge remote-tracking branch 'origin/main' into cbyrne/ws-workflow-me…
christian-byrne Nov 4, 2026
eb29b67
refactor(api-client): derive the workflow_metadata wire types from co…
christian-byrne Nov 4, 2026
5f19223
fix(api-client): keep the generated types out of the published declar…
christian-byrne Nov 4, 2026
feca331
fix(types): bundle the generated package into the published declarations
christian-byrne Nov 4, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci-oss-assets-validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ jobs:
if pnpx license-checker-rseidelsohn@4 \
--production \
--summary \
--excludePackages '@comfyorg/comfyui-frontend;@comfyorg/account-core;@comfyorg/account-ui;@comfyorg/api-client;@comfyorg/billing-contract;@comfyorg/billing-core;@comfyorg/design-system;@comfyorg/graph-model;@comfyorg/ingest-types;@comfyorg/keybindings;@comfyorg/layer-editor;@comfyorg/mask-editor;@comfyorg/registry-types;@comfyorg/settings-core;@comfyorg/shared-frontend-utils;@comfyorg/tailwind-utils;@comfyorg/telemetry;@comfyorg/test-utils;@comfyorg/viewport3d;@comfyorg/comfyui-electron-types;@comfyorg/comfy-multi-player' \
--excludePackages '@comfyorg/comfyui-frontend;@comfyorg/account-core;@comfyorg/comfyui-api-types;@comfyorg/account-ui;@comfyorg/api-client;@comfyorg/billing-contract;@comfyorg/billing-core;@comfyorg/design-system;@comfyorg/graph-model;@comfyorg/ingest-types;@comfyorg/keybindings;@comfyorg/layer-editor;@comfyorg/mask-editor;@comfyorg/registry-types;@comfyorg/settings-core;@comfyorg/shared-frontend-utils;@comfyorg/tailwind-utils;@comfyorg/telemetry;@comfyorg/test-utils;@comfyorg/viewport3d;@comfyorg/comfyui-electron-types;@comfyorg/comfy-multi-player' \
--clarificationsFile .github/license-clarifications.json \
--onlyAllow 'MIT;MIT*;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC;0BSD;BlueOak-1.0.0;Python-2.0;CC0-1.0;Unlicense;(MIT OR Apache-2.0);(MIT OR GPL-3.0);(Apache-2.0 OR MIT);(MPL-2.0 OR Apache-2.0);MPL-2.0;CC-BY-4.0;CC-BY-3.0;GPL-3.0-only'; then
echo ''
Expand Down
94 changes: 94 additions & 0 deletions browser_tests/tests/wsReconnectStaleJob.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,100 @@ async function triggerReconnect(
}

test.describe('WebSocket reconnect with stale job', { tag: '@ui' }, () => {
test('keeps the visible workflow running status when cleanup forgets one job', async ({
comfyPage,
getWebSocket,
nextWebSocket
}) => {
const ws = await getWebSocket()
const exec = new ExecutionHelper(comfyPage, ws)
const topbar = comfyPage.menu.topbar

const { orphanedIndicator, sharedIndicator, staleJobIds, runningJobId } =
await test.step('Strand one job per workflow behind a live sibling', async () => {
await comfyPage.workflow.waitForActiveWorkflow()
await comfyPage.workflow.waitForWorkflowIdle()

const orphanedJobId = await exec.run()
exec.executionStart(orphanedJobId)

await comfyPage.workflow.loadWorkflow('nodes/single_save_image_node')
await comfyPage.workflow.waitForWorkflowIdle()

const sharedStaleJobId = await exec.run()
const runningJobId = await exec.run()
exec.executionStart(sharedStaleJobId)
exec.executionStart(runningJobId)

await comfyPage.workflow.newBlankWorkflow()
await comfyPage.workflow.waitForWorkflowIdle()
await expect(topbar.tabs).toHaveCount(3)

const orphanedIndicator = topbar
.getTab(0)
.getByRole('img', { name: 'Running' })
const sharedIndicator = topbar
.getTab(1)
.getByRole('img', { name: 'Running' })
await expect(orphanedIndicator).toBeVisible()
await expect(sharedIndicator).toBeVisible()

return {
orphanedIndicator,
sharedIndicator,
staleJobIds: [orphanedJobId, sharedStaleJobId],
runningJobId
}
})

await test.step('Recover the remaining job after stale-job cleanup', async () => {
const remainingJobScenario: Scenario = {
name: 'preserves the remaining same-workflow job',
queueBody: () =>
JSON.stringify(
jobsResponse([
{ id: runningJobId, status: 'in_progress', create_time: 0 }
])
),
expectsActiveAfter: true
}
const staleJobReads = await Promise.all(
staleJobIds.map((staleJobId) =>
mockJobsRoute(
comfyPage,
new RegExp(`/api/jobs/${staleJobId}$`),
'{}',
404
)
)
)

let socket = ws
for (let pass = 1; pass <= 3; pass++) {
const nextSocketPromise = nextWebSocket()
await triggerReconnect(
comfyPage,
socket,
remainingJobScenario,
runningJobId
)
await expect
.poll(() => staleJobReads.map((reads) => reads() >= 2 * pass))
.toEqual([true, true])
socket = await nextSocketPromise
}

await expect(
orphanedIndicator,
'the workflow whose only job went unreadable must release its badge'
).toBeHidden()
await expect(
sharedIndicator,
'the same cleanup batch must leave the still-running sibling alone'
).toBeVisible()
})
})

test.describe('app mode skeleton', () => {
test.beforeEach(async ({ comfyPage }) => {
await comfyPage.appMode.enterAppModeWithInputs([[KSAMPLER_NODE, 'seed']])
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# ADR-EXECUTION-ROUTING-0049: Workflow ownership for execution websocket frames

Date: 2026-10-26

## Status

Proposed

## Context

One `LGraph` and one `LGraphCanvas` serve every open workflow tab, so execution state written by node id is shared state. A run started in one tab painted its progress and outputs onto whichever tab was in front. Reported on #1293, #2660, #6730 and Comfy-Org/ComfyUI#6581, under the umbrella #3377.

[ADR-GRAPH-DOCUMENT-0026](GRAPH-DOCUMENT-0026-frontend-document-model.md) D2 states the rule: a frame arriving on a push channel resolves its target document at ingress via the job map, never via the active pointer. It also states what to do at the boundary — "events with no job→uid resolution (page reloaded mid-run, jobs queued by another client) go to an explicit _unattributed_ bucket rather than being applied to the active document by default … misdelivery is not the fallback."

`#21666` implemented D2's addressing rule for the Agent, keying the ownership gate on `jobIdToWorkflowInstanceId` and gating the node-output and preview paths. That shipped without unit coverage, and without the wire contract Comfy-Org/ComfyUI#16763 and #16809 were merged to provide: the frontend sent no `workflow_metadata`, so the only id the server could stamp came from `extra_pnginfo`.

This ADR records what the resolution order is, why the graph id is its weakest signal, and that the boundary departs from D2.

## Decision

**Resolution order, strongest first.** The open tab's `instanceId`, then the session path, then the workflow id stamped on the frame. When none resolves, the `activeJobId` guard decides.

`instanceId` is the only identifier unique per open tab. A path identifies one open workflow, so it is next. The frame's own workflow id is weakest, because every copy of a graph shares it: `ensureWorkflowId` keeps the id already present in a pasted graph, so copy a workflow, open a tab and paste gives two tabs carrying one id. Keying on it first would make each tab match the other's jobs. Its value is placing a job that no per-tab mapping knows about at all — one queued before a page reload, or by another client — which is the gap `instanceId` alone cannot close and the reason the wire contract exists.

**The all-zero UUID is not an id.** `LGraph._id` defaults to it and the codebase reads it as "no id yet", replacing it on configure and on clear. It is not sent as a routing key, and an incoming one is discarded rather than compared, because comparing it as a real id matches nothing and rejected a fresh graph's own frames. Withholding it from the request does not keep it off the wire: the same graph goes out under `extra_pnginfo` and the server falls back to the id there.

**The stamp is remembered per job, not re-read per frame.** The first stamped frame of a job records `jobId → workflow_id`, and the workflow-id leg reads it when the caller holds no frame. Three callers hold none: `adoptActiveWorkflowJob` on a tab switch, the `executed` events `useReconnectQueueRefresh` rebuilds by hand on reconnect, and `progress_text` when its backend supplies `prompt_id`. Without it those three skipped the leg and reached the `activeJobId` fallback, so a reconnect replay could paint a foreign job's output onto the visible graph — `handleExecutionStart` enrols a job in `queuedJobs` before the gate runs, and the replay follows every job in that map, so a rejected foreign job was still replayed unstamped.

First stamp wins: core merges `workflow_metadata` at `send_sync` enqueue time rather than at publication, so a frame already queued for a job cannot be relabelled by the next prompt, and a later disagreeing stamp belongs to a different job. The map is separate from `jobIdToWorkflowId` on purpose — that one is reactive, exported, and backs `runErrorKeyForJob`, so recording a foreign job's stamp there would file that job's errors against a local tab whenever the two share a graph id, which is the copy-paste case the resolution order exists to handle. Being populated for foreign jobs, it carries its own FIFO bound rather than relying on the local-job eviction paths.

**A rejection by a real identifier is not a diagnostic.** Only a rejection decided by the `activeJobId` fallback is reported, because that is the one leg that guesses. The resolver returns which leg decided; a mismatched instance, path or workflow id is a definite answer. Reporting every rejection raised one `warning_rejecting_unowned_execution_start` per foreign job start on ordinary traffic, since Cloud's per-user fan-out shows every open tab every other tab's job starts.

**Unresolvable frames fall back to the active job.** This is the documented exception to D2's boundary rule. Such a frame is delivered to the tab in front when no job is active or the frame belongs to `activeJobId`, rather than quarantined. A frame with no `prompt_id` is always delivered.

**What is gated is decided by the consumer's key, not by the frame being foreign.** Node-keyed writes are gated, because the visible graph may hold the same node ids. Job-keyed writes are not: `useJobPreviewStore().setPreviewUrl` feeds the queue panel thumbnail for a background run. Account-level signals are not, because a sign-in or credit precondition belongs to the account rather than to one workflow.

## Alternatives considered

**Quarantine unresolvable frames, as D2 prescribes.** Rejected. The common unresolvable case is a page reloaded mid-run, where the mappings are gone and the only open tab is the one being looked at. Quarantining would stop a single-tab user's own run updating after a refresh, a regression for the common case in service of a rule written for the multi-tab case. D2's boundary also assumes a surfacing UI for the unattributed bucket, which does not exist.

**Gate on the workflow id alone.** This was tried on #20045 and was wrong, for the copy-paste reason above. Recorded because the id looks like the obvious key.

The same id-first shape is still open on #11951, which proposes a `messageMatchesActiveWorkflow` resolver consulting the frame's workflow id ahead of session ownership, and it is the inverse of the order decided here. Two tabs holding one copied graph id would match each other's jobs under it, which is exactly what the instance and path legs exist to prevent. Its resolver is therefore superseded by this ADR; remaining work on that branch should be carried onto this gate rather than restoring the competing policy during integration. #20982 overlaps the same seam and is covered by the same ruling. Whichever of #22186's move of the API and execution contracts into `packages/api-client` lands second has to carry this PR's request field, preview decoding and stamped types into that package.

**A `contextvars` owner on the backend.** Rejected on a mechanism error: `prompt_worker` is a daemon thread and `send_json` runs on the event loop, so a ContextVar set in the worker is unset at send time. The attribute merged in the body of `send_sync` is what shipped.

## Consequences

**The `activeJobId` fallback is load-bearing and is the weakest point.** Single-tab behaviour after a reload is preserved, and it is narrower than a blanket permissive rule, since a foreign job's unresolvable frame is dropped while another job is active. What remains is that with two tabs open and a frame unresolvable, it can land on the wrong one.

**`status` and current core `progress_text` both stay unattributable.** Neither supplies a `prompt_id` to the gate. `status` carries none because one server-wide socket serves every workflow. `progress_text` is a packed binary frame that `send_sync` cannot stamp, and the frontend decodes a job id out of it only when the server sets `supports_progress_text_metadata`, which no core version does — tracked as Comfy-Org/ComfyUI#16887. With the flag absent the gate is called with no job id at all, so it returns before any remembered-stamp lookup, and #16887 is what unblocks the leg rather than merely improving its first frame. A backend that does supply `prompt_id` gets the remembered workflow stamp without having to add `workflow_id` to every text frame.

**The two backends merge in opposite orders.** Core merges `{**workflow_metadata, **data}`, so a frame's own fields win. The Cloud gateway assigns `prompt_id` and `workflow_id` after the fact, so the backend wins, and it does not read `workflow_metadata` at all. Neither order should be relied on; the routing ids that arrive are authoritative whichever side set them.

**Revisit if** an unattributed-frame surface is built, at which point D2's boundary becomes implementable and this exception should be removed rather than inherited.
4 changes: 4 additions & 0 deletions docs/architecture/domains/boundaries.json
Original file line number Diff line number Diff line change
Expand Up @@ -841,7 +841,9 @@
"src/stores/executionErrorStore.ts",
"src/stores/executionErrorStore.test.ts",
"src/stores/executionStore.ts",
"src/stores/executionStore.terminalRecovery.test.ts",
"src/stores/executionStore.test.ts",
"src/stores/executionStore.workflowOwnership.test.ts",
"src/stores/jobPreviewStore.ts",
"src/stores/jobPreviewStore.test.ts",
"src/stores/nodeOutputStore.ts",
Expand Down Expand Up @@ -881,6 +883,8 @@
"src/scripts/api.getSettings.test.ts",
"src/scripts/api.getShareableAssets.test.ts",
"src/scripts/api.logs.test.ts",
"src/scripts/api.previewMetadata.test.ts",
"src/scripts/api.queuePrompt.test.ts",
"src/scripts/api.responses.test.ts",
"src/scripts/api.sessionOverride.test.ts",
"src/scripts/api.socketReset.test.ts",
Expand Down
1 change: 1 addition & 0 deletions packages/api-client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
},
"dependencies": {
"@comfyorg/account-core": "workspace:*",
"@comfyorg/comfyui-api-types": "workspace:*",
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
"@comfyorg/graph-model": "workspace:*",
"@comfyorg/ingest-types": "workspace:*",
"@comfyorg/object-info-parser": "workspace:*",
Expand Down
11 changes: 10 additions & 1 deletion packages/api-client/src/ComfyApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
zGetExtensionsResponse,
zPostAssetsFromWorkflowResponse
} from '@comfyorg/ingest-types/zod'
import type { PromptWorkflowMetadata } from '@comfyorg/comfyui-api-types'
import axios from 'axios'
import { get } from 'es-toolkit/compat'
import { trimEnd } from 'es-toolkit'
Expand Down Expand Up @@ -72,6 +73,7 @@ import type { JobAssetsResult, JobDetail, JobListItem } from './jobTypes'
import type { ComfyNodeDef } from '@comfyorg/object-info-parser'
import type { AuthCredential, AuthScheme } from './authTypes'
import type { NodeExecutionId } from '@comfyorg/graph-model/nodeIdentification'
import { zeroUuid } from '@comfyorg/graph-model/uuid'
import { getFetchRouteTemplate } from './fetchRouteTemplate'
import { retryingSend } from './fetchWithRetry'
import { NO_RETRY_POLICY } from './retryPolicy'
Expand All @@ -89,6 +91,7 @@ interface QueuePromptRequestBody {
client_id: string
prompt: ComfyApiWorkflow
partial_execution_targets?: NodeExecutionId[]
workflow_metadata?: Pick<PromptWorkflowMetadata, 'workflow_id'>
extra_data: {
extra_pnginfo: {
workflow: ComfyWorkflowJSON
Expand Down Expand Up @@ -213,6 +216,7 @@ interface BackendApiCalls {
displayNodeId: string
realNodeId: string
jobId: string
workflowId?: string
}
progress_text: ProgressTextWsMessage
progress_state: ProgressStateWsMessage
Expand Down Expand Up @@ -1105,7 +1109,8 @@ export class ComfyApi extends EventTarget {
displayNodeId: metadata.display_node_id,
parentNodeId: metadata.parent_node_id,
realNodeId: metadata.real_node_id,
jobId: metadata.prompt_id
jobId: metadata.prompt_id,
workflowId: metadata.workflow_id
})

this.dispatchCustomEvent('b_preview', imageBlob4)
Expand Down Expand Up @@ -1344,6 +1349,10 @@ export class ComfyApi extends EventTarget {
...(options?.partialExecutionTargets && {
partial_execution_targets: options.partialExecutionTargets
}),
...(workflow.id &&
workflow.id !== zeroUuid && {
workflow_metadata: { workflow_id: workflow.id }
}),
extra_data: {
auth_token_comfy_org: this.authToken,
api_key_comfy_org: this.apiKey,
Expand Down
25 changes: 21 additions & 4 deletions packages/api-client/src/executionTypes.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { PromptWorkflowMetadata } from '@comfyorg/comfyui-api-types'
import type { PromptInfo } from '@comfyorg/ingest-types'
import { z } from 'zod'
import { zNodeId } from '@comfyorg/graph-model/workflowSchema'
Expand All @@ -6,6 +7,22 @@ import { resultItemType } from './resultItemTypeSchema'

export type JobId = string

/**
* Core's OpenAPI defines no websocket frame schemas. It documents that
* `workflow_metadata` is echoed onto every JSON frame carrying a `prompt_id`,
* so this field is the request contract read back rather than a separate one.
*
* What transfers from the spec is the map's VALUE type, not the map: core types
* `workflow_metadata` as an arbitrary string map, and the frame carries that
* map's contents merged into the frame object. So a `Pick` or `Partial` of the
* map would describe the wrong shape here even though it would compile.
*
* The generated package is bundled into the published declarations by
* `tsconfig.types.json`, so this reference is inlined rather than emitted as an
* import of a private package that an external consumer could not resolve.
*/
type StampedWorkflowId = { workflow_id?: PromptWorkflowMetadata[string] }

export const zResultItem = z.object({
filename: z.string().optional(),
subfolder: z.string().optional(),
Expand Down Expand Up @@ -39,7 +56,7 @@ export interface StatusWsMessage {
status?: StatusWsMessageStatus | null
sid?: string | null
}
export interface ProgressWsMessage {
export interface ProgressWsMessage extends StampedWorkflowId {
value: number
max: number
prompt_id: JobId
Expand All @@ -55,11 +72,11 @@ export interface NodeProgressState {
parent_node_id?: NodeId
real_node_id?: NodeId
}
export interface ProgressStateWsMessage {
export interface ProgressStateWsMessage extends StampedWorkflowId {
prompt_id: JobId
nodes: Record<NodeId, NodeProgressState>
}
export interface ExecutingWsMessage {
export interface ExecutingWsMessage extends StampedWorkflowId {
node: NodeId | null
display_node?: NodeId
prompt_id: JobId
Expand All @@ -70,7 +87,7 @@ export interface ExecutedWsMessage extends ExecutingWsMessage {
output: NodeExecutionOutput
merge?: boolean
}
interface ExecutionWsMessageBase {
interface ExecutionWsMessageBase extends StampedWorkflowId {
prompt_id: JobId
timestamp: number
}
Expand Down
Loading
Loading