From c623990a08563487c0942fdfebd8fe31306e447e Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 3 Sep 2026 03:25:12 +0000 Subject: [PATCH 1/2] fix: recover a stale active job on every queue poll, not just reconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit executionStore.clearActiveJobIfStale() was only wired into useReconnectQueueRefresh (the WS-reconnect path). Routine queueStore.update() polling never called it, so a dropped terminal WebSocket message (execution_success/error/interrupted) left activeJobId, nodeProgressStates, and previews pinned to a job that has already left the Running/Pending queue snapshot, until the next reconnect. Call clearActiveJobIfStale() from queueStore.update() itself whenever the queue fetch succeeds, gated on the fresh Running/Pending snapshot alone — independent of whether the job has shown up in the concurrently-fetched history yet. getQueue and getHistory are fetched with Promise.allSettled and can disagree in the narrow window right after a job finishes; gating on history reintroduces the exact race this fixes. Follow-up to https://github.com/Comfy-Org/ComfyUI_frontend/pull/11866#discussion_r3178588879 (jaeone94, review thread on src/stores/queueStore.ts). That PR was closed (superseded by #11951 for its executionStore.ts gating half); this is the queueStore.ts race jaeone94 flagged and glary-bot agreed to address in a follow-up, which never happened. --- .../useReconnectQueueRefresh.test.ts | 7 ++- src/stores/queueStore.test.ts | 44 +++++++++++++++++++ src/stores/queueStore.ts | 8 ++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/composables/useReconnectQueueRefresh.test.ts b/src/composables/useReconnectQueueRefresh.test.ts index 2d431f50378..9fd5ad2c2c0 100644 --- a/src/composables/useReconnectQueueRefresh.test.ts +++ b/src/composables/useReconnectQueueRefresh.test.ts @@ -45,7 +45,12 @@ describe('useReconnectQueueRefresh', () => { const refresh = useReconnectQueueRefresh() await refresh() - expect(clearSpy).toHaveBeenCalledTimes(1) + // Called twice: once inside queueStore.update() itself (routine + // reconciliation, added so a dropped terminal WS message recovers on + // every poll, not just reconnect) and once here for the reconnect path's + // own explicit reconciliation. Both compute the same set and the call is + // idempotent, so the duplication is harmless. + expect(clearSpy).toHaveBeenCalledTimes(2) expect(clearSpy).toHaveBeenCalledWith( new Set(['run-1', 'pend-1', 'pend-2']) ) diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 72cde5d0628..3e14871d28e 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -1168,4 +1168,48 @@ describe('useQueueStore', () => { expect(store.isLoading).toBe(false) }) }) + + describe('update() - stale active job recovery', () => { + it('reconciles the active job against the fresh queue snapshot, independent of history', async () => { + // Regression for the race in PR #11866 review thread + // (discussion_r3178588879): getQueue and getHistory are fetched + // concurrently, so a job that just finished can be absent from + // Running/Pending while not yet present in history. If a terminal WS + // message is dropped, executionStore.activeJobId must still be + // recovered from the queue snapshot alone — it cannot wait on history. + mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([]) // job not yet in history either + const executionStore = useExecutionStore() + const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') + + await store.update() + + expect(clearSpy).toHaveBeenCalledWith(new Set()) + }) + + it('does not clear the active job when it is still in the queue snapshot', async () => { + mockGetQueue.mockResolvedValue({ + Running: [createRunningJob(0, 'run-1')], + Pending: [] + }) + mockGetHistory.mockResolvedValue([]) + const executionStore = useExecutionStore() + const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') + + await store.update() + + expect(clearSpy).toHaveBeenCalledWith(new Set(['run-1'])) + }) + + it('skips stale-job reconciliation when the queue fetch fails', async () => { + mockGetQueue.mockRejectedValue(new Error('queue down')) + mockGetHistory.mockResolvedValue([]) + const executionStore = useExecutionStore() + const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') + + await store.update() + + expect(clearSpy).not.toHaveBeenCalled() + }) + }) }) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 68633054eaf..224cbb773a7 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -566,6 +566,14 @@ export const useQueueStore = defineStore('queue', () => { ...queue.Pending.map((j) => j.id) ]) executionStore.reconcileInitializingJobs(activeJobIds) + // Recover from a dropped terminal WS message: if the active job no + // longer appears in the fresh Running/Pending snapshot, clear it here + // rather than waiting for a WS reconnect. Independent of whether the + // job has shown up in `historyResult` yet — `getQueue` and + // `getHistory` are fetched concurrently and can disagree in the + // narrow window right after a job finishes, so gating this on + // history would reintroduce the same race. + executionStore.clearActiveJobIfStale(activeJobIds) } else { console.error('Failed to fetch queue:', queueResult.reason) } From c9cd3c4fb16aa522e80b4f36f54c5f240ab04e11 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 2 Sep 2026 20:36:27 -0700 Subject: [PATCH 2/2] fix: skip stale-job clearing when execution_start arrived mid-fetch --- .../useReconnectQueueRefresh.test.ts | 6 +-- src/stores/executionStore.test.ts | 20 ++++++++++ src/stores/executionStore.ts | 22 ++++++++-- src/stores/queueStore.test.ts | 40 ++++++++++++++----- src/stores/queueStore.ts | 12 ++---- 5 files changed, 73 insertions(+), 27 deletions(-) diff --git a/src/composables/useReconnectQueueRefresh.test.ts b/src/composables/useReconnectQueueRefresh.test.ts index 9fd5ad2c2c0..e4a3469a031 100644 --- a/src/composables/useReconnectQueueRefresh.test.ts +++ b/src/composables/useReconnectQueueRefresh.test.ts @@ -45,11 +45,7 @@ describe('useReconnectQueueRefresh', () => { const refresh = useReconnectQueueRefresh() await refresh() - // Called twice: once inside queueStore.update() itself (routine - // reconciliation, added so a dropped terminal WS message recovers on - // every poll, not just reconnect) and once here for the reconnect path's - // own explicit reconciliation. Both compute the same set and the call is - // idempotent, so the duplication is harmless. + // Once inside queueStore.update(), once from the reconnect path itself. expect(clearSpy).toHaveBeenCalledTimes(2) expect(clearSpy).toHaveBeenCalledWith( new Set(['run-1', 'pend-1', 'pend-2']) diff --git a/src/stores/executionStore.test.ts b/src/stores/executionStore.test.ts index c62c0264a0b..d0cbca83452 100644 --- a/src/stores/executionStore.test.ts +++ b/src/stores/executionStore.test.ts @@ -1036,6 +1036,26 @@ describe('useExecutionStore - clearActiveJobIfStale', () => { expect(store.activeJobId).toBeNull() expect(store.queuedJobs['other']).toBeDefined() }) + + it('preserves a job that started at or after the snapshot was requested', () => { + store.activeJobId = 'job-1' + store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 1000 } } + + store.clearActiveJobIfStale(new Set(), 1000) + + expect(store.activeJobId).toBe('job-1') + expect(store.queuedJobs['job-1']).toBeDefined() + }) + + it('clears a job that started before the snapshot was requested', () => { + store.activeJobId = 'job-1' + store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 999 } } + + store.clearActiveJobIfStale(new Set(), 1000) + + expect(store.activeJobId).toBeNull() + expect(store.queuedJobs['job-1']).toBeUndefined() + }) }) describe('useExecutionStore - progress_text startup guard', () => { diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index d0931e11d35..8b802480f6c 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -781,12 +781,26 @@ export const useExecutionStore = defineStore('execution', () => { /** * Clears the active job if the server's queue snapshot doesn't list it. - * Used after WS reconnect to recover from stale state when a job finished - * during the disconnect window. + * Recovers from a dropped terminal WS message, after a reconnect or on a + * routine queue refresh. Pass `snapshotRequestedAt` (performance.now() + * taken before the queue fetch) so a job whose `execution_start` arrived + * while the fetch was in flight is not mistaken for stale: the snapshot + * predates the job, so its absence proves nothing. */ - function clearActiveJobIfStale(activeJobIds: Set) { + function clearActiveJobIfStale( + activeJobIds: Set, + snapshotRequestedAt?: number + ) { const id = activeJobId.value - if (id && !activeJobIds.has(id)) resetExecutionState(id) + if (!id || activeJobIds.has(id)) return + const startedAt = queuedJobs.value[id]?.executionStartedAt + if ( + snapshotRequestedAt !== undefined && + startedAt !== undefined && + startedAt >= snapshotRequestedAt + ) + return + resetExecutionState(id) } function isJobInitializing(jobId: JobId | number | undefined): boolean { diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 3e14871d28e..da45ac6de94 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -1170,21 +1170,27 @@ describe('useQueueStore', () => { }) describe('update() - stale active job recovery', () => { - it('reconciles the active job against the fresh queue snapshot, independent of history', async () => { + function setActiveJob(id: string, executionStartedAt: number) { + const executionStore = useExecutionStore() + executionStore.activeJobId = id + executionStore.queuedJobs = { [id]: { nodes: {}, executionStartedAt } } + return executionStore + } + + it('clears an active job missing from the fresh queue snapshot, independent of history', async () => { // Regression for the race in PR #11866 review thread // (discussion_r3178588879): getQueue and getHistory are fetched // concurrently, so a job that just finished can be absent from // Running/Pending while not yet present in history. If a terminal WS // message is dropped, executionStore.activeJobId must still be - // recovered from the queue snapshot alone — it cannot wait on history. + // recovered from the queue snapshot alone. It cannot wait on history. mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) mockGetHistory.mockResolvedValue([]) // job not yet in history either - const executionStore = useExecutionStore() - const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') + const executionStore = setActiveJob('job-1', performance.now() - 1000) await store.update() - expect(clearSpy).toHaveBeenCalledWith(new Set()) + expect(executionStore.activeJobId).toBeNull() }) it('does not clear the active job when it is still in the queue snapshot', async () => { @@ -1193,23 +1199,37 @@ describe('useQueueStore', () => { Pending: [] }) mockGetHistory.mockResolvedValue([]) + const executionStore = setActiveJob('run-1', performance.now() - 1000) + + await store.update() + + expect(executionStore.activeJobId).toBe('run-1') + }) + + it('keeps a job whose execution_start arrived while the fetch was in flight', async () => { + // Auto-queue fires the next POST /prompt from the same status event that + // triggers update(), so the GET /queue snapshot can predate the new job + // while its execution_start WS message lands before the response does. + mockGetQueue.mockImplementation(async () => { + setActiveJob('late-1', performance.now()) + return { Running: [], Pending: [] } + }) + mockGetHistory.mockResolvedValue([]) const executionStore = useExecutionStore() - const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') await store.update() - expect(clearSpy).toHaveBeenCalledWith(new Set(['run-1'])) + expect(executionStore.activeJobId).toBe('late-1') }) it('skips stale-job reconciliation when the queue fetch fails', async () => { mockGetQueue.mockRejectedValue(new Error('queue down')) mockGetHistory.mockResolvedValue([]) - const executionStore = useExecutionStore() - const clearSpy = vi.spyOn(executionStore, 'clearActiveJobIfStale') + const executionStore = setActiveJob('job-1', performance.now() - 1000) await store.update() - expect(clearSpy).not.toHaveBeenCalled() + expect(executionStore.activeJobId).toBe('job-1') }) }) }) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 224cbb773a7..00c9b3aa838 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -540,6 +540,7 @@ export const useQueueStore = defineStore('queue', () => { dirty = false isLoading.value = true try { + const snapshotRequestedAt = performance.now() const [queueResult, historyResult] = await Promise.allSettled([ api.getQueue({ throwOnError: true }), api.getHistory(maxHistoryItems.value) @@ -566,14 +567,9 @@ export const useQueueStore = defineStore('queue', () => { ...queue.Pending.map((j) => j.id) ]) executionStore.reconcileInitializingJobs(activeJobIds) - // Recover from a dropped terminal WS message: if the active job no - // longer appears in the fresh Running/Pending snapshot, clear it here - // rather than waiting for a WS reconnect. Independent of whether the - // job has shown up in `historyResult` yet — `getQueue` and - // `getHistory` are fetched concurrently and can disagree in the - // narrow window right after a job finishes, so gating this on - // history would reintroduce the same race. - executionStore.clearActiveJobIfStale(activeJobIds) + // Not gated on `historyResult`: getQueue and getHistory run + // concurrently and can disagree right after a job finishes. + executionStore.clearActiveJobIfStale(activeJobIds, snapshotRequestedAt) } else { console.error('Failed to fetch queue:', queueResult.reason) }