diff --git a/src/composables/useReconnectQueueRefresh.test.ts b/src/composables/useReconnectQueueRefresh.test.ts index 2d431f50378..e4a3469a031 100644 --- a/src/composables/useReconnectQueueRefresh.test.ts +++ b/src/composables/useReconnectQueueRefresh.test.ts @@ -45,7 +45,8 @@ describe('useReconnectQueueRefresh', () => { const refresh = useReconnectQueueRefresh() await refresh() - expect(clearSpy).toHaveBeenCalledTimes(1) + // Once inside queueStore.update(), once from the reconnect path itself. + expect(clearSpy).toHaveBeenCalledTimes(2) expect(clearSpy).toHaveBeenCalledWith( new Set(['run-1', 'pend-1', 'pend-2']) ) diff --git a/src/stores/executionStore.test.ts b/src/stores/executionStore.test.ts index c62c0264a0b..d0cbca83452 100644 --- a/src/stores/executionStore.test.ts +++ b/src/stores/executionStore.test.ts @@ -1036,6 +1036,26 @@ describe('useExecutionStore - clearActiveJobIfStale', () => { expect(store.activeJobId).toBeNull() expect(store.queuedJobs['other']).toBeDefined() }) + + it('preserves a job that started at or after the snapshot was requested', () => { + store.activeJobId = 'job-1' + store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 1000 } } + + store.clearActiveJobIfStale(new Set(), 1000) + + expect(store.activeJobId).toBe('job-1') + expect(store.queuedJobs['job-1']).toBeDefined() + }) + + it('clears a job that started before the snapshot was requested', () => { + store.activeJobId = 'job-1' + store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 999 } } + + store.clearActiveJobIfStale(new Set(), 1000) + + expect(store.activeJobId).toBeNull() + expect(store.queuedJobs['job-1']).toBeUndefined() + }) }) describe('useExecutionStore - progress_text startup guard', () => { diff --git a/src/stores/executionStore.ts b/src/stores/executionStore.ts index d0931e11d35..8b802480f6c 100644 --- a/src/stores/executionStore.ts +++ b/src/stores/executionStore.ts @@ -781,12 +781,26 @@ export const useExecutionStore = defineStore('execution', () => { /** * Clears the active job if the server's queue snapshot doesn't list it. - * Used after WS reconnect to recover from stale state when a job finished - * during the disconnect window. + * Recovers from a dropped terminal WS message, after a reconnect or on a + * routine queue refresh. Pass `snapshotRequestedAt` (performance.now() + * taken before the queue fetch) so a job whose `execution_start` arrived + * while the fetch was in flight is not mistaken for stale: the snapshot + * predates the job, so its absence proves nothing. */ - function clearActiveJobIfStale(activeJobIds: Set) { + function clearActiveJobIfStale( + activeJobIds: Set, + snapshotRequestedAt?: number + ) { const id = activeJobId.value - if (id && !activeJobIds.has(id)) resetExecutionState(id) + if (!id || activeJobIds.has(id)) return + const startedAt = queuedJobs.value[id]?.executionStartedAt + if ( + snapshotRequestedAt !== undefined && + startedAt !== undefined && + startedAt >= snapshotRequestedAt + ) + return + resetExecutionState(id) } function isJobInitializing(jobId: JobId | number | undefined): boolean { diff --git a/src/stores/queueStore.test.ts b/src/stores/queueStore.test.ts index 72cde5d0628..da45ac6de94 100644 --- a/src/stores/queueStore.test.ts +++ b/src/stores/queueStore.test.ts @@ -1168,4 +1168,68 @@ describe('useQueueStore', () => { expect(store.isLoading).toBe(false) }) }) + + describe('update() - stale active job recovery', () => { + function setActiveJob(id: string, executionStartedAt: number) { + const executionStore = useExecutionStore() + executionStore.activeJobId = id + executionStore.queuedJobs = { [id]: { nodes: {}, executionStartedAt } } + return executionStore + } + + it('clears an active job missing from the fresh queue snapshot, independent of history', async () => { + // Regression for the race in PR #11866 review thread + // (discussion_r3178588879): getQueue and getHistory are fetched + // concurrently, so a job that just finished can be absent from + // Running/Pending while not yet present in history. If a terminal WS + // message is dropped, executionStore.activeJobId must still be + // recovered from the queue snapshot alone. It cannot wait on history. + mockGetQueue.mockResolvedValue({ Running: [], Pending: [] }) + mockGetHistory.mockResolvedValue([]) // job not yet in history either + const executionStore = setActiveJob('job-1', performance.now() - 1000) + + await store.update() + + expect(executionStore.activeJobId).toBeNull() + }) + + it('does not clear the active job when it is still in the queue snapshot', async () => { + mockGetQueue.mockResolvedValue({ + Running: [createRunningJob(0, 'run-1')], + Pending: [] + }) + mockGetHistory.mockResolvedValue([]) + const executionStore = setActiveJob('run-1', performance.now() - 1000) + + await store.update() + + expect(executionStore.activeJobId).toBe('run-1') + }) + + it('keeps a job whose execution_start arrived while the fetch was in flight', async () => { + // Auto-queue fires the next POST /prompt from the same status event that + // triggers update(), so the GET /queue snapshot can predate the new job + // while its execution_start WS message lands before the response does. + mockGetQueue.mockImplementation(async () => { + setActiveJob('late-1', performance.now()) + return { Running: [], Pending: [] } + }) + mockGetHistory.mockResolvedValue([]) + const executionStore = useExecutionStore() + + await store.update() + + expect(executionStore.activeJobId).toBe('late-1') + }) + + it('skips stale-job reconciliation when the queue fetch fails', async () => { + mockGetQueue.mockRejectedValue(new Error('queue down')) + mockGetHistory.mockResolvedValue([]) + const executionStore = setActiveJob('job-1', performance.now() - 1000) + + await store.update() + + expect(executionStore.activeJobId).toBe('job-1') + }) + }) }) diff --git a/src/stores/queueStore.ts b/src/stores/queueStore.ts index 68633054eaf..00c9b3aa838 100644 --- a/src/stores/queueStore.ts +++ b/src/stores/queueStore.ts @@ -540,6 +540,7 @@ export const useQueueStore = defineStore('queue', () => { dirty = false isLoading.value = true try { + const snapshotRequestedAt = performance.now() const [queueResult, historyResult] = await Promise.allSettled([ api.getQueue({ throwOnError: true }), api.getHistory(maxHistoryItems.value) @@ -566,6 +567,9 @@ export const useQueueStore = defineStore('queue', () => { ...queue.Pending.map((j) => j.id) ]) executionStore.reconcileInitializingJobs(activeJobIds) + // Not gated on `historyResult`: getQueue and getHistory run + // concurrently and can disagree right after a job finishes. + executionStore.clearActiveJobIfStale(activeJobIds, snapshotRequestedAt) } else { console.error('Failed to fetch queue:', queueResult.reason) }