Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/composables/useReconnectQueueRefresh.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ describe('useReconnectQueueRefresh', () => {
const refresh = useReconnectQueueRefresh()
await refresh()

expect(clearSpy).toHaveBeenCalledTimes(1)
// Once inside queueStore.update(), once from the reconnect path itself.
expect(clearSpy).toHaveBeenCalledTimes(2)
expect(clearSpy).toHaveBeenCalledWith(
new Set(['run-1', 'pend-1', 'pend-2'])
)
Expand Down
20 changes: 20 additions & 0 deletions src/stores/executionStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,26 @@ describe('useExecutionStore - clearActiveJobIfStale', () => {
expect(store.activeJobId).toBeNull()
expect(store.queuedJobs['other']).toBeDefined()
})

it('preserves a job that started at or after the snapshot was requested', () => {
store.activeJobId = 'job-1'
store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 1000 } }

store.clearActiveJobIfStale(new Set(), 1000)

expect(store.activeJobId).toBe('job-1')
expect(store.queuedJobs['job-1']).toBeDefined()
})

it('clears a job that started before the snapshot was requested', () => {
store.activeJobId = 'job-1'
store.queuedJobs = { 'job-1': { nodes: {}, executionStartedAt: 999 } }

store.clearActiveJobIfStale(new Set(), 1000)

expect(store.activeJobId).toBeNull()
expect(store.queuedJobs['job-1']).toBeUndefined()
})
})

describe('useExecutionStore - progress_text startup guard', () => {
Expand Down
22 changes: 18 additions & 4 deletions src/stores/executionStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -781,12 +781,26 @@ export const useExecutionStore = defineStore('execution', () => {

/**
* Clears the active job if the server's queue snapshot doesn't list it.
* Used after WS reconnect to recover from stale state when a job finished
* during the disconnect window.
* Recovers from a dropped terminal WS message, after a reconnect or on a
* routine queue refresh. Pass `snapshotRequestedAt` (performance.now()
* taken before the queue fetch) so a job whose `execution_start` arrived
* while the fetch was in flight is not mistaken for stale: the snapshot
* predates the job, so its absence proves nothing.
*/
function clearActiveJobIfStale(activeJobIds: Set<JobId>) {
function clearActiveJobIfStale(
activeJobIds: Set<JobId>,
snapshotRequestedAt?: number
) {
const id = activeJobId.value
if (id && !activeJobIds.has(id)) resetExecutionState(id)
if (!id || activeJobIds.has(id)) return
const startedAt = queuedJobs.value[id]?.executionStartedAt
if (
snapshotRequestedAt !== undefined &&
startedAt !== undefined &&
startedAt >= snapshotRequestedAt
)
return
resetExecutionState(id)
}

function isJobInitializing(jobId: JobId | number | undefined): boolean {
Expand Down
64 changes: 64 additions & 0 deletions src/stores/queueStore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1168,4 +1168,68 @@ describe('useQueueStore', () => {
expect(store.isLoading).toBe(false)
})
})

describe('update() - stale active job recovery', () => {
function setActiveJob(id: string, executionStartedAt: number) {
const executionStore = useExecutionStore()
executionStore.activeJobId = id
executionStore.queuedJobs = { [id]: { nodes: {}, executionStartedAt } }
return executionStore
}

it('clears an active job missing from the fresh queue snapshot, independent of history', async () => {
// Regression for the race in PR #11866 review thread
// (discussion_r3178588879): getQueue and getHistory are fetched
// concurrently, so a job that just finished can be absent from
// Running/Pending while not yet present in history. If a terminal WS
// message is dropped, executionStore.activeJobId must still be
// recovered from the queue snapshot alone. It cannot wait on history.
mockGetQueue.mockResolvedValue({ Running: [], Pending: [] })
mockGetHistory.mockResolvedValue([]) // job not yet in history either
const executionStore = setActiveJob('job-1', performance.now() - 1000)

await store.update()

expect(executionStore.activeJobId).toBeNull()
})
Comment thread
christian-byrne marked this conversation as resolved.
Resolved

it('does not clear the active job when it is still in the queue snapshot', async () => {
mockGetQueue.mockResolvedValue({
Running: [createRunningJob(0, 'run-1')],
Pending: []
})
mockGetHistory.mockResolvedValue([])
const executionStore = setActiveJob('run-1', performance.now() - 1000)

await store.update()

expect(executionStore.activeJobId).toBe('run-1')
})

it('keeps a job whose execution_start arrived while the fetch was in flight', async () => {
// Auto-queue fires the next POST /prompt from the same status event that
// triggers update(), so the GET /queue snapshot can predate the new job
// while its execution_start WS message lands before the response does.
mockGetQueue.mockImplementation(async () => {
setActiveJob('late-1', performance.now())
return { Running: [], Pending: [] }
})
mockGetHistory.mockResolvedValue([])
const executionStore = useExecutionStore()

await store.update()

expect(executionStore.activeJobId).toBe('late-1')
})

it('skips stale-job reconciliation when the queue fetch fails', async () => {
mockGetQueue.mockRejectedValue(new Error('queue down'))
mockGetHistory.mockResolvedValue([])
const executionStore = setActiveJob('job-1', performance.now() - 1000)

await store.update()

expect(executionStore.activeJobId).toBe('job-1')
})
})
})
4 changes: 4 additions & 0 deletions src/stores/queueStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,7 @@ export const useQueueStore = defineStore('queue', () => {
dirty = false
isLoading.value = true
try {
const snapshotRequestedAt = performance.now()
const [queueResult, historyResult] = await Promise.allSettled([
api.getQueue({ throwOnError: true }),
api.getHistory(maxHistoryItems.value)
Expand All @@ -566,6 +567,9 @@ export const useQueueStore = defineStore('queue', () => {
...queue.Pending.map((j) => j.id)
])
executionStore.reconcileInitializingJobs(activeJobIds)
// Not gated on `historyResult`: getQueue and getHistory run
// concurrently and can disagree right after a job finishes.
executionStore.clearActiveJobIfStale(activeJobIds, snapshotRequestedAt)
} else {
console.error('Failed to fetch queue:', queueResult.reason)
}
Expand Down
Loading