From 504a8aaec5b490069c986ce5ee2dc25e11eff594 Mon Sep 17 00:00:00 2001 From: Dante Date: Thu, 27 Aug 2026 01:27:06 +0000 Subject: [PATCH] fix(auth): retry unified refresh and rotate session cookie on reactive re-mint (#16063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes intermittent "Image failed to load / Image does not exist" on Cloud (FE-1595, also reported by an enterprise pilot): under `unified_cloud_auth`, one transient token-refresh failure permanently stops session-cookie rotation, so ``-based previews and asset thumbnails break while jobs keep running. ## Why this is needed / Root cause Node previews and Media Assets thumbnails load through plain `` tags. An `` request cannot carry an `Authorization` header, so those loads are authenticated solely by the 2-hour Firebase session cookie (`services/ingest/server/implementation/auth.go` in cloud). Before unified auth, the cookie was re-minted on every Firebase `onIdTokenChanged` (~hourly, with the Firebase SDK's built-in retries). Under `unified_cloud_auth` (100% since Jul 31) that driver is intentionally gated off in `authStore.ts`, and rotation is driven only by the proactive unified refresh: ``` mint (JWT TTL 1.5h) ──setTimeout(expiry−5m)──▶ refreshUnified() ──success──▶ notifyTokenRefreshed() ──▶ POST /auth/session (new 2h cookie) │ └─transient failure──▶ console.warn only • no retry scheduled → proactive chain dead • reactive 401 re-mint (remintUnifiedOnce) restores API traffic but deliberately did NOT rotate the cookie → cookie expires at 2h → every load 401s until the NEXT scheduled refresh (up to ~85 min) or a page reload ``` This matches the reported shape exactly: job completes and bills normally (Bearer-authenticated API/WS), only image loads fail; only sessions open longer than 2h are affected (reload mints a fresh cookie); it self-heals when a later refresh lands — hence "worked fine last week" and Rob's "resolved itself after a few minutes". ## How we change it Two seams in `workspaceAuthStore.ts`, both flag-gated inside the existing unified lifecycle (legacy path untouched): - **What**: `refreshUnified()` now re-arms itself after a transient failure via `scheduleUnifiedRefreshRetry()` — bounded exponential backoff (5s/10s/20s, reusing `MAX_SCHEDULED_REFRESH_RETRIES`), reusing `unifiedRefreshTimerId` so `clearUnifiedContext()` still cancels pending retries. Permanent-failure teardown behavior is unchanged. - **What**: `remintUnifiedOnce()` (the reactive 401 re-mint) now calls `notifyTokenRefreshed()` on success, so cookie rotation rides the same recovery path as API traffic instead of waiting for the next scheduled refresh. `useSessionCookie` already serializes/dedupes concurrent creates. - Retry counter resets on every successful mint (`scheduleUnifiedRefresh`) and on context teardown (`clearUnifiedContext`). ## AS IS / TO BE - AS IS: after one missed proactive refresh, API traffic self-heals via reactive re-mint but the session cookie never rotates again until the next scheduled refresh or a reload — image loads 401 for up to ~85 minutes. - TO BE: transient refresh failures retry within seconds; even if retries exhaust, the first reactive re-mint (any 401'd API call) rotates the cookie, so image loads recover together with API traffic. No UI change, so no app screenshots; the failure is a timing window (2h+ session + one failed refresh) reproducible only by forcing token-endpoint failures. The regression is covered by unit tests instead. ## Regression tests - `bumps the rotation trigger on a successful reactive re-mint` - `retries a transiently failed proactive refresh with backoff and rotates on success` (also asserts the normal schedule re-arms) - `bounds refresh retries and keeps the slot for reactive recovery` (no toast, slot preserved, no unbounded retry loop) - Existing suites unchanged and green: `useWorkspaceAuth.test.ts` (93), `remintRetry.test.ts` + `authStore.test.ts` + `useSessionCookie.test.ts` (140) ## Review Focus - `remintUnifiedOnce` can be invoked by concurrent 401 replays; each success bumps the rotation trigger, relying on `useSessionCookie`'s in-flight dedupe — confirm that's acceptable vs. gating the bump. - Retry backoff base (5s) is a judgment call for post-wake/network-blip recovery; the cap (3) mirrors the legacy scheduled-retry bound. Fixes FE-1595 (linear.app/comfyorg/issue/FE-1595) --- .../workspace/stores/useWorkspaceAuth.test.ts | 193 ++++++++++++++++++ .../workspace/stores/workspaceAuthStore.ts | 50 ++++- 2 files changed, 239 insertions(+), 4 deletions(-) diff --git a/src/platform/workspace/stores/useWorkspaceAuth.test.ts b/src/platform/workspace/stores/useWorkspaceAuth.test.ts index 4d03f20df99..330df5c3e82 100644 --- a/src/platform/workspace/stores/useWorkspaceAuth.test.ts +++ b/src/platform/workspace/stores/useWorkspaceAuth.test.ts @@ -2258,6 +2258,26 @@ describe('useWorkspaceAuthStore', () => { expect(mockNotifyTokenRefreshed).toHaveBeenCalledTimes(1) }) + it('bumps the rotation trigger on a successful reactive re-mint', async () => { + mockUnifiedCloudAuthEnabled.value = true + mockGetIdToken.mockResolvedValue('firebase-token-xyz') + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + ok: true, + json: () => Promise.resolve(personalTokenResponse) + }) + ) + + const store = useWorkspaceAuthStore() + await store.mintAtLogin() + expect(mockNotifyTokenRefreshed).not.toHaveBeenCalled() + + await store.remintUnifiedOnce('unified-token-1') + + expect(mockNotifyTokenRefreshed).toHaveBeenCalledTimes(1) + }) + it('remintUnifiedOnce re-mints once and, on a permanent failure, surfaces it and tears down without looping', async () => { mockUnifiedCloudAuthEnabled.value = true mockGetIdToken.mockResolvedValue('firebase-token-xyz') @@ -2703,6 +2723,179 @@ describe('useWorkspaceAuthStore', () => { expect(unifiedToken.value).toBe('unified-token-1') }) + it('retries a transiently failed proactive refresh with backoff and rotates on success', async () => { + mockUnifiedCloudAuthEnabled.value = true + mockGetIdToken.mockResolvedValue('firebase-token-xyz') + const expiresInMs = 3600 * 1000 + const okResponse = () => ({ + ok: true, + json: () => + Promise.resolve({ + ...personalTokenResponse, + expires_at: new Date(Date.now() + expiresInMs).toISOString() + }) + }) + const mockFetch = vi + .fn() + .mockResolvedValueOnce(okResponse()) + .mockResolvedValueOnce({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + text: () => Promise.resolve(JSON.stringify({ message: 'try again' })) + }) + .mockImplementation(() => Promise.resolve(okResponse())) + vi.stubGlobal('fetch', mockFetch) + + const store = useWorkspaceAuthStore() + const { unifiedToken } = storeToRefs(store) + + await store.mintAtLogin() + + const refreshDelay = expiresInMs - 5 * 60 * 1000 + await vi.advanceTimersByTimeAsync(refreshDelay) + expect(mockFetch).toHaveBeenCalledTimes(2) + expect(mockNotifyTokenRefreshed).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(5000) + + expect(mockFetch).toHaveBeenCalledTimes(3) + expect(mockNotifyTokenRefreshed).toHaveBeenCalledTimes(1) + expect(unifiedToken.value).toBe('unified-token-1') + + // The successful retry re-arms the normal proactive schedule. + await vi.advanceTimersByTimeAsync(refreshDelay) + expect(mockFetch).toHaveBeenCalledTimes(4) + }) + + it('bounds refresh retries and keeps the slot for reactive recovery', async () => { + mockUnifiedCloudAuthEnabled.value = true + mockGetIdToken.mockResolvedValue('firebase-token-xyz') + const expiresInMs = 3600 * 1000 + const mockFetch = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + ...personalTokenResponse, + expires_at: new Date(Date.now() + expiresInMs).toISOString() + }) + }) + .mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + text: () => Promise.resolve(JSON.stringify({ message: 'try again' })) + }) + vi.stubGlobal('fetch', mockFetch) + + const store = useWorkspaceAuthStore() + const { unifiedToken } = storeToRefs(store) + + await store.mintAtLogin() + + await vi.advanceTimersByTimeAsync(expiresInMs - 5 * 60 * 1000) + await vi.advanceTimersByTimeAsync(5000) + await vi.advanceTimersByTimeAsync(10000) + await vi.advanceTimersByTimeAsync(20000) + expect(mockFetch).toHaveBeenCalledTimes(5) + + await vi.advanceTimersByTimeAsync(60 * 60 * 1000) + + expect(mockFetch).toHaveBeenCalledTimes(5) + expect(mockToastAdd).not.toHaveBeenCalled() + expect(unifiedToken.value).toBe('unified-token-1') + }) + + it('re-arms the retry when a swallowed mint failure resolves false (owner null mid-refresh)', async () => { + mockUnifiedCloudAuthEnabled.value = true + mockGetIdToken.mockResolvedValue('firebase-token-xyz') + const expiresInMs = 3600 * 1000 + const mockFetch = vi.fn().mockImplementation(() => + Promise.resolve({ + ok: true, + json: () => + Promise.resolve({ + ...personalTokenResponse, + expires_at: new Date(Date.now() + expiresInMs).toISOString() + }) + }) + ) + vi.stubGlobal('fetch', mockFetch) + + const store = useWorkspaceAuthStore() + const { unifiedToken } = storeToRefs(store) + + await store.mintAtLogin() + expect(mockFetch).toHaveBeenCalledTimes(1) + + // Firebase is re-initializing at refresh time: requestToken throws + // NOT_AUTHENTICATED, which performUnifiedMint swallows to `false`. + mockCurrentUser.value = null + await vi.advanceTimersByTimeAsync(expiresInMs - 5 * 60 * 1000) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(mockToastAdd).not.toHaveBeenCalled() + + mockCurrentUser.value = { uid: 'user-a' } + await vi.advanceTimersByTimeAsync(5000) + + expect(mockFetch).toHaveBeenCalledTimes(2) + expect(mockNotifyTokenRefreshed).toHaveBeenCalledTimes(1) + expect(unifiedToken.value).toBe('unified-token-1') + }) + + it('does not stomp a superseding workspace switch schedule with a stale-refresh retry', async () => { + mockUnifiedCloudAuthEnabled.value = true + mockGetIdToken.mockResolvedValue('firebase-token-xyz') + const expiresInMs = 3600 * 1000 + const okResponse = (token: string) => ({ + ok: true, + json: () => + Promise.resolve({ + ...mockTokenResponse, + token, + expires_at: new Date(Date.now() + expiresInMs).toISOString() + }) + }) + let releaseStaleRefresh = () => {} + const staleRefreshGate = new Promise((resolve) => { + releaseStaleRefresh = resolve + }) + const mockFetch = vi + .fn() + .mockImplementationOnce(() => + Promise.resolve(okResponse('workspace-token-a')) + ) + .mockImplementationOnce(() => + staleRefreshGate.then(() => okResponse('stale-refresh-token')) + ) + .mockImplementation(() => + Promise.resolve(okResponse('workspace-token-b')) + ) + vi.stubGlobal('fetch', mockFetch) + + const store = useWorkspaceAuthStore() + + await store.switchWorkspace('workspace-a') + + // The proactive refresh fires and hangs; a workspace switch supersedes + // it and arms its own schedule before the stale mint resolves false. + await vi.advanceTimersByTimeAsync(expiresInMs - 5 * 60 * 1000) + const switchPromise = store.switchWorkspace('workspace-b') + releaseStaleRefresh() + await switchPromise + await vi.advanceTimersByTimeAsync(0) + + expect(store.getUnifiedToken()).toBe('workspace-token-b') + + // No 5s retry pending: the next mint is workspace-b's own scheduled + // refresh, not a stale-refresh backoff retry. + const fetchCallsAfterSwitch = mockFetch.mock.calls.length + await vi.advanceTimersByTimeAsync(5000) + expect(mockFetch).toHaveBeenCalledTimes(fetchCallsAfterSwitch) + }) + it('surfaces an error toast and resolves false when the login mint hits a permanent auth error', async () => { mockUnifiedCloudAuthEnabled.value = true mockGetIdToken.mockResolvedValue('firebase-token-xyz') diff --git a/src/platform/workspace/stores/workspaceAuthStore.ts b/src/platform/workspace/stores/workspaceAuthStore.ts index e4d749c9eba..3eaab9dd666 100644 --- a/src/platform/workspace/stores/workspaceAuthStore.ts +++ b/src/platform/workspace/stores/workspaceAuthStore.ts @@ -43,6 +43,8 @@ export type WorkspaceTokenResponse = z.infer< const MAX_SCHEDULED_REFRESH_RETRIES = 3 +const UNIFIED_REFRESH_RETRY_BASE_MS = 5000 + const RECOVERY_COOLDOWN_MS = 5000 // Retain expired unified-token contexts briefly so a concurrent reactive-401 @@ -133,6 +135,7 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { // The unified lifecycle keeps its own timer + request-id so it never shares // mutable state with the legacy switchWorkspace/refreshToken machinery. let unifiedRefreshTimerId: ReturnType | null = null + let unifiedRefreshRetryCount = 0 // Request ID to prevent stale refresh operations from overwriting newer workspace contexts let refreshRequestId = 0 @@ -734,15 +737,40 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { function scheduleUnifiedRefresh(expiresAt: number): void { stopUnifiedRefreshTimer() + unifiedRefreshRetryCount = 0 const now = Date.now() const refreshAt = expiresAt - TOKEN_REFRESH_BUFFER_MS const delay = Math.max(0, refreshAt - now) unifiedRefreshTimerId = setTimeout(() => { + unifiedRefreshTimerId = null void refreshUnified() }, delay) } + /** + * Re-arms the proactive unified refresh after a transient failure, with + * bounded exponential backoff. Without this the proactive chain dies on a + * single network blip: reactive 401 re-mints recover API traffic, but + * cookie-authenticated /media loads have no 401-retry path of their own + * and stay broken once the session cookie expires (FE-1595). Reuses + * `unifiedRefreshTimerId` so clearing the unified context cancels a pending + * retry. + */ + function scheduleUnifiedRefreshRetry(): boolean { + if (unifiedRefreshRetryCount >= MAX_SCHEDULED_REFRESH_RETRIES) { + return false + } + const delay = UNIFIED_REFRESH_RETRY_BASE_MS * 2 ** unifiedRefreshRetryCount + unifiedRefreshRetryCount += 1 + stopUnifiedRefreshTimer() + unifiedRefreshTimerId = setTimeout(() => { + unifiedRefreshTimerId = null + void refreshUnified() + }, delay) + return true + } + function pruneExpiredUnifiedTokenContexts(now: number): void { for (const [token, context] of issuedUnifiedTokenContexts) { if (context.expiresAt + ISSUED_TOKEN_CONTEXT_GRACE_MS < now) { @@ -754,6 +782,7 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { function clearUnifiedContext(): void { unifiedRefreshRequestId++ stopUnifiedRefreshTimer() + unifiedRefreshRetryCount = 0 unifiedToken.value = null unifiedTokenOwnerUid.value = null issuedUnifiedTokenContexts.clear() @@ -845,11 +874,17 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { try { const minted = await mintUnified(target) - // Only a refresh re-mint rotates the session cookie; the initial login - // mint and workspace switches do not. A stale (discarded) re-mint does not - // rotate either. + // Only re-mints rotate the session cookie; the initial login mint and + // workspace switches establish it themselves. A stale (discarded) + // re-mint does not rotate either. if (minted) { useAuthStore().notifyTokenRefreshed() + } else if (unifiedRefreshTimerId === null) { + // A mint failure while the owner uid is momentarily null (Firebase + // re-initializing post-wake) resolves false instead of throwing. Only + // a false with no armed timer is a dead chain: a superseding mint has + // already scheduled its own refresh. + scheduleUnifiedRefreshRetry() } } catch (err) { // Guard the toast on a live token so concurrent permanent failures across @@ -862,7 +897,13 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { : undefined ) } else { - console.warn('Unified token refresh failed:', err) + const retryScheduled = scheduleUnifiedRefreshRetry() + console.warn( + retryScheduled + ? 'Unified token refresh failed; retrying shortly:' + : 'Unified token refresh failed; retries exhausted, awaiting a reactive re-mint:', + err + ) } } } @@ -909,6 +950,7 @@ export const useWorkspaceAuthStore = defineStore('workspaceAuth', () => { try { const minted = await mintUnified(target) if (!minted) return null + useAuthStore().notifyTokenRefreshed() return getUnifiedToken() ?? null } catch (err) { // Mirror refreshUnified: a permanent failure tears down the session;