diff --git a/src/platform/workspace/stores/teamWorkspaceStore.test.ts b/src/platform/workspace/stores/teamWorkspaceStore.test.ts index 54ca6835c47..fd160d6c4b2 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.test.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.test.ts @@ -8,7 +8,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { useCurrentUser } from '@/composables/auth/useCurrentUser' import { WORKSPACE_STORAGE_KEYS } from '@/platform/workspace/workspaceConstants' -import { sortWorkspaces, useTeamWorkspaceStore } from './teamWorkspaceStore' +import { + INIT_RETRY_COOLDOWN_MS, + sortWorkspaces, + useTeamWorkspaceStore +} from './teamWorkspaceStore' const mockDistributionTypes = vi.hoisted(() => ({ isCloud: true })) @@ -315,6 +319,7 @@ describe('useTeamWorkspaceStore', () => { expect(store.error).toBeInstanceOf(Error) // Should have been called 4 times (initial + 3 retries) expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(4) + expect(store.initFailedAtDiscovery()).toBe(true) }) it('does not reinitialize if already initialized', async () => { @@ -361,6 +366,27 @@ describe('useTeamWorkspaceStore', () => { expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(2) }) + it('rejects opted-in retries during the failure cooldown', async () => { + mockWorkspaceApi.list.mockResolvedValueOnce({ workspaces: [] }) + const store = useTeamWorkspaceStore() + + await expect(store.initialize()).rejects.toThrow( + 'No workspaces available' + ) + // A per-submission token request inside the cooldown rejects at once + // instead of re-running the full retry/backoff cycle. + await expect( + store.initialize({ honorFailureCooldown: true }) + ).rejects.toThrow('No workspaces available') + expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(1) + + await vi.advanceTimersByTimeAsync(INIT_RETRY_COOLDOWN_MS) + await store.initialize({ honorFailureCooldown: true }) + + expect(store.initState).toBe('ready') + expect(mockWorkspaceApi.list).toHaveBeenCalledTimes(2) + }) + it('can initialize the next user after identity state is reset', async () => { const store = useTeamWorkspaceStore() await store.initialize() @@ -440,6 +466,7 @@ describe('useTeamWorkspaceStore', () => { 'No workspaces available' ) expect(store.initState).toBe('error') + expect(store.initFailedAtDiscovery()).toBe(false) }) it('does not activate a workspace when token exchange fails', async () => { @@ -458,6 +485,7 @@ describe('useTeamWorkspaceStore', () => { expect(store.initState).toBe('error') expect(store.activeWorkspaceId).toBeNull() expect(store.error).toEqual(new Error('Token exchange failed')) + expect(store.initFailedAtDiscovery()).toBe(false) }) }) diff --git a/src/platform/workspace/stores/teamWorkspaceStore.ts b/src/platform/workspace/stores/teamWorkspaceStore.ts index fd8fc905f61..db86d21ee30 100644 --- a/src/platform/workspace/stores/teamWorkspaceStore.ts +++ b/src/platform/workspace/stores/teamWorkspaceStore.ts @@ -159,6 +159,10 @@ function clearLastWorkspaceId(): void { const MAX_OWNED_WORKSPACES = 10 const MAX_INIT_RETRIES = 3 const BASE_RETRY_DELAY_MS = 1000 +// After a failed init, callers that opt in reject immediate re-init attempts +// for this long so repeated token requests don't each re-run the full +// retry/backoff cycle. +export const INIT_RETRY_COOLDOWN_MS = 30_000 export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { const initState = ref('uninitialized') @@ -179,6 +183,10 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { let identityGeneration = 0 let initializationPromise: Promise | null = null let pendingWorkspaceSwitch: Promise | null = null + let lastInitFailureAt: number | null = null + // Whether the last failed init failed while listing workspaces, before any + // workspace was resolved or activated. + let lastInitFailedAtDiscovery = false function isStaleIdentity(generation: number): boolean { return generation !== identityGeneration @@ -321,6 +329,7 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { const isApiKeySession = useCurrentUser().isApiKeyLogin.value for (let attempt = 0; attempt <= MAX_INIT_RETRIES; attempt++) { + let discovering = false try { // An API-key credential is bound to exactly one workspace on the // server. There is no discovery, switching, or token exchange for it: @@ -346,7 +355,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { if (hasValidSession && workspaceAuthStore.currentWorkspace) { // Valid session exists - fetch workspace list and verify access + discovering = true const response = await workspaceApi.list() + discovering = false if (isStaleIdentity(generation)) return workspaces.value = sortWorkspaces( response.workspaces.map(createWorkspaceState) @@ -388,7 +399,9 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { } // 2. No valid session - fetch workspaces and pick default + discovering = true const response = await workspaceApi.list() + discovering = false if (isStaleIdentity(generation)) return workspaces.value = sortWorkspaces( response.workspaces.map(createWorkspaceState) @@ -447,6 +460,8 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { ) { error.value = e instanceof Error ? e : new Error('Unknown error') initState.value = 'error' + lastInitFailureAt = Date.now() + lastInitFailedAtDiscovery = discovering isFetchingWorkspaces.value = false throw e } @@ -466,12 +481,29 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { isFetchingWorkspaces.value = false } - function initialize(): Promise { + function initialize(options?: { + honorFailureCooldown?: boolean + }): Promise { if (initializationPromise) return initializationPromise if (initState.value !== 'uninitialized' && initState.value !== 'error') { return Promise.resolve() } + // Opt-in cooldown after a failed init, for per-submission token requests: + // re-running the full retry/backoff cycle on every submission would stall + // each one behind up to MAX_INIT_RETRIES doomed attempts. Reject until the + // cooldown expires; initState stays 'error', so the caller takes its + // fallback path. Other callers (the auth gate's retry button, agent + // consent) keep retrying immediately. + if ( + options?.honorFailureCooldown && + initState.value === 'error' && + lastInitFailureAt !== null && + Date.now() - lastInitFailureAt < INIT_RETRY_COOLDOWN_MS + ) { + return Promise.reject(error.value ?? new Error('Workspace init failed')) + } + const promise = performInitialization() initializationPromise = promise void promise.then( @@ -485,6 +517,14 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { return promise } + /** + * Whether the last failed initialize() failed while listing workspaces, + * as opposed to an empty list, the workspace switch or its token exchange. + */ + function initFailedAtDiscovery(): boolean { + return initState.value === 'error' && lastInitFailedAtDiscovery + } + /** * Re-fetch workspaces from API without changing active workspace. */ @@ -1023,6 +1063,8 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { mutableWorkspaceTransitionGeneration.value++ pendingWorkspaceSwitch = null initializationPromise = null + lastInitFailureAt = null + lastInitFailedAtDiscovery = false initState.value = 'uninitialized' workspaces.value = [] mutableActiveWorkspaceId.value = null @@ -1070,6 +1112,7 @@ export const useTeamWorkspaceStore = defineStore('teamWorkspace', () => { // Initialization & Cleanup initialize, + initFailedAtDiscovery, destroy, resetForIdentityChange, refreshWorkspaces, diff --git a/src/stores/__tests__/authTokenPriority.test.ts b/src/stores/__tests__/authTokenPriority.test.ts index c3b99ed4188..261833d30ee 100644 --- a/src/stores/__tests__/authTokenPriority.test.ts +++ b/src/stores/__tests__/authTokenPriority.test.ts @@ -327,15 +327,22 @@ describe('auth token priority chain', () => { expect(mockUser.getIdToken).not.toHaveBeenCalled() }) - it('fails queue authentication closed after workspace initialization errors', async () => { + it('falls back to the Firebase token when workspace initialization errors', async () => { + // On a remote non-Cloud origin the workspaces request is CORS-blocked + // (#17271); queue execution must still carry a credential so the + // backend can resolve the user's personal workspace instead of 401ing. mockDistributionTypes.isCloud = false useTeamWorkspaceStore().initState = 'uninitialized' vi.mocked(useTeamWorkspaceStore().initialize).mockRejectedValue( new Error('Network error') ) + vi.mocked(useTeamWorkspaceStore().initFailedAtDiscovery).mockReturnValue( + true + ) - await expect(store.getWorkspaceAuthToken()).resolves.toBeUndefined() - expect(mockUser.getIdToken).not.toHaveBeenCalled() + await expect(store.getWorkspaceAuthToken()).resolves.toBe( + 'firebase-token' + ) expect( vi.mocked(useWorkspaceAuthStore().ensureWorkspaceToken) ).not.toHaveBeenCalled() diff --git a/src/stores/__tests__/authTokenRemoteOriginFallback.test.ts b/src/stores/__tests__/authTokenRemoteOriginFallback.test.ts new file mode 100644 index 00000000000..8374eb89eed --- /dev/null +++ b/src/stores/__tests__/authTokenRemoteOriginFallback.test.ts @@ -0,0 +1,159 @@ +import { useApiKeyAuthStore } from '@/stores/apiKeyAuthStore' +import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore' +import { useWorkspaceAuthStore } from '@/platform/workspace/stores/workspaceAuthStore' +import { fromPartial } from '@total-typescript/shoehorn' +import type { User } from 'firebase/auth' +import * as firebaseAuth from 'firebase/auth' +import type { Mock } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { useAuthStore } from '@/stores/authStore' +import { stubFirebaseAuthHarness } from '@/utils/__tests__/stubAccountIdentityPort' + +const { mockDistributionTypes } = vi.hoisted(() => ({ + mockDistributionTypes: { + isCloud: true, + isDesktop: true + } +})) + +vi.mock(import('@/composables/useFeatureFlags')) + +vi.mock(import('firebase/auth')) + +vi.mock(import('@/platform/telemetry')) + +vi.mock(import('@/services/dialogService')) + +vi.mock(import('@/platform/distribution/types'), () => mockDistributionTypes) + +type MockUser = Omit & { + getIdToken: Mock +} + +/** + * Regression coverage for #17271: on a remote non-Cloud origin (e.g. a + * RunPod HTTPS proxy) the browser CORS-blocks GET cloud.comfy.org/api/workspaces, + * so teamWorkspaceStore.initialize() rejects and getWorkspaceAuthToken must + * still hand queue execution a usable credential — the Firebase ID token — + * instead of failing closed and 401ing the backend. + */ +describe('getWorkspaceAuthToken remote-origin workspace init failure', () => { + let store: ReturnType + let authStateObservers: Array<(user: User | null) => void> + + const mockUser = fromPartial({ + uid: 'test-user-id', + email: 'test@example.com', + getIdToken: vi.fn().mockResolvedValue('firebase-token') + }) + + beforeEach(() => { + mockDistributionTypes.isCloud = false + stubFirebaseAuthHarness() + authStateObservers = [] + vi.mocked(firebaseAuth.onAuthStateChanged).mockImplementation( + (_, callback) => { + const observer = callback as (user: User | null) => void + authStateObservers.push(observer) + observer(mockUser) + return vi.fn() + } + ) + store = useAuthStore() + useWorkspaceAuthStore().destroy() + Object.assign(useTeamWorkspaceStore(), { activeWorkspaceId: null }) + useTeamWorkspaceStore().initState = 'uninitialized' + vi.mocked(useTeamWorkspaceStore().initFailedAtDiscovery).mockReturnValue( + true + ) + vi.mocked(useWorkspaceAuthStore().ensureWorkspaceToken).mockResolvedValue( + null + ) + vi.mocked( + useTeamWorkspaceStore().resetForIdentityChange + ).mockImplementation(() => {}) + vi.mocked(useApiKeyAuthStore().getAuthHeader).mockReturnValue(null) + Object.assign(useApiKeyAuthStore(), { isAuthenticated: false }) + mockUser.getIdToken.mockClear() + mockUser.getIdToken.mockResolvedValue('firebase-token') + }) + + it('falls back to the Firebase token when workspace init fails with a network error', async () => { + vi.mocked(useTeamWorkspaceStore().initialize).mockRejectedValue( + new Error('Network Error') + ) + + await expect(store.getWorkspaceAuthToken()).resolves.toBe('firebase-token') + }) + + it('asks the store to honor its failure cooldown', async () => { + vi.mocked(useTeamWorkspaceStore().initialize).mockRejectedValue( + new Error('Network Error') + ) + + await store.getWorkspaceAuthToken() + + expect(useTeamWorkspaceStore().initialize).toHaveBeenCalledWith({ + honorFailureCooldown: true + }) + }) + + it('returns no credential when the account changes during workspace init', async () => { + const otherUser = fromPartial({ + uid: 'other-user-id', + email: 'other@example.com', + getIdToken: vi.fn().mockResolvedValue('other-firebase-token') + }) + vi.mocked(useTeamWorkspaceStore().initialize).mockImplementation( + async () => { + for (const observer of authStateObservers) observer(otherUser) + throw new Error('Network Error') + } + ) + + await expect(store.getWorkspaceAuthToken()).resolves.toBeUndefined() + expect(otherUser.getIdToken).not.toHaveBeenCalled() + }) + + it('sends no credential when init fails after workspace discovery', async () => { + vi.mocked(useTeamWorkspaceStore().initialize).mockRejectedValue( + new Error('Token exchange failed') + ) + vi.mocked(useTeamWorkspaceStore().initFailedAtDiscovery).mockReturnValue( + false + ) + + await expect(store.getWorkspaceAuthToken()).resolves.toBeUndefined() + expect(mockUser.getIdToken).not.toHaveBeenCalled() + }) + + it('returns no credential when the account changes during a successful init', async () => { + const otherUser = fromPartial({ + uid: 'other-user-id', + email: 'other@example.com', + getIdToken: vi.fn().mockResolvedValue('other-firebase-token') + }) + vi.mocked(useTeamWorkspaceStore().initialize).mockImplementation( + async () => { + for (const observer of authStateObservers) observer(otherUser) + Object.assign(useTeamWorkspaceStore(), { + activeWorkspaceId: 'other-workspace' + }) + } + ) + + await expect(store.getWorkspaceAuthToken()).resolves.toBeUndefined() + expect(useWorkspaceAuthStore().ensureWorkspaceToken).not.toHaveBeenCalled() + expect(otherUser.getIdToken).not.toHaveBeenCalled() + }) + + it('still falls back when a previous boot left the store in the error state', async () => { + useTeamWorkspaceStore().initState = 'error' + vi.mocked(useTeamWorkspaceStore().initialize).mockRejectedValue( + new Error('Network Error') + ) + + await expect(store.getWorkspaceAuthToken()).resolves.toBe('firebase-token') + }) +}) diff --git a/src/stores/authStore.ts b/src/stores/authStore.ts index 2320cf5144b..1a53a215766 100644 --- a/src/stores/authStore.ts +++ b/src/stores/authStore.ts @@ -451,10 +451,27 @@ export const useAuthStore = defineStore('auth', () => { teamWorkspaceStore.initState === 'loading' || teamWorkspaceStore.initState === 'error') ) { + const requestingIdentity = currentUserCredentialIdentity() + let initFailed = false try { - await teamWorkspaceStore.initialize() + await teamWorkspaceStore.initialize({ honorFailureCooldown: true }) } catch { + initFailed = true + } + // The account may have changed while initialization was in flight; + // never hand this request another account's credential. + if (currentUserCredentialIdentity() !== requestingIdentity) return undefined + if (initFailed) { + // A remote non-Cloud origin (e.g. a proxied self-hosted instance) + // cannot reach cloud.comfy.org/api/workspaces: the browser blocks the + // cross-origin request, so workspace discovery fails regardless of + // retries. The signed-in user still has a valid Firebase ID token and + // the backend resolves the personal workspace from it, so fall back + // to it rather than sending no credential and 401ing node execution. + // Any later failure (switch or token exchange) sends no credential. + if (!teamWorkspaceStore.initFailedAtDiscovery()) return undefined + return (await getIdToken()) ?? undefined } }