diff --git a/src/main/cloud/claims.test.ts b/src/main/cloud/claims.test.ts index bbc405b3d..a9aaab2e8 100644 --- a/src/main/cloud/claims.test.ts +++ b/src/main/cloud/claims.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest' -import { identityOf, statusFromAccessToken, subjectOf, workspaceIdOf } from './claims' +import { + canMintWorkspaceTokens, + holdsPartnerNodeScope, + identityOf, + statusFromAccessToken, + subjectOf, + workspaceIdOf +} from './claims' function jwt(payload: Record): string { const b64 = (o: unknown): string => Buffer.from(JSON.stringify(o)).toString('base64url') @@ -40,6 +47,25 @@ describe('claims', () => { expect(subjectOf('garbage')).toBeNull() }) + it.each<[string, unknown, boolean, boolean]>([ + [ + 'both scopes', + 'comfy-cloud:user:read comfy-cloud:jobs:write comfy-cloud:workspace-token:mint', + true, + true + ], + ['only the mint scope', 'comfy-cloud:user:read comfy-cloud:workspace-token:mint', true, false], + ['only the partner-node scope', 'comfy-cloud:jobs:write', false, true], + ['a scope that only starts like one', 'comfy-cloud:jobs:write:all', false, false], + ['no scope claim', undefined, false, false], + ['a scope claim that is not text', ['comfy-cloud:jobs:write'], false, false] + ])('reads the mint and partner-node scopes from %s', (_name, scope, canMint, partnerNode) => { + const token = jwt({ sub: 'u-1', scope }) + + expect(canMintWorkspaceTokens(token)).toBe(canMint) + expect(holdsPartnerNodeScope(token)).toBe(partnerNode) + }) + it('reads the identity the hosted view needs, or null without a subject', () => { expect(identityOf(jwt({ sub: 'u-1', email: 'a@b.co', workspace_id: 'w-9' }))).toEqual({ userId: 'u-1', diff --git a/src/main/cloud/claims.ts b/src/main/cloud/claims.ts index 933bb0838..0a874a1fa 100644 --- a/src/main/cloud/claims.ts +++ b/src/main/cloud/claims.ts @@ -1,6 +1,6 @@ import { Buffer } from 'node:buffer' -import { WORKSPACE_MINT_SCOPE } from './config' +import { PARTNER_NODE_MINT_SCOPE, WORKSPACE_MINT_SCOPE } from './config' import type { AuthStatus } from './types' interface AccessTokenClaims { @@ -42,9 +42,17 @@ export function workspaceIdOf(accessToken: string): string | null { return decodeJwtPayload(accessToken)?.workspace_id ?? null } +function holdsScope(accessToken: string, scope: string): boolean { + const granted = decodeJwtPayload(accessToken)?.scope + return typeof granted === 'string' && granted.split(' ').includes(scope) +} + export function canMintWorkspaceTokens(accessToken: string): boolean { - const scope = decodeJwtPayload(accessToken)?.scope - return typeof scope === 'string' && scope.split(' ').includes(WORKSPACE_MINT_SCOPE) + return holdsScope(accessToken, WORKSPACE_MINT_SCOPE) +} + +export function holdsPartnerNodeScope(accessToken: string): boolean { + return holdsScope(accessToken, PARTNER_NODE_MINT_SCOPE) } export function lifetimeSecondsOf(accessToken: string): number | null { diff --git a/src/main/cloud/loopback.test.ts b/src/main/cloud/loopback.test.ts index db44e4259..9ee6f2deb 100644 --- a/src/main/cloud/loopback.test.ts +++ b/src/main/cloud/loopback.test.ts @@ -75,14 +75,14 @@ describe('loopback listener', () => { }) it.for([ - { query: 'error=access_denied', reason: 'cancelled' }, - { query: 'error=server_error', reason: 'idp_error' }, - { query: '', reason: 'idp_error' } - ])('reports a callback with $query as $reason', async ({ query, reason }) => { + { query: 'error=access_denied', reason: 'cancelled', oauthError: 'access_denied' }, + { query: 'error=server_error', reason: 'idp_error', oauthError: 'server_error' }, + { query: '', reason: 'idp_error', oauthError: undefined } + ])('reports a callback with $query as $reason', async ({ query, reason, oauthError }) => { listener = await startLoopbackListener({ expectedState: 'st', timeoutMs: 5000 }) const codeP = listener.waitForCode() await get(`${listener.redirectUri}?state=st&${query}`) - await expect(codeP).rejects.toMatchObject({ name: 'SignInFailure', reason }) + await expect(codeP).rejects.toMatchObject({ name: 'SignInFailure', reason, oauthError }) }) it('reports a callback that never comes as a timeout', async () => { diff --git a/src/main/cloud/oauth.ts b/src/main/cloud/oauth.ts index 9f88a610c..f56855aaf 100644 --- a/src/main/cloud/oauth.ts +++ b/src/main/cloud/oauth.ts @@ -195,11 +195,13 @@ export async function refresh( return toTokens(r, refreshToken) } +export type RevokeResult = 'revoked' | 'refused' | 'retry' + export async function revoke( refreshToken: string, clientId: string, revokeUrl: string = CLOUD_CONFIG.revokeUrl -): Promise<'settled' | 'retry'> { +): Promise { const controller = new AbortController() const timer = setTimeout(() => controller.abort(), TOKEN_REQUEST_TIMEOUT_MS) try { @@ -210,10 +212,10 @@ export async function revoke( redirect: 'manual', signal: controller.signal }) - if (resp.ok) return 'settled' + if (resp.ok) return 'revoked' if (resp.status !== 400 && resp.status !== 401) return 'retry' const detail = await resp.text().catch(() => '') - return PERMANENT_REVOKE_ERRORS.has(oauthErrorOf(detail) ?? '') ? 'settled' : 'retry' + return PERMANENT_REVOKE_ERRORS.has(oauthErrorOf(detail) ?? '') ? 'refused' : 'retry' } catch { return 'retry' } finally { diff --git a/src/main/cloud/revocation.test.ts b/src/main/cloud/revocation.test.ts index bdfe641da..85b49eec1 100644 --- a/src/main/cloud/revocation.test.ts +++ b/src/main/cloud/revocation.test.ts @@ -13,6 +13,8 @@ vi.mock('fs', async (importOriginal) => { return { ...actual, rmSync: vi.fn(actual.rmSync), renameSync: vi.fn(actual.renameSync) } }) +vi.mock('../lib/telemetry', () => ({ emit: vi.fn() })) + vi.mock('electron', () => ({ app: { getPath: vi.fn(() => electronState.userData), isPackaged: false }, safeStorage: { @@ -32,7 +34,8 @@ vi.mock('electron', () => ({ import { safeStorage } from 'electron' import { CLOUD_CONFIG } from './config' -import { Revocations } from './revocation' +import * as mainTelemetry from '../lib/telemetry' +import { GRANT_REVOCATION_EVENT, Revocations } from './revocation' import { _resetForTest, addPendingRevocation, loadPendingRevocations } from './tokenStore' import type { PendingRevocation } from './tokenStore' @@ -68,6 +71,13 @@ function revocation(refreshToken: string): PendingRevocation { } } +function revocationEvents(): unknown[] { + return vi + .mocked(mainTelemetry.emit) + .mock.calls.filter(([event]) => event === GRANT_REVOCATION_EVENT) + .map(([, context]) => context) +} + function queuedTokens(): string[] { _resetForTest() return loadPendingRevocations() @@ -79,6 +89,7 @@ beforeEach(() => { electronState.userData = fs.mkdtempSync(path.join(os.tmpdir(), 'comfy-cloud-revocation-')) _resetForTest() requests = [] + vi.mocked(mainTelemetry.emit).mockClear() written.clear() replies = [] replyFor = {} @@ -171,6 +182,40 @@ describe('Revocations', () => { expect(queuedTokens()).toEqual([]) }) + it('reports a revocation Cloud confirmed', async () => { + const revocations = new Revocations() + + revocations.add(revocation('refresh-1')) + await revocations.drain() + + expect(revocationEvents()).toEqual([{ outcome: 'revoked' }]) + }) + + it('reports a revocation that has to wait once, then that Cloud confirmed it', async () => { + vi.useFakeTimers() + replies = [offline, offline, ok] + const revocations = new Revocations() + + revocations.add(revocation('refresh-1')) + await revocations.drain() + await vi.advanceTimersByTimeAsync(5_000) + expect(revocationEvents()).toEqual([{ outcome: 'retry' }]) + + await vi.advanceTimersByTimeAsync(10_000) + + expect(requests).toHaveLength(3) + expect(revocationEvents()).toEqual([{ outcome: 'retry' }, { outcome: 'revoked' }]) + }) + + it('reports a revocation it gave up on after the token outlived its lifetime', async () => { + addPendingRevocation({ ...revocation('refresh-old'), queuedAt: Date.now() - 720 * 3_600_000 }) + + await new Revocations().drain() + + expect(requests).toEqual([]) + expect(revocationEvents()).toEqual([{ outcome: 'expired' }]) + }) + it('backs off between attempts while Cloud stays unreachable', async () => { vi.useFakeTimers() replies = [offline, offline, offline] @@ -221,6 +266,7 @@ describe('Revocations', () => { expect(requests).toHaveLength(1) expect(queuedTokens()).toEqual([]) + expect(revocationEvents()).toEqual([{ outcome: 'refused' }]) }) it('revokes a token queued before a relaunch', async () => { @@ -237,6 +283,20 @@ describe('Revocations', () => { expect(requests.map((request) => request.token)).toEqual(['refresh-1', 'refresh-1']) expect(queuedTokens()).toEqual([]) + expect(revocationEvents()).toEqual([{ outcome: 'retry' }, { outcome: 'revoked' }]) + }) + + it('reports a waiting revocation again after a relaunch that still cannot reach Cloud', async () => { + vi.useFakeTimers() + replies = [offline, offline] + const before = new Revocations() + before.add(revocation('refresh-1')) + await before.drain() + + _resetForTest() + await new Revocations().drain() + + expect(revocationEvents()).toEqual([{ outcome: 'retry' }, { outcome: 'retry' }]) }) it('sends every queued token, including one queued while another is in flight', async () => { @@ -330,6 +390,7 @@ describe('Revocations', () => { expect(requests.map((request) => request.token)).toEqual(['refresh-1']) expect(fs.readdirSync(dir)).toEqual([]) + expect(revocationEvents()).toEqual([{ outcome: 'revoked' }]) }) it('keeps a token it could not save in memory and leaves no partial file', async () => { @@ -354,6 +415,41 @@ describe('Revocations', () => { } }) + it.each<[string, () => () => void]>([ + [ + 'the disk is full', + () => { + vi.mocked(fs.renameSync).mockImplementation(() => { + throw Object.assign(new Error('disk full'), { code: 'ENOSPC' }) + }) + return () => vi.mocked(fs.renameSync).mockReset() + } + ], + [ + 'secure storage is unavailable', + () => { + vi.mocked(safeStorage.isEncryptionAvailable).mockReturnValue(false) + return () => vi.mocked(safeStorage.isEncryptionAvailable).mockReturnValue(true) + } + ] + ])( + 'reports a revocation it could not save because %s, and still that Cloud confirmed it', + async (_name, breakStorage) => { + const repairStorage = breakStorage() + const revocations = new Revocations() + + try { + revocations.add(revocation('refresh-1')) + await revocations.drain() + + expect(requests.map((request) => request.token)).toEqual(['refresh-1']) + expect(revocationEvents()).toEqual([{ outcome: 'not_saved' }, { outcome: 'revoked' }]) + } finally { + repairStorage() + } + } + ) + it('revokes a token queued by an earlier build, dating it by its file', async () => { const dir = path.join(electronState.userData, 'comfy-cloud-revocations') const monthAgo = new Date(Date.now() - 31 * 24 * 3_600_000) diff --git a/src/main/cloud/revocation.ts b/src/main/cloud/revocation.ts index f2539e8b8..fe97a739d 100644 --- a/src/main/cloud/revocation.ts +++ b/src/main/cloud/revocation.ts @@ -1,8 +1,12 @@ +import * as mainTelemetry from '../lib/telemetry' import { CLOUD_REFRESH_TOKEN_LIFETIME_MS, revokeUrlOf } from './config' import { revoke } from './oauth' +import type { RevokeResult } from './oauth' import { addPendingRevocation, loadPendingRevocations, removePendingRevocation } from './tokenStore' import type { PendingRevocation } from './tokenStore' +export const GRANT_REVOCATION_EVENT = 'comfy.desktop.auth.grant_revocation' + const FIRST_RETRY_MS = 5_000 const MAX_RETRY_MS = 15 * 60_000 @@ -12,9 +16,11 @@ export class Revocations { private failures = 0 private retryTimer: ReturnType | null = null private readonly settledButSaved = new Set() + private readonly reportedWaiting = new Set() add(revocation: Omit): void { - addPendingRevocation({ ...revocation, queuedAt: Date.now() }) + const saved = addPendingRevocation({ ...revocation, queuedAt: Date.now() }) + if (!saved) mainTelemetry.emit(GRANT_REVOCATION_EVENT, { outcome: 'not_saved' }) if (this.draining) this.drainAgain = true else void this.drain() } @@ -49,10 +55,11 @@ export class Revocations { return } const expired = next.queuedAt + CLOUD_REFRESH_TOKEN_LIFETIME_MS <= Date.now() - if ( - !expired && - (await revoke(next.refreshToken, next.clientId, revokeUrlOf(next.issuer))) === 'retry' - ) { + const outcome = expired + ? 'expired' + : await revoke(next.refreshToken, next.clientId, revokeUrlOf(next.issuer)) + this.report(next.refreshToken, outcome) + if (outcome === 'retry') { failed.add(next.refreshToken) } else if (!removePendingRevocation(next.refreshToken)) { this.settledButSaved.add(next.refreshToken) @@ -60,6 +67,16 @@ export class Revocations { } } + private report(refreshToken: string, outcome: RevokeResult | 'expired'): void { + if (outcome === 'retry') { + if (this.reportedWaiting.has(refreshToken)) return + this.reportedWaiting.add(refreshToken) + } else { + this.reportedWaiting.delete(refreshToken) + } + mainTelemetry.emit(GRANT_REVOCATION_EVENT, { outcome }) + } + private scheduleRetry(): void { const delay = Math.min(FIRST_RETRY_MS * 2 ** this.failures, MAX_RETRY_MS) this.failures += 1 diff --git a/src/main/cloud/session.test.ts b/src/main/cloud/session.test.ts index 0b85bb137..b5b3a767b 100644 --- a/src/main/cloud/session.test.ts +++ b/src/main/cloud/session.test.ts @@ -1,3 +1,5 @@ +import * as storeFs from 'fs' +import type * as FsModule from 'fs' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' @@ -6,6 +8,10 @@ import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from const electronState = vi.hoisted(() => ({ userData: '' })) +vi.mock('fs', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, rmSync: vi.fn(actual.rmSync) } +}) vi.mock('electron', () => ({ app: { getPath: vi.fn(() => electronState.userData) }, safeStorage: { @@ -46,10 +52,15 @@ import { openAuthorizePage, refresh, signIn } from './oauth' import type * as OAuthModule from './oauth' import { CloudSession, + EXPIRED_GRANT_REJECTED_EVENT, + GRANT_REFRESH_FAILED_EVENT, MINT_SCOPE_NOT_GRANTED_EVENT, + SESSION_SIGN_IN_EVENT, + SIGNED_OUT_EVENT, WORKSPACE_RETURNED_TO_ANCHOR_EVENT, WORKSPACE_SWITCH_BROWSER_EVENT, - WORKSPACE_SWITCH_EVENT + WORKSPACE_SWITCH_EVENT, + WORKSPACE_TOKEN_MINT_FAILED_EVENT } from './session' import { SignInFailure } from './signInFailure' import { @@ -162,6 +173,12 @@ function deferred() { return { promise, resolve } } +function eventsNamed(name: string): unknown[] { + return mocked(mainTelemetry.emit) + .mock.calls.filter(([event]) => event === name) + .map(([, context]) => context) +} + function switchEvents(): unknown[] { return mocked(mainTelemetry.emit) .mock.calls.filter(([event]) => event === WORKSPACE_SWITCH_EVENT) @@ -2252,6 +2269,408 @@ describe('CloudSession status and catalog', () => { }) }) +describe('CloudSession sign-out telemetry', () => { + it('reports a sign-out the person asked for', () => { + saveGrant(grantOn('w1')) + + new CloudSession().logout() + + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'user', stored_session_removed: true } + ]) + }) + + it('says so when a sign-out could not remove the stored session', () => { + saveGrant(grantOn('w1')) + const session = new CloudSession() + mocked(storeFs.rmSync).mockImplementationOnce(() => { + throw Object.assign(new Error('locked'), { code: 'EPERM' }) + }) + + session.logout() + + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'user', stored_session_removed: false } + ]) + }) + + it('says so when secure storage is unavailable and the stored session could not be removed', () => { + mocked(safeStorage.isEncryptionAvailable).mockReturnValue(false) + onTestFinished(() => { + mocked(safeStorage.isEncryptionAvailable).mockReturnValue(true) + }) + saveGrant(grantOn('w1')) + const session = new CloudSession() + mocked(storeFs.rmSync).mockImplementationOnce(() => { + throw Object.assign(new Error('locked'), { code: 'EPERM' }) + }) + + session.logout() + + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'user', stored_session_removed: false } + ]) + }) + + it('reports nothing when nobody is signed in', () => { + new CloudSession().logout() + + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([]) + }) +}) + +describe('CloudSession sign-in telemetry', () => { + const SIGNED_IN = { + purpose: 'sign_in', + outcome: 'completed', + mint_scope: true, + partner_node_scope: true, + session_saved: true + } + + it('reports a completed sign-in once, however many callers asked for it', async () => { + const pending = deferred>>() + mocked(signIn).mockReturnValue(pending.promise) + const session = new CloudSession() + + const logins = [session.login(), session.login()] + pending.resolve(signedInResult(grantOn('w1'))) + await Promise.all(logins) + + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([SIGNED_IN]) + }) + + it.each<[string, string, boolean, boolean]>([ + ['only the read scope', READ_ONLY_SCOPE, false, false], + ['no partner-node scope', `${READ_ONLY_SCOPE} comfy-cloud:workspace-token:mint`, true, false], + ['no mint scope', `${READ_ONLY_SCOPE} comfy-cloud:jobs:write`, false, true] + ])('reports the scopes of a grant with %s', async (_name, scope, mintScope, partnerNodeScope) => { + mocked(signIn).mockResolvedValue(signedInResult(grantOn('w1', { scope }))) + + await new CloudSession().login() + + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { ...SIGNED_IN, mint_scope: mintScope, partner_node_scope: partnerNodeScope } + ]) + }) + + it('says so when the signed-in session is kept in memory only', async () => { + mocked(safeStorage.isEncryptionAvailable).mockReturnValue(false) + onTestFinished(() => { + mocked(safeStorage.isEncryptionAvailable).mockReturnValue(true) + }) + mocked(signIn).mockResolvedValue(signedInResult(grantOn('w1'))) + const session = new CloudSession() + + await session.login() + + expect(session.isSignedIn()).toBe(true) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([{ ...SIGNED_IN, session_saved: false }]) + }) + + it('reports whether the sign-in itself was written, whatever a later write does', async () => { + mocked(safeStorage.encryptString).mockImplementationOnce(() => { + throw new Error('keychain locked') + }) + onTestFinished(() => { + mocked(safeStorage.encryptString).mockReset() + }) + mocked(signIn).mockResolvedValue(signedInResult(grantOn('w1'))) + + await new CloudSession().login() + + expect(vaultOnDisk()).toMatchObject({ activeWorkspaceId: 'w1' }) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([{ ...SIGNED_IN, session_saved: false }]) + }) + + it('reports the sign-in as saved when only the grant it replaced could not be queued', async () => { + saveGrant(grantOn('w1')) + mocked(signIn).mockResolvedValue(signedInResult(grantOn('w1', { revision: 'b' }))) + mocked(safeStorage.isEncryptionAvailable).mockReturnValueOnce(true).mockReturnValue(false) + onTestFinished(() => { + mocked(safeStorage.isEncryptionAvailable).mockReset() + }) + + await new CloudSession().login() + + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([SIGNED_IN]) + }) + + it.each<[string, () => unknown, Record]>([ + [ + 'the person declined', + () => new SignInFailure('access_denied', 'cancelled', { oauthError: 'access_denied' }), + { reason: 'cancelled', oauth_error: 'access_denied' } + ], + [ + 'Cloud rejected the request', + () => new SignInFailure('invalid_scope', 'idp_error', { oauthError: 'invalid_scope' }), + { reason: 'idp_error', oauth_error: 'invalid_scope' } + ], + [ + 'nobody answered in time', + () => new SignInFailure('timed out', 'timeout'), + { reason: 'timeout' } + ], + [ + 'Cloud refused the code', + () => refreshRefused('invalid_grant', 400), + { reason: 'server_error', http_status: 400, oauth_error: 'invalid_grant' } + ], + [ + 'Cloud answered with an error this build does not know', + () => refreshRefused('account ws_3f9c2a71 is locked', 400), + { reason: 'server_error', http_status: 400, oauth_error: 'other' } + ], + ['the listener could not start', () => new Error('EADDRINUSE'), { reason: 'unknown' }] + ])('reports a sign-in that failed because %s', async (_name, failure, reported) => { + const error = failure() + mocked(signIn).mockRejectedValue(error) + const session = new CloudSession() + + await expect(session.login()).rejects.toBe(error) + + expect(session.isSignedIn()).toBe(false) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { purpose: 'sign_in', outcome: 'failed', ...reported } + ]) + }) + + it('reports a sign-in that finished after a sign-out as superseded', async () => { + const pending = deferred>>() + mocked(signIn).mockReturnValue(pending.promise) + const session = new CloudSession() + + const login = session.login() + session.logout() + pending.resolve(signedInResult(grantOn('w1'))) + await login + + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { purpose: 'sign_in', outcome: 'superseded' } + ]) + }) + + it('reports the sign-in a workspace switch needed', async () => { + saveGrant(grantOn('w1')) + mintReply = ({ workspaceId }) => + mintRequests.length === 1 ? json(403, { code: 'sso_required' }) : minted(workspaceId) + browserSignInReturns(signedInResult(grantOn('w2'))) + + const reply = await new CloudSession().switchWorkspace('w2') + + expect(reply).toMatchObject({ workspaceId: 'w2', reached: true }) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { ...SIGNED_IN, purpose: 'workspace_switch' } + ]) + }) + + it('reports a workspace switch whose sign-in the person abandoned', async () => { + saveGrant(grantOn('w1')) + mintReply = () => json(403, { code: 'sso_required' }) + mocked(signIn).mockImplementation(async (options) => { + options?.onAuthorizeUrl?.('https://cloud.test/authorize') + throw new SignInFailure('timed out', 'timeout') + }) + + const reply = await new CloudSession().switchWorkspace('w2') + + expect(reply).toMatchObject({ workspaceId: 'w1', workspaceSwitchFailure: 'retryable' }) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { purpose: 'workspace_switch', outcome: 'failed', reason: 'timeout' } + ]) + }) + + it('reports a workspace switch sign-in that finished after a sign-out as superseded', async () => { + saveGrant(grantOn('w1')) + mintReply = () => json(403, { code: 'sso_required' }) + const pending = deferred>>() + mocked(signIn).mockReturnValue(pending.promise) + const session = new CloudSession() + + const switching = session.switchWorkspace('w2') + await vi.waitFor(() => expect(signIn).toHaveBeenCalledOnce()) + session.logout() + pending.resolve(signedInResult(grantOn('w2'))) + await switching + + expect(session.isSignedIn()).toBe(false) + expect(eventsNamed(SESSION_SIGN_IN_EVENT)).toEqual([ + { purpose: 'workspace_switch', outcome: 'superseded' } + ]) + }) +}) + +describe('CloudSession refresh telemetry', () => { + const NEARLY_EXPIRED = Date.now() + 30_000 + + it.each<[string, () => SignInFailure, Record]>([ + [ + 'Cloud is unreachable', + () => refreshFailure('network'), + { outcome: 'unavailable', reason: 'network' } + ], + [ + 'Cloud is slow', + () => refreshFailure('timeout'), + { outcome: 'unavailable', reason: 'timeout' } + ], + [ + 'Cloud answers 503', + () => refreshFailure('http', 503), + { outcome: 'unavailable', reason: 'server_error', http_status: 503 } + ], + [ + 'Cloud ended the grant', + () => refreshRefused(), + { outcome: 'ended', reason: 'server_error', http_status: 400, oauth_error: 'invalid_grant' } + ] + ])('reports a refresh that failed because %s', async (_name, failure, reported) => { + saveGrant(grantOn('w1', { expiresAt: NEARLY_EXPIRED })) + mocked(refresh).mockRejectedValue(failure()) + + await new CloudSession().getGrantAccessToken() + + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toEqual([reported]) + }) + + it('reports a refresh that keeps failing the same way once, and again after one works', async () => { + saveGrant(grantOn('w1', { expiresAt: NEARLY_EXPIRED })) + const session = new CloudSession() + mocked(refresh).mockRejectedValue(refreshFailure('network')) + + await session.getGrantAccessToken() + await session.getGrantAccessToken() + await session.getGrantAccessToken() + + expect(refresh).toHaveBeenCalledTimes(3) + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toHaveLength(1) + + mocked(refresh).mockResolvedValueOnce( + grantOn('w1', { expiresAt: NEARLY_EXPIRED, revision: 'b' }) + ) + await session.getGrantAccessToken() + await session.getGrantAccessToken() + + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toEqual([ + { outcome: 'unavailable', reason: 'network' }, + { outcome: 'unavailable', reason: 'network' } + ]) + }) + + it('reports each way a refresh fails once while none works, even when they alternate', async () => { + saveGrant(grantOn('w1', { expiresAt: NEARLY_EXPIRED })) + const session = new CloudSession() + mocked(refresh) + .mockRejectedValueOnce(refreshFailure('network')) + .mockRejectedValueOnce(refreshFailure('http', 503)) + .mockRejectedValueOnce(refreshFailure('network')) + .mockRejectedValueOnce(refreshFailure('http', 503)) + + await session.getGrantAccessToken() + await session.getGrantAccessToken() + await session.getGrantAccessToken() + await session.getGrantAccessToken() + + expect(refresh).toHaveBeenCalledTimes(4) + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toEqual([ + { outcome: 'unavailable', reason: 'network' }, + { outcome: 'unavailable', reason: 'server_error', http_status: 503 } + ]) + }) + + it('reports the same failure again for the grant of a new sign-in', async () => { + saveGrant(grantOn('w1', { expiresAt: NEARLY_EXPIRED })) + const session = new CloudSession() + mocked(refresh).mockRejectedValue(refreshFailure('network')) + await session.getGrantAccessToken() + mocked(signIn).mockResolvedValue( + signedInResult(grantOn('w1', { expiresAt: NEARLY_EXPIRED, revision: 'b' })) + ) + + await session.login() + await session.getGrantAccessToken() + + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toHaveLength(2) + }) + + it('reports nothing for a refresh that works', async () => { + saveGrant(grantOn('w1', { expiresAt: NEARLY_EXPIRED })) + mocked(refresh).mockResolvedValue(grantOn('w1', { revision: 'b' })) + + await new CloudSession().getGrantAccessToken() + + expect(refresh).toHaveBeenCalledOnce() + expect(eventsNamed(GRANT_REFRESH_FAILED_EVENT)).toEqual([]) + }) +}) + +describe('CloudSession workspace mint telemetry', () => { + it.each<[string, MintReply, Record]>([ + [ + 'a refusal Cloud names', + () => json(403, { code: 'insufficient_scope' }), + { reason: 'http', http_status: 403, error_code: 'insufficient_scope' } + ], + [ + 'a refusal this build does not know', + () => json(403, { code: 'ws_3f9c2a71' }), + { reason: 'http', http_status: 403, error_code: 'other' } + ], + ['an error with no code', () => json(500, {}), { reason: 'http', http_status: 500 }], + [ + 'no answer', + () => { + throw new TypeError('fetch failed') + }, + { reason: 'network' } + ] + ])( + 'reports a switch that Desktop could not mint for after %s', + async (_name, reply, reported) => { + saveGrant(grantOn('w1')) + mintReply = reply + + const switched = await new CloudSession().switchWorkspace('w2') + + expect(switched).toMatchObject({ workspaceId: 'w1', reached: false }) + expect(eventsNamed(WORKSPACE_TOKEN_MINT_FAILED_EVENT)).toEqual([reported]) + } + ) + + it.each<[string, () => string, string]>([ + ['another workspace', () => mintedJwt('w3'), 'workspace_mismatch'], + ['another account', () => mintedJwt('w2', 'user-2'), 'subject_mismatch'] + ])( + 'reports a token Cloud minted for %s as a failed mint, once for the switches sharing it', + async (_name, token, problem) => { + saveGrant(grantOn('w1')) + mintReply = () => json(200, { token: token(), expires_at: new Date(FUTURE).toISOString() }) + const session = new CloudSession() + + const switches = await Promise.all([ + session.switchWorkspace('w2'), + session.switchWorkspace('w2') + ]) + + expect(switches[0]).toMatchObject({ workspaceId: 'w1', workspaceSwitchFailure: 'retryable' }) + expect(session.heldWorkspaceToken('w2')).toBeNull() + expect(eventsNamed(WORKSPACE_TOKEN_MINT_FAILED_EVENT)).toEqual([ + { reason: 'unusable_token', problem } + ]) + } + ) + + it('reports nothing for a switch Desktop minted for', async () => { + saveGrant(grantOn('w1')) + + await new CloudSession().switchWorkspace('w2') + + expect(mintRequests).toHaveLength(1) + expect(eventsNamed(WORKSPACE_TOKEN_MINT_FAILED_EVENT)).toEqual([]) + }) +}) + describe('CloudSession token provider', () => { it('logs out when the grant token is rejected', () => { const grant = grantOn('w1') @@ -2263,6 +2682,10 @@ describe('CloudSession token provider', () => { expect(session.isSignedIn()).toBe(false) expect(onSignedOut).toHaveBeenCalledOnce() + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'grant_rejected', stored_session_removed: true } + ]) + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([]) }) it.each([ @@ -2283,6 +2706,8 @@ describe('CloudSession token provider', () => { expect(session.isSignedIn()).toBe(true) expect(onSignedOut).not.toHaveBeenCalled() + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([{ refresh: 'unavailable' }]) + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([]) } ) @@ -2298,6 +2723,33 @@ describe('CloudSession token provider', () => { await vi.waitFor(() => expect(onSignedOut).toHaveBeenCalledOnce()) expect(refresh).toHaveBeenCalledOnce() expect(session.isSignedIn()).toBe(false) + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([{ refresh: 'ended' }]) + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'refresh_refused', stored_session_removed: true } + ]) + }) + + it('reports each rejected request and signs out once when two share one refused refresh', async () => { + const grant = grantOn('w1', { expiresAt: PAST }) + saveGrant(grant) + mocked(refresh).mockRejectedValue(refreshRefused()) + const session = new CloudSession() + const onSignedOut = vi.fn() + const provider = session.asTokenProvider(onSignedOut) + + provider.onUnauthorized?.(grant.accessToken) + provider.onUnauthorized?.(grant.accessToken) + + await vi.waitFor(() => expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toHaveLength(2)) + expect(refresh).toHaveBeenCalledOnce() + expect(onSignedOut).toHaveBeenCalledOnce() + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([ + { refresh: 'ended' }, + { refresh: 'ended' } + ]) + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([ + { reason: 'refresh_refused', stored_session_removed: true } + ]) }) it('leaves a newer sign-in alone when the refusal for a rejected expired grant lands late', async () => { @@ -2324,6 +2776,8 @@ describe('CloudSession token provider', () => { expect(onSignedOut).not.toHaveBeenCalled() expect(vaultOnDisk().grant).toEqual(fresh) + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([{ refresh: 'ended' }]) + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([]) }) it('keeps the refreshed grant when a rejected grant token had only expired', async () => { @@ -2339,6 +2793,10 @@ describe('CloudSession token provider', () => { await vi.waitFor(() => expect(vaultOnDisk().grant).toEqual(rotated)) expect(session.isSignedIn()).toBe(true) expect(onSignedOut).not.toHaveBeenCalled() + await vi.waitFor(() => + expect(eventsNamed(EXPIRED_GRANT_REJECTED_EVENT)).toEqual([{ refresh: 'refreshed' }]) + ) + expect(eventsNamed(SIGNED_OUT_EVENT)).toEqual([]) }) it('drops only a rejected minted token, stays signed in, and re-mints with no browser', async () => { diff --git a/src/main/cloud/session.ts b/src/main/cloud/session.ts index 40b355e0e..c859321d3 100644 --- a/src/main/cloud/session.ts +++ b/src/main/cloud/session.ts @@ -1,11 +1,18 @@ import type { TokenProvider } from '../comfybuilder' import * as mainTelemetry from '../lib/telemetry' import { isWorkspaceMintSwitchEnabled } from '../lib/workspaceMintSwitchFlag' -import { canMintWorkspaceTokens, grantFamilyOf, subjectOf, workspaceIdOf } from './claims' +import { + canMintWorkspaceTokens, + grantFamilyOf, + holdsPartnerNodeScope, + subjectOf, + workspaceIdOf +} from './claims' import { CLOUD_CONFIG } from './config' import { openAuthorizePage, refresh, signIn } from './oauth' +import type { OAuthOptions } from './oauth' import { Revocations } from './revocation' -import { SignInFailure } from './signInFailure' +import { SignInFailure, signInFailureReport } from './signInFailure' import { activateWorkspace, clearTokens, @@ -13,6 +20,7 @@ import { endGrant, getAuthStatus, getCachedWorkspaceName, + isSecureStorageUnavailable, loadActiveWorkspaceId, loadGrant, loadGrantOrigin, @@ -32,7 +40,12 @@ import type { WorkspaceSwitchIntent, WorkspaceSwitchStatus } from './types' -import { classifyMint, mintWorkspaceToken } from './workspaceToken' +import { + classifyMint, + mintedTokenProblem, + mintFailureReport, + mintWorkspaceToken +} from './workspaceToken' import type { ReauthorizeReason, WorkspaceMintOutcome } from './workspaceToken' import { listWorkspaceMembers, listWorkspaces } from './workspaces' @@ -43,6 +56,15 @@ export const WORKSPACE_SWITCH_EVENT = 'comfy.desktop.auth.workspace_switch' export const WORKSPACE_SWITCH_BROWSER_EVENT = 'comfy.desktop.auth.workspace_switch_browser' export const MINT_SCOPE_NOT_GRANTED_EVENT = 'comfy.desktop.auth.mint_scope_not_granted' export const WORKSPACE_RETURNED_TO_ANCHOR_EVENT = 'comfy.desktop.auth.workspace_returned_to_anchor' +export const SIGNED_OUT_EVENT = 'comfy.desktop.auth.signed_out' +export const EXPIRED_GRANT_REJECTED_EVENT = 'comfy.desktop.auth.expired_grant_rejected' +export const SESSION_SIGN_IN_EVENT = 'comfy.desktop.auth.session_sign_in' +export const GRANT_REFRESH_FAILED_EVENT = 'comfy.desktop.auth.grant_refresh_failed' +export const WORKSPACE_TOKEN_MINT_FAILED_EVENT = 'comfy.desktop.auth.workspace_token_mint_failed' + +type SignOutReason = 'user' | 'grant_rejected' | 'refresh_refused' + +type SignInPurpose = 'sign_in' | 'workspace_switch' export interface SessionSwitchReply extends WorkspaceSwitchStatus { reached: boolean @@ -101,6 +123,22 @@ function reportBrowser(reason: ReauthorizeReason): void { mainTelemetry.emit(WORKSPACE_SWITCH_BROWSER_EVENT, { reauthorize_reason: reason }) } +function reportSignIn(purpose: SignInPurpose, result: mainTelemetry.TelemetryContext): void { + mainTelemetry.emit(SESSION_SIGN_IN_EVENT, { purpose, ...result }) +} + +async function reportedSignIn( + purpose: SignInPurpose, + options: OAuthOptions +): ReturnType { + try { + return await signIn(options) + } catch (error) { + reportSignIn(purpose, { outcome: 'failed', ...signInFailureReport(error) }) + throw error + } +} + export class CloudSession { private readonly refreshing = new Map>() private readonly rotatedTo = new Map() @@ -114,6 +152,7 @@ export class CloudSession { private returnedToAnchorHandler: (() => void) | null = null private readonly revocations = new Revocations() private readonly retiredRefreshTokens = new Set() + private readonly reportedRefreshFailures = new Set() private authGeneration = 0 @@ -142,29 +181,38 @@ export class CloudSession { } private async signInAtGeneration(generation: number): Promise { - const { tokens, status } = await signIn({ + const { tokens, status } = await reportedSignIn('sign_in', { onAuthorizeUrl: (url) => { if (generation === this.authGeneration) this.loginAuthorizeUrl = url } }) if (generation !== this.authGeneration) { this.revokeUnstored(tokens.refreshToken) + reportSignIn('sign_in', { outcome: 'superseded' }) return this.status() } - this.storeSignedInGrant(tokens) + this.storeSignedInGrant(tokens, 'sign_in') const anchor = workspaceIdOf(tokens.accessToken) if (anchor) activateWorkspace(anchor) return status } - private storeSignedInGrant(tokens: AuthTokens): void { + private storeSignedInGrant(tokens: AuthTokens, purpose: SignInPurpose): void { const replaced = loadGrant()?.refreshToken const origin = loadGrantOrigin() this.mintBlockedUntil = 0 + this.reportedRefreshFailures.clear() const saved = saveGrant(tokens) + const sessionSaved = saved && !isSecureStorageUnavailable() if (saved && replaced !== tokens.refreshToken) this.retireGrant(replaced, origin) - if (!canMintWorkspaceTokens(tokens.accessToken)) - mainTelemetry.emit(MINT_SCOPE_NOT_GRANTED_EVENT) + const mintScope = canMintWorkspaceTokens(tokens.accessToken) + if (!mintScope) mainTelemetry.emit(MINT_SCOPE_NOT_GRANTED_EVENT) + reportSignIn(purpose, { + outcome: 'completed', + mint_scope: mintScope, + partner_node_scope: holdsPartnerNodeScope(tokens.accessToken), + session_saved: sessionSaved + }) } private clearPendingLogin(): void { @@ -172,15 +220,16 @@ export class CloudSession { this.loginAuthorizeUrl = null } - logout(): void { - const refreshToken = loadGrant()?.refreshToken + logout(reason: SignOutReason = 'user'): void { + const grant = loadGrant() const origin = loadGrantOrigin() this.authGeneration += 1 this.switching = null this.mintBlockedUntil = 0 this.clearPendingLogin() - this.retireGrant(refreshToken, origin) - clearTokens() + this.retireGrant(grant?.refreshToken, origin) + const removed = clearTokens() + if (grant) mainTelemetry.emit(SIGNED_OUT_EVENT, { reason, stored_session_removed: removed }) } private retireGrant(refreshToken: string | undefined, origin: GrantOrigin | null): void { @@ -299,6 +348,7 @@ export class CloudSession { private async rotateGrant(grant: AuthTokens, refreshToken: string): Promise { try { const rotated = await refresh(refreshToken) + this.reportedRefreshFailures.clear() const stored = replaceGrant(grant.accessToken, refreshToken, rotated) if (stored) this.rotatedTo.set(grant.accessToken, rotated.accessToken) if (!stored && this.retiredRefreshTokens.has(refreshToken)) { @@ -306,12 +356,22 @@ export class CloudSession { } return 'refreshed' } catch (error) { - return isDefinitiveRefreshRefusal(error) ? 'ended' : 'unavailable' + const outcome = isDefinitiveRefreshRefusal(error) ? 'ended' : 'unavailable' + this.reportRefreshFailure(outcome, error) + return outcome } finally { this.retiredRefreshTokens.delete(refreshToken) } } + private reportRefreshFailure(outcome: Exclude, error: unknown): void { + const failure = { outcome, ...signInFailureReport(error) } + const report = JSON.stringify(failure) + if (this.reportedRefreshFailures.has(report)) return + this.reportedRefreshFailures.add(report) + mainTelemetry.emit(GRANT_REFRESH_FAILED_EVENT, failure) + } + private mintForWorkspace(workspaceId: string, generation: number): Promise { const key = JSON.stringify([generation, workspaceId]) const inFlight = this.minting.get(key) @@ -332,11 +392,10 @@ export class CloudSession { ): SettledMintOutcome { if (outcome.kind !== 'minted' || generation !== this.authGeneration) return outcome const grant = loadGrant() - if ( - !grant || - workspaceIdOf(outcome.token) !== workspaceId || - subjectOf(outcome.token) !== subjectOf(grant.accessToken) - ) { + if (!grant) return RETRYABLE + const problem = mintedTokenProblem(outcome.token, workspaceId, subjectOf(grant.accessToken)) + if (problem) { + mainTelemetry.emit(WORKSPACE_TOKEN_MINT_FAILED_EVENT, { reason: 'unusable_token', problem }) return RETRYABLE } saveWorkspaceToken(workspaceId, { accessToken: outcome.token, expiresAt: outcome.expiresAt }) @@ -383,7 +442,10 @@ export class CloudSession { Date.now() < this.mintBlockedUntil ) return RETRYABLE - const outcome = classifyMint(await mintWorkspaceToken(grant.accessToken, workspaceId)) + const mint = await mintWorkspaceToken(grant.accessToken, workspaceId) + if (mint.kind === 'failed') + mainTelemetry.emit(WORKSPACE_TOKEN_MINT_FAILED_EVENT, mintFailureReport(mint)) + const outcome = classifyMint(mint) if (outcome.kind === 'reauthorize' && outcome.reason === 'grant_ended') { this.endGrantFamily(grant.accessToken) } @@ -525,7 +587,7 @@ export class CloudSession { const viewConsent = this.switchIntent === 'requested' if (viewConsent) this.viewConsentOpen = true try { - ;({ tokens } = await signIn({ + ;({ tokens } = await reportedSignIn('workspace_switch', { workspaceId, onAuthorizeUrl: () => { browserOpened = true @@ -540,9 +602,10 @@ export class CloudSession { } if (generation !== this.authGeneration) { this.revokeUnstored(tokens.refreshToken) + reportSignIn('workspace_switch', { outcome: 'superseded' }) return this.unreached() } - this.storeSignedInGrant(tokens) + this.storeSignedInGrant(tokens, 'workspace_switch') const canMint = canMintWorkspaceTokens(tokens.accessToken) const isAnchor = workspaceId === workspaceIdOf(tokens.accessToken) @@ -579,7 +642,7 @@ export class CloudSession { void this.signOutIfGrantEnded(grant, onSignedOut) return } - this.logout() + this.logout('grant_rejected') onSignedOut?.() } } @@ -588,9 +651,10 @@ export class CloudSession { private async signOutIfGrantEnded(grant: AuthTokens, onSignedOut?: () => void): Promise { const generation = this.authGeneration const refreshed = await this.refreshGrant(grant) + mainTelemetry.emit(EXPIRED_GRANT_REJECTED_EVENT, { refresh: refreshed }) if (refreshed !== 'ended' || generation !== this.authGeneration) return if (loadGrant()?.accessToken !== grant.accessToken) return - this.logout() + this.logout('refresh_refused') onSignedOut?.() } } diff --git a/src/main/cloud/signInFailure.test.ts b/src/main/cloud/signInFailure.test.ts new file mode 100644 index 000000000..d6f074ce5 --- /dev/null +++ b/src/main/cloud/signInFailure.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' + +import { idpCallbackFailure, SignInFailure, signInFailureReport } from './signInFailure' + +describe('signInFailureReport', () => { + it.each<[string, unknown, Record]>([ + [ + 'a sign-in the person declined', + idpCallbackFailure('access_denied'), + { reason: 'cancelled', oauth_error: 'access_denied' } + ], + [ + 'a request Cloud rejected at the authorize step', + idpCallbackFailure('invalid_scope'), + { reason: 'idp_error', oauth_error: 'invalid_scope' } + ], + [ + 'a callback error this build does not know', + idpCallbackFailure('user a@example.com is suspended'), + { reason: 'idp_error', oauth_error: 'other' } + ], + [ + 'a refused token request', + new SignInFailure('refused', 'server_error', { + httpStatus: 400, + oauthError: 'invalid_grant' + }), + { reason: 'server_error', http_status: 400, oauth_error: 'invalid_grant' } + ], + [ + 'a token request that failed with no OAuth error', + new SignInFailure('unavailable', 'server_error', { httpStatus: 503 }), + { reason: 'server_error', http_status: 503 } + ], + [ + 'a request that never reached Cloud', + new SignInFailure('offline', 'network'), + { reason: 'network' } + ], + ['anything that is not a sign-in failure', new Error('EADDRINUSE'), { reason: 'unknown' }] + ])('reports %s', (_name, error, report) => { + expect(signInFailureReport(error)).toEqual(report) + }) +}) diff --git a/src/main/cloud/signInFailure.ts b/src/main/cloud/signInFailure.ts index 0fe2208d5..469328611 100644 --- a/src/main/cloud/signInFailure.ts +++ b/src/main/cloud/signInFailure.ts @@ -29,6 +29,44 @@ const DECLINED_OAUTH_ERRORS: ReadonlySet = new Set(['access_denied']) export function idpCallbackFailure(oauthError: string): SignInFailure { return new SignInFailure( oauthError, - DECLINED_OAUTH_ERRORS.has(oauthError) ? 'cancelled' : 'idp_error' + DECLINED_OAUTH_ERRORS.has(oauthError) ? 'cancelled' : 'idp_error', + { oauthError } ) } + +const REPORTABLE_OAUTH_ERRORS: ReadonlySet = new Set([ + 'access_denied', + 'invalid_client', + 'invalid_grant', + 'invalid_request', + 'invalid_resource', + 'invalid_scope', + 'invalid_target', + 'login_required', + 'server_error', + 'temporarily_unavailable', + 'unauthorized_client', + 'unsupported_grant_type', + 'unsupported_response_type' +]) +const UNLISTED_OAUTH_ERROR = 'other' + +export interface SignInFailureReport { + reason: SignInFailureReason | 'unknown' + http_status?: number + oauth_error?: string +} + +export function signInFailureReport(error: unknown): SignInFailureReport { + if (!(error instanceof SignInFailure)) return { reason: 'unknown' } + const { reason, httpStatus, oauthError } = error + return { + reason, + ...(httpStatus === undefined ? {} : { http_status: httpStatus }), + ...(oauthError === undefined + ? {} + : { + oauth_error: REPORTABLE_OAUTH_ERRORS.has(oauthError) ? oauthError : UNLISTED_OAUTH_ERROR + }) + } +} diff --git a/src/main/cloud/tokenStore.test.ts b/src/main/cloud/tokenStore.test.ts index 07ad35635..6c17889ab 100644 --- a/src/main/cloud/tokenStore.test.ts +++ b/src/main/cloud/tokenStore.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' @@ -20,10 +20,15 @@ vi.mock('electron', () => ({ } })) -import { safeStorage } from 'electron' +vi.mock('../lib/telemetry', () => ({ emit: vi.fn() })) +import { app, safeStorage } from 'electron' + +import * as mainTelemetry from '../lib/telemetry' import { CLOUD_CONFIG } from './config' import { + STORED_SESSION_FAILED_EVENT, + STORED_SESSION_RESTORED_EVENT, _resetForTest, activateWorkspace, addPendingRevocation, @@ -83,11 +88,19 @@ function readVault(): Record { > } +function eventsNamed(name: string): unknown[] { + return vi + .mocked(mainTelemetry.emit) + .mock.calls.filter(([event]) => event === name) + .map(([, context]) => context) +} + describe('grant vault', () => { beforeEach(() => { electronState.userData = fs.mkdtempSync(path.join(os.tmpdir(), 'comfy-cloud-tokens-')) electronState.encryptionAvailable = true _resetForTest() + vi.mocked(mainTelemetry.emit).mockClear() }) afterEach(() => { @@ -513,4 +526,154 @@ describe('grant vault', () => { expect(loadGrant()).toBeNull() expect(getAuthStatus()).toEqual({ signedIn: false }) }) + + describe('stored session telemetry', () => { + it('reports a session restored from disk once, however often it is read', () => { + saveGrant(tokens('user-1', 'w1')) + _resetForTest() + + loadGrant() + loadActiveWorkspaceId() + getAuthStatus() + + expect(eventsNamed(STORED_SESSION_RESTORED_EVENT)).toEqual([{ format: 'current' }]) + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([]) + }) + + it.each<[string, unknown]>([ + [ + 'a version 1 vault', + { + version: 1, + activeKey: 'workspace:w1', + bundles: { 'workspace:w1': tokens('user-1', 'w1') } + } + ], + ['the original single-token payload', tokens('user-1', 'w1')] + ])('reports %s moved to the current format', (_name, payload) => { + writeVault(payload) + + expect(loadGrant()).not.toBeNull() + + expect(eventsNamed(STORED_SESSION_RESTORED_EVENT)).toEqual([{ format: 'legacy' }]) + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([]) + }) + + it.each<[string, () => void, string]>([ + [ + 'cannot be decrypted', + () => fs.writeFileSync(vaultPath(), JSON.stringify({ version: 2 })), + 'undecryptable' + ], + [ + 'is not JSON', + () => fs.writeFileSync(vaultPath(), safeStorage.encryptString('not json')), + 'invalid' + ], + [ + 'has an unknown version', + () => writeVault({ version: 3, grant: tokens('user-1', 'w1') }), + 'invalid' + ], + [ + 'is a version 1 vault with no active session', + () => writeVault({ version: 1, activeKey: 'workspace:w2', bundles: {} }), + 'invalid' + ], + ['cannot be read', () => fs.mkdirSync(vaultPath()), 'unreadable'] + ])('reports once a stored session that %s', (_name, store, reason) => { + store() + + expect(loadGrant()).toBeNull() + expect(getAuthStatus()).toEqual({ signedIn: false }) + expect(loadGrant()).toBeNull() + + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([{ operation: 'load', reason }]) + expect(eventsNamed(STORED_SESSION_RESTORED_EVENT)).toEqual([]) + }) + + it('reads a session as signed out, and says it was unreadable, when the read throws no error object', () => { + vi.mocked(app.getPath).mockImplementationOnce(() => { + throw null + }) + onTestFinished(() => { + vi.mocked(app.getPath).mockReset() + }) + + expect(loadGrant()).toBeNull() + + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([ + { operation: 'load', reason: 'unreadable' } + ]) + }) + + it('reports nothing when no session is stored', () => { + expect(loadGrant()).toBeNull() + + expect(vi.mocked(mainTelemetry.emit)).not.toHaveBeenCalled() + }) + + it('reports nothing for a session saved and read back in the same process', () => { + saveGrant(tokens('user-1', 'w1')) + saveWorkspaceToken('w2', mintedToken('user-1', 'w2')) + + expect(loadGrant()).not.toBeNull() + + expect(vi.mocked(mainTelemetry.emit)).not.toHaveBeenCalled() + }) + + it('reports once a session kept only in memory because secure storage is unavailable', () => { + electronState.encryptionAvailable = false + + expect(saveGrant(tokens('user-1', 'w1'))).toBe(true) + saveWorkspaceToken('w2', mintedToken('user-1', 'w2')) + activateWorkspace('w2') + + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([ + { operation: 'save', reason: 'secure_storage_unavailable' } + ]) + }) + + it('reports once a session that could not be written to disk', () => { + const keychainLocked = (): never => { + throw new Error('keychain locked') + } + vi.mocked(safeStorage.encryptString) + .mockImplementationOnce(keychainLocked) + .mockImplementationOnce(keychainLocked) + onTestFinished(() => { + vi.mocked(safeStorage.encryptString).mockReset() + }) + const grant = tokens('user-1', 'w1') + + expect(saveGrant(grant)).toBe(false) + saveWorkspaceToken('w2', mintedToken('user-1', 'w2')) + + expect(fs.existsSync(vaultPath())).toBe(false) + expect(loadGrant()).toEqual(grant) + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([ + { operation: 'save', reason: 'write_failed' } + ]) + }) + + it('reports a legacy session that was read but could not be saved in the current format', () => { + const legacy = tokens('user-1', 'w1') + writeVault(legacy) + const stored = fs.readFileSync(vaultPath()) + vi.mocked(safeStorage.encryptString).mockImplementationOnce(() => { + throw new Error('keychain locked') + }) + onTestFinished(() => { + vi.mocked(safeStorage.encryptString).mockReset() + }) + + expect(loadGrant()).toEqual(legacy) + + expect(fs.readFileSync(vaultPath())).toEqual(stored) + expect(eventsNamed(STORED_SESSION_RESTORED_EVENT)).toEqual([{ format: 'legacy' }]) + expect(eventsNamed(STORED_SESSION_FAILED_EVENT)).toEqual([ + { operation: 'save', reason: 'write_failed' } + ]) + }) + }) }) diff --git a/src/main/cloud/tokenStore.ts b/src/main/cloud/tokenStore.ts index f16ea0740..dcc5ec17d 100644 --- a/src/main/cloud/tokenStore.ts +++ b/src/main/cloud/tokenStore.ts @@ -3,6 +3,7 @@ import { app, safeStorage } from 'electron' import * as fs from 'fs' import * as path from 'path' +import * as mainTelemetry from '../lib/telemetry' import { statusFromAccessToken, subjectOf, workspaceIdOf } from './claims' import { CLOUD_CONFIG, CLOUD_REFRESH_TOKEN_LIFETIME_MS } from './config' import type { AuthStatus, AuthTokens, Workspace } from './types' @@ -11,6 +12,15 @@ const AUTH_FILENAME = 'comfy-cloud-auth.bin' const REVOCATIONS_DIRNAME = 'comfy-cloud-revocations' const VAULT_VERSION = 2 +export const STORED_SESSION_RESTORED_EVENT = 'comfy.desktop.auth.stored_session_restored' +export const STORED_SESSION_FAILED_EVENT = 'comfy.desktop.auth.stored_session_failed' + +type StoredSessionFormat = 'current' | 'legacy' + +type StoredSessionFailure = + | { operation: 'load'; reason: 'unreadable' | 'undecryptable' | 'invalid' } + | { operation: 'save'; reason: 'write_failed' | 'secure_storage_unavailable' } + export interface GrantOrigin { issuer: string clientId: string @@ -44,6 +54,22 @@ let cachedVault: TokenVault | null = null const unsavedRevocations = new Map() let authFilePath: string | null = null let secureStorageUnavailable = false +const reportedStoredSessions = new Set() + +function reportOncePerProcess(event: string, context: mainTelemetry.TelemetryContext): void { + const report = JSON.stringify([event, context]) + if (reportedStoredSessions.has(report)) return + reportedStoredSessions.add(report) + mainTelemetry.emit(event, context) +} + +function reportRestored(format: StoredSessionFormat): void { + reportOncePerProcess(STORED_SESSION_RESTORED_EVENT, { format }) +} + +function reportStorageFailure(failure: StoredSessionFailure): void { + reportOncePerProcess(STORED_SESSION_FAILED_EVENT, failure) +} export function isSecureStorageUnavailable(): boolean { return secureStorageUnavailable @@ -132,6 +158,7 @@ function migratedVault(legacy: LegacyVault): TokenVault | null { function persistVault(vault: TokenVault): boolean { cachedVault = vault if (!encryptionAvailable()) { + reportStorageFailure({ operation: 'save', reason: 'secure_storage_unavailable' }) try { fs.rmSync(filePath(), { force: true }) return true @@ -143,29 +170,58 @@ function persistVault(vault: TokenVault): boolean { fs.writeFileSync(filePath(), safeStorage.encryptString(JSON.stringify(vault))) return true } catch { + reportStorageFailure({ operation: 'save', reason: 'write_failed' }) return false } } -function loadVault(): TokenVault | null { - if (cachedVault) return cachedVault - if (!encryptionAvailable()) return null +type StoredVault = + | { kind: 'none' } + | { kind: 'failed'; reason: Extract['reason'] } + | { kind: 'current'; vault: TokenVault } + | { kind: 'legacy'; vault: TokenVault } + +function readStoredVault(): StoredVault { + let sealed: Buffer try { - const parsed: unknown = JSON.parse(safeStorage.decryptString(fs.readFileSync(filePath()))) - if (isVault(parsed)) { - cachedVault = parsed - return cachedVault - } + sealed = fs.readFileSync(filePath()) + } catch (error) { + const missing = (error as NodeJS.ErrnoException | null)?.code === 'ENOENT' + return missing ? { kind: 'none' } : { kind: 'failed', reason: 'unreadable' } + } + let opened: string + try { + opened = safeStorage.decryptString(sealed) + } catch { + return { kind: 'failed', reason: 'undecryptable' } + } + try { + const parsed: unknown = JSON.parse(opened) + if (isVault(parsed)) return { kind: 'current', vault: parsed } const migrated = isLegacyVault(parsed) ? migratedVault(parsed) : isTokens(parsed) ? vaultFor(parsed) : null - if (migrated) persistVault(migrated) - return migrated + return migrated ? { kind: 'legacy', vault: migrated } : { kind: 'failed', reason: 'invalid' } } catch { + return { kind: 'failed', reason: 'invalid' } + } +} + +function loadVault(): TokenVault | null { + if (cachedVault) return cachedVault + if (!encryptionAvailable()) return null + const stored = readStoredVault() + if (stored.kind === 'none') return null + if (stored.kind === 'failed') { + reportStorageFailure({ operation: 'load', reason: stored.reason }) return null } + if (stored.kind === 'current') cachedVault = stored.vault + else persistVault(stored.vault) + reportRestored(stored.kind) + return stored.vault } function holdsToken(vault: TokenVault, workspaceId: string | null): boolean { @@ -294,11 +350,14 @@ export function endGrant(expectedAccessToken: string): void { persistVault(vault) } -export function clearTokens(): void { +export function clearTokens(): boolean { cachedVault = null try { fs.rmSync(filePath(), { force: true }) - } catch {} + return true + } catch { + return false + } } function revocationsDir(): string { @@ -387,8 +446,10 @@ export function loadPendingRevocations(): PendingRevocations { return { pending: [...saved.pending, ...unsaved], complete: saved.complete } } -export function addPendingRevocation(revocation: PendingRevocation): void { - if (!writeRevocation(revocation)) unsavedRevocations.set(revocation.refreshToken, revocation) +export function addPendingRevocation(revocation: PendingRevocation): boolean { + const saved = writeRevocation(revocation) + if (!saved) unsavedRevocations.set(revocation.refreshToken, revocation) + return saved } export function removePendingRevocation(refreshToken: string): boolean { @@ -434,4 +495,5 @@ export function _resetForTest(): void { unsavedRevocations.clear() authFilePath = null secureStorageUnavailable = false + reportedStoredSessions.clear() } diff --git a/src/main/cloud/workspaceToken.ts b/src/main/cloud/workspaceToken.ts index f8ef33271..5d2c86ff9 100644 --- a/src/main/cloud/workspaceToken.ts +++ b/src/main/cloud/workspaceToken.ts @@ -1,4 +1,4 @@ -import { lifetimeSecondsOf } from './claims' +import { lifetimeSecondsOf, subjectOf, workspaceIdOf } from './claims' import { CLOUD_CONFIG } from './config' @@ -28,6 +28,64 @@ export type WorkspaceMintOutcome = | { kind: 'refused' } | { kind: 'retryable'; retryAfterSeconds?: number } +const REPORTABLE_ERROR_CODES: ReadonlySet = new Set([ + 'account_outside_organization', + 'auth_type_not_allowed', + 'insufficient_scope', + 'internal_error', + 'invalid_token', + 'not_found', + 'oauth_derived_mint_refused', + 'oauth_grant_ended', + 'oauth_token_mint_refused', + 'oauth_workspace_mint_disabled', + 'partner_node_job_ended', + 'partner_node_renew_disabled', + 'partner_node_requires_sign_in', + 'partner_node_session_ended', + 'partner_node_session_expired', + 'partner_node_session_revoked', + 'partner_node_token_disabled', + 'personal_workspace_disabled', + 'rate_limited', + 'resource_invalid', + 'session_expired', + 'sso_required', + 'token_revoked', + 'unauthorized' +]) +const UNLISTED_ERROR_CODE = 'other' + +function reportableErrorCode(code: string): string { + const known = code.toLowerCase() + return REPORTABLE_ERROR_CODES.has(known) ? known : UNLISTED_ERROR_CODE +} + +export type MintFailureReport = + | { reason: 'network' } + | { reason: 'http'; http_status: number; error_code?: string } + +export function mintFailureReport( + failure: Extract +): MintFailureReport { + if (failure.status === null) return { reason: 'network' } + const { status, code } = failure + return code + ? { reason: 'http', http_status: status, error_code: reportableErrorCode(code) } + : { reason: 'http', http_status: status } +} + +export type MintedTokenProblem = 'workspace_mismatch' | 'subject_mismatch' + +export function mintedTokenProblem( + token: string, + workspaceId: string, + subject: string | null +): MintedTokenProblem | null { + if (workspaceIdOf(token) !== workspaceId) return 'workspace_mismatch' + return subjectOf(token) === subject ? null : 'subject_mismatch' +} + export function classifyMint(mint: WorkspaceTokenMint): WorkspaceMintOutcome { if (mint.kind === 'minted') return mint const { status, code, retryAfterSeconds } = mint diff --git a/src/main/lib/embeddedSession.test.ts b/src/main/lib/embeddedSession.test.ts index 0b3b19e12..61208ccd9 100644 --- a/src/main/lib/embeddedSession.test.ts +++ b/src/main/lib/embeddedSession.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ entries: [] as unknown[] })) +vi.mock('./telemetry', () => ({ emit: vi.fn() })) vi.mock('./embeddedSessionFlag', () => ({ isEmbeddedSessionEnabled: async () => (mocks.flagRead ? mocks.flagRead : mocks.enabled) })) @@ -47,6 +48,8 @@ vi.mock('../host/registry', () => ({ import { EMBEDDED_SESSION_CHANNELS, + EMBEDDED_SESSION_DISABLED_EVENT, + _resetForTest as resetEmbeddedSession, partnerNodeTokenForSender, workspaceTokenForSender, broadcastEmbeddedSessionChanged, @@ -55,6 +58,7 @@ import { } from './embeddedSession' import type { EmbeddedSessionSender } from './embeddedSession' import { CLOUD_CONFIG } from '../cloud/config' +import * as mainTelemetry from './telemetry' import { _resetForTest as resetViewGrants } from './viewWorkspaceGrants' function jwt(claims: Record): string { @@ -116,8 +120,16 @@ beforeEach(() => { })) mocks.current = 'ws-1' resetViewGrants() + resetEmbeddedSession() + vi.mocked(mainTelemetry.emit).mockClear() }) +function disabledEvents(): unknown[] { + return vi + .mocked(mainTelemetry.emit) + .mock.calls.filter(([event]) => event === EMBEDDED_SESSION_DISABLED_EVENT) +} + describe('embeddedSession', () => { it.each([ ['local ComfyUI on 127.0.0.1', 'http://127.0.0.1:8000', {}], @@ -307,6 +319,49 @@ describe('embeddedSession', () => { expect(mocks.getGrantAccessToken).not.toHaveBeenCalled() }) + it('reports once per process that the ops flag left a local view without the session', async () => { + mocks.enabled = false + const first = view('http://127.0.0.1:8000') + const second = view('http://localhost:8188') + + await stateForSender(first.event) + await stateForSender(second.event) + await workspaceTokenForSender(second.event, 'ws-1') + + expect(disabledEvents()).toEqual([[EMBEDDED_SESSION_DISABLED_EVENT]]) + }) + + it.each<[string, (event: EmbeddedSessionSender) => Promise]>([ + ['a workspace token', (event) => workspaceTokenForSender(event, 'ws-1')], + ['a partner-node token', (event) => partnerNodeTokenForSender(event, 'ws-1')] + ])('reports a local view that only asks for %s while the ops flag is off', async (_name, ask) => { + mocks.enabled = false + const { event } = view('http://127.0.0.1:8000') + + await expect(ask(event)).resolves.toBeNull() + + expect(disabledEvents()).toEqual([[EMBEDDED_SESSION_DISABLED_EVENT]]) + }) + + it('reports nothing for a change announced while the ops flag is off and no view asked', async () => { + mocks.enabled = false + view('http://127.0.0.1:8000') + + await broadcastEmbeddedSessionChanged() + + expect(disabledEvents()).toEqual([]) + }) + + it('reports nothing while the ops flag is on, or for a view it does not trust', async () => { + const local = view('http://127.0.0.1:8000') + const remote = view('http://192.168.1.20:8188') + + await stateForSender(local.event) + await expect(stateForSender(remote.event)).resolves.toEqual({ status: 'disabled' }) + + expect(disabledEvents()).toEqual([]) + }) + it('keeps a view signed in on the grant itself when Cloud did not grant the mint scope', async () => { const grant = jwt({ sub: 'user-1', workspace_id: 'ws-1', scope: 'comfy-cloud:user:read' }) mocks.getGrantAccessToken.mockResolvedValue(grant) diff --git a/src/main/lib/embeddedSession.ts b/src/main/lib/embeddedSession.ts index 358ccd562..deafcb7fe 100644 --- a/src/main/lib/embeddedSession.ts +++ b/src/main/lib/embeddedSession.ts @@ -7,8 +7,11 @@ import type { ComfyWindowEntry } from '../host/registry' import type { WorkspaceSwitchOutcome } from '../cloud/types' import type { ComfyDesktop2AuthState } from '../../types/comfyDesktopBridge' import { isEmbeddedSessionEnabled } from './embeddedSessionFlag' +import * as mainTelemetry from './telemetry' import { getViewWorkspaceGrants } from './viewWorkspaceGrants' +export const EMBEDDED_SESSION_DISABLED_EVENT = 'comfy.desktop.auth.embedded_session_disabled' + export const EMBEDDED_SESSION_CHANNELS = { getState: 'desktop2-auth:get-state', getWorkspaceToken: 'desktop2-auth:get-workspace-token', @@ -62,8 +65,24 @@ export function isTrustedSender({ sender, senderFrame }: EmbeddedSessionSender): return servesEntry(entry, senderFrame.url) } -async function sessionState(): Promise { - if (!(await isEmbeddedSessionEnabled())) return DISABLED +let disabledReported = false + +function reportDisabledOncePerProcess(): void { + if (disabledReported) return + disabledReported = true + mainTelemetry.emit(EMBEDDED_SESSION_DISABLED_EVENT) +} + +async function enabledForLocalView(): Promise { + const enabled = await isEmbeddedSessionEnabled() + if (!enabled) reportDisabledOncePerProcess() + return enabled +} + +async function sessionState( + enabled: () => Promise = isEmbeddedSessionEnabled +): Promise { + if (!(await enabled())) return DISABLED const session = getCloudSession() const grant = await session.getGrantAccessToken() const identity = grant ? identityOf(grant) : null @@ -81,7 +100,7 @@ async function sessionState(): Promise { export async function stateForSender( event: EmbeddedSessionSender ): Promise { - return isTrustedSender(event) ? sessionState() : DISABLED + return isTrustedSender(event) ? sessionState(enabledForLocalView) : DISABLED } export async function workspaceTokenForSender( @@ -90,7 +109,7 @@ export async function workspaceTokenForSender( ): Promise { if (typeof workspaceId !== 'string' || workspaceId === '') return null if (!isTrustedSender(event)) return null - return getViewWorkspaceGrants().tokenFor(event.sender, workspaceId, isEmbeddedSessionEnabled) + return getViewWorkspaceGrants().tokenFor(event.sender, workspaceId, enabledForLocalView) } export async function partnerNodeTokenForSender( @@ -102,7 +121,7 @@ export async function partnerNodeTokenForSender( return getViewWorkspaceGrants().partnerNodeTokenFor( event.sender, workspaceId, - isEmbeddedSessionEnabled + enabledForLocalView ) } @@ -133,3 +152,7 @@ export async function broadcastEmbeddedSessionChanged(): Promise { contents.send(EMBEDDED_SESSION_CHANNELS.changed, state) } } + +export function _resetForTest(): void { + disabledReported = false +} diff --git a/src/main/lib/ipc/switchCloudWorkspace.session.test.ts b/src/main/lib/ipc/switchCloudWorkspace.session.test.ts index eedb38251..24091848e 100644 --- a/src/main/lib/ipc/switchCloudWorkspace.session.test.ts +++ b/src/main/lib/ipc/switchCloudWorkspace.session.test.ts @@ -26,7 +26,7 @@ vi.mock('electron', () => ({ vi.mock('../../cloud/oauth', () => ({ signIn: vi.fn(), refresh: vi.fn(), - revoke: vi.fn(async () => 'settled'), + revoke: vi.fn(async () => 'revoked'), openAuthorizePage: vi.fn(async () => {}) })) vi.mock('../../devplatform/session', () => ({ diff --git a/src/main/lib/viewWorkspaceGrants.test.ts b/src/main/lib/viewWorkspaceGrants.test.ts index 860b06861..362663583 100644 --- a/src/main/lib/viewWorkspaceGrants.test.ts +++ b/src/main/lib/viewWorkspaceGrants.test.ts @@ -14,8 +14,11 @@ import { CLOUD_CONFIG, WORKSPACE_MINT_SCOPE } from '../cloud/config' import type { WorkspaceTokenMint } from '../cloud/workspaceToken' import * as mainTelemetry from './telemetry' import { + PARTNER_NODE_TOKEN_FAILED_EVENT, + PARTNER_NODE_TOKEN_ISSUED_EVENT, VIEW_TOKEN_MINT_FAILED_EVENT, VIEW_TOKEN_MINTED_EVENT, + VIEW_TOKEN_REFUSED_EVENT, ViewWorkspaceGrants } from './viewWorkspaceGrants' @@ -109,6 +112,22 @@ describe('ViewWorkspaceGrants', () => { it.each([ [{ kind: 'failed', status: 403 } as const, { reason: 'http', http_status: 403 }], + [ + { kind: 'failed', status: 401, code: 'oauth_grant_ended' } as const, + { reason: 'http', http_status: 401, error_code: 'oauth_grant_ended' } + ], + [ + { kind: 'failed', status: 401, code: 'UNAUTHORIZED' } as const, + { reason: 'http', http_status: 401, error_code: 'unauthorized' } + ], + [ + { kind: 'failed', status: 500, code: 'Internal error for a@example.com' } as const, + { reason: 'http', http_status: 500, error_code: 'other' } + ], + [ + { kind: 'failed', status: 403, code: 'ws_3f9c2a71' } as const, + { reason: 'http', http_status: 403, error_code: 'other' } + ], [{ kind: 'failed', status: null } as const, { reason: 'network' }] ])('reports a failed mint %j', async (result, properties) => { mint.mockResolvedValueOnce(result) @@ -118,6 +137,93 @@ describe('ViewWorkspaceGrants', () => { expect(mainTelemetry.emit).toHaveBeenCalledWith(VIEW_TOKEN_MINT_FAILED_EVENT, properties) }) + it.each<[string, () => string, string]>([ + ['for another workspace', () => mintedToken('ws-2'), 'workspace_mismatch'], + ['for another account', () => jwt({ sub: 'user-2', workspace_id: 'ws-1' }), 'subject_mismatch'], + ['that can mint', () => accessToken('ws-1'), 'mint_scope'] + ])( + 'reports a token Cloud minted %s as a failed mint, once for the views that shared it', + async (_name, token, problem) => { + mint.mockResolvedValueOnce({ + kind: 'minted', + token: token(), + expiresAt: Date.now() + 900_000 + }) + + await expect( + Promise.all([grants.tokenFor(viewA, 'ws-1'), grants.tokenFor(viewB, 'ws-1')]) + ).resolves.toEqual([null, null]) + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [VIEW_TOKEN_MINT_FAILED_EVENT, { reason: 'unusable_token', problem }] + ]) + } + ) + + it('reports once a view that asks for the active workspace it was never let into', async () => { + grants.allowView(viewA, 'ws-2') + signInAt('ws-2') + + await expect(grants.tokenFor(viewB, 'ws-2')).resolves.toBeNull() + await expect(grants.partnerNodeTokenFor(viewB, 'ws-2')).resolves.toBeNull() + await expect(grants.tokenFor(viewB, 'ws-2')).resolves.toBeNull() + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([[VIEW_TOKEN_REFUSED_EVENT]]) + }) + + it('reports each view that was refused the active workspace', async () => { + const viewC = { id: 'c' } as unknown as WebContents + grants.allowView(viewA, 'ws-2') + signInAt('ws-2') + + await grants.tokenFor(viewB, 'ws-2') + await grants.tokenFor(viewC, 'ws-2') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [VIEW_TOKEN_REFUSED_EVENT], + [VIEW_TOKEN_REFUSED_EVENT] + ]) + }) + + it('reports a view refused again after it was handed a token in between', async () => { + grants.allowView(viewA, 'ws-2') + signInAt('ws-2') + await grants.tokenFor(viewB, 'ws-2') + grants.allowDesktop('ws-2') + await expect(grants.tokenFor(viewB, 'ws-2')).resolves.toBe(mintedToken('ws-2')) + grants.allowView(viewA, 'ws-3') + signInAt('ws-3') + + await expect(grants.tokenFor(viewB, 'ws-3')).resolves.toBeNull() + + expect( + vi + .mocked(mainTelemetry.emit) + .mock.calls.filter(([event]) => event === VIEW_TOKEN_REFUSED_EVENT) + ).toHaveLength(2) + }) + + it('reports a refused view again once every view was shut out and it is still refused', async () => { + grants.allowView(viewA, 'ws-2') + signInAt('ws-2') + await grants.tokenFor(viewB, 'ws-2') + + grants.endAll() + await grants.tokenFor(viewB, 'ws-2') + await grants.tokenFor(viewB, 'ws-2') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [VIEW_TOKEN_REFUSED_EVENT], + [VIEW_TOKEN_REFUSED_EVENT] + ]) + }) + + it('reports nothing for a view that asks for a workspace Desktop is not in', async () => { + await expect(grants.tokenFor(viewA, 'ws-2')).resolves.toBeNull() + + expect(vi.mocked(mainTelemetry.emit)).not.toHaveBeenCalled() + }) + it('hands a view the minted token for the workspace Desktop started in, never the access token', async () => { const token = await grants.tokenFor(viewA, 'ws-1') @@ -474,6 +580,162 @@ describe('ViewWorkspaceGrants partner-node tokens', () => { expect(partnerMint).toHaveBeenLastCalledWith(accessToken('ws-1'), 'ws-2') }) + it('reports the partner-node token it minted and the one it renewed', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + vi.setSystemTime(Date.now() + 31 * 60_000) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation: 'mint' }], + [PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation: 'renew' }] + ]) + }) + + it('reports a partner-node mint Cloud refused, with the code Cloud gave', async () => { + partnerMint.mockResolvedValueOnce({ kind: 'failed', status: 403, code: 'insufficient_scope' }) + + await expect(grants.partnerNodeTokenFor(viewA, 'ws-1')).resolves.toBeNull() + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [ + PARTNER_NODE_TOKEN_FAILED_EVENT, + { operation: 'mint', reason: 'http', http_status: 403, error_code: 'insufficient_scope' } + ] + ]) + }) + + it('reports a partner-node renewal that could not reach Cloud', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + vi.mocked(mainTelemetry.emit).mockClear() + partnerRenew.mockResolvedValueOnce({ kind: 'failed', status: null }) + vi.setSystemTime(Date.now() + 31 * 60_000) + + await grants.partnerNodeTokenFor(viewA, 'ws-1') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [PARTNER_NODE_TOKEN_FAILED_EVENT, { operation: 'renew', reason: 'network' }] + ]) + }) + + it('reports a refused renewal and the new session that replaced it', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + vi.mocked(mainTelemetry.emit).mockClear() + partnerRenew.mockResolvedValueOnce({ + kind: 'failed', + status: 403, + code: 'partner_node_session_ended' + }) + partnerMint.mockResolvedValueOnce(partnerMinted(partnerToken('ws-1', 'sid-2'))) + vi.setSystemTime(Date.now() + 31 * 60_000) + + await grants.partnerNodeTokenFor(viewA, 'ws-1') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [ + PARTNER_NODE_TOKEN_FAILED_EVENT, + { + operation: 'renew', + reason: 'http', + http_status: 403, + error_code: 'partner_node_session_ended' + } + ], + [PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation: 'mint' }] + ]) + }) + + it.each<[string, () => WorkspaceTokenMint, string]>([ + ['for another workspace', () => partnerMinted(partnerToken('ws-2')), 'workspace_mismatch'], + [ + 'for another account', + () => partnerMinted(jwt({ sub: 'user-2', workspace_id: 'ws-1', aud: 'comfy-partner-node' })), + 'subject_mismatch' + ], + [ + 'that expires within a minute', + () => partnerMinted(partnerToken('ws-1'), 30_000), + 'expires_too_soon' + ] + ])( + 'reports a partner-node token Cloud minted %s as a failed mint', + async (_name, issued, problem) => { + partnerMint.mockResolvedValueOnce(issued()) + + await expect(grants.partnerNodeTokenFor(viewA, 'ws-1')).resolves.toBeNull() + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [PARTNER_NODE_TOKEN_FAILED_EVENT, { operation: 'mint', reason: 'unusable_token', problem }] + ]) + } + ) + + it.each<[string, () => WorkspaceTokenMint, string]>([ + [ + 'for another workspace', + () => partnerMinted(partnerToken('ws-2', 'sid-1', 1)), + 'workspace_mismatch' + ], + [ + 'for another account', + () => partnerMinted(jwt({ sub: 'user-2', workspace_id: 'ws-1', aud: 'comfy-partner-node' })), + 'subject_mismatch' + ], + [ + 'for another workspace that also leaves under an hour', + () => partnerMinted(partnerToken('ws-2', 'sid-1', 1), 40 * 60_000), + 'expires_too_soon' + ] + ])('reports a renewal Cloud answered %s as failed', async (_name, renewed, problem) => { + vi.useFakeTimers({ toFake: ['Date'] }) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + vi.mocked(mainTelemetry.emit).mockClear() + partnerRenew.mockImplementationOnce(async () => renewed()) + vi.setSystemTime(Date.now() + 31 * 60_000) + + await grants.partnerNodeTokenFor(viewA, 'ws-1') + + expect(vi.mocked(mainTelemetry.emit).mock.calls[0]).toEqual([ + PARTNER_NODE_TOKEN_FAILED_EVENT, + { operation: 'renew', reason: 'unusable_token', problem } + ]) + }) + + it('reports a renewal that left under an hour as failed, and the new session that replaced it', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + vi.mocked(mainTelemetry.emit).mockClear() + partnerRenew.mockResolvedValueOnce(partnerMinted(partnerToken('ws-1', 'sid-1', 1), 40 * 60_000)) + partnerMint.mockResolvedValueOnce(partnerMinted(partnerToken('ws-1', 'sid-2'))) + vi.setSystemTime(Date.now() + 31 * 60_000) + + await expect(grants.partnerNodeTokenFor(viewA, 'ws-1')).resolves.toBe( + partnerToken('ws-1', 'sid-2') + ) + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [ + PARTNER_NODE_TOKEN_FAILED_EVENT, + { operation: 'renew', reason: 'unusable_token', problem: 'expires_too_soon' } + ], + [PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation: 'mint' }] + ]) + }) + + it('reports one partner-node mint for views that shared it and none for a reused token', async () => { + await Promise.all([ + grants.partnerNodeTokenFor(viewA, 'ws-1'), + grants.partnerNodeTokenFor(viewB, 'ws-1') + ]) + await grants.partnerNodeTokenFor(viewA, 'ws-1') + + expect(vi.mocked(mainTelemetry.emit).mock.calls).toEqual([ + [PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation: 'mint' }] + ]) + }) + it('mints once for concurrent requests and reuses the token while it has over an hour left', async () => { await Promise.all([ grants.partnerNodeTokenFor(viewA, 'ws-1'), diff --git a/src/main/lib/viewWorkspaceGrants.ts b/src/main/lib/viewWorkspaceGrants.ts index 05f859ac4..0a1dc09a9 100644 --- a/src/main/lib/viewWorkspaceGrants.ts +++ b/src/main/lib/viewWorkspaceGrants.ts @@ -3,11 +3,17 @@ import type { WebContents } from 'electron' import { canMintWorkspaceTokens, grantFamilyOf, subjectOf, workspaceIdOf } from '../cloud/claims' import { classifyMint, + mintedTokenProblem, + mintFailureReport, mintPartnerNodeToken, mintWorkspaceToken, renewPartnerNodeToken } from '../cloud/workspaceToken' -import type { WorkspaceTokenMint } from '../cloud/workspaceToken' +import type { + MintedTokenProblem, + MintFailureReport, + WorkspaceTokenMint +} from '../cloud/workspaceToken' import { getCloudSession, rejectAccessToken } from '../devplatform/session' import * as mainTelemetry from './telemetry' @@ -18,16 +24,53 @@ const PARTNER_NODE_MAX_RETRY_MS = 5 * 60_000 export const VIEW_TOKEN_MINTED_EVENT = 'comfy.desktop.auth.view_token_minted' export const VIEW_TOKEN_MINT_FAILED_EVENT = 'comfy.desktop.auth.view_token_mint_failed' +export const PARTNER_NODE_TOKEN_ISSUED_EVENT = 'comfy.desktop.auth.partner_node_token_issued' +export const PARTNER_NODE_TOKEN_FAILED_EVENT = 'comfy.desktop.auth.partner_node_token_failed' +export const VIEW_TOKEN_REFUSED_EVENT = 'comfy.desktop.auth.view_token_refused' -function reportMint(result: WorkspaceTokenMint): void { - if (result.kind === 'minted') { - mainTelemetry.emit(VIEW_TOKEN_MINTED_EVENT) - return - } - mainTelemetry.emit( - VIEW_TOKEN_MINT_FAILED_EVENT, - result.status === null ? { reason: 'network' } : { reason: 'http', http_status: result.status } - ) +type IssuedToken = Extract + +type TokenProblem = MintedTokenProblem | 'mint_scope' | 'expires_too_soon' + +type MintFailure = MintFailureReport | { reason: 'unusable_token'; problem: TokenProblem } + +function workspaceTokenProblem( + token: string, + workspaceId: string, + subject: string | null +): TokenProblem | null { + const problem = mintedTokenProblem(token, workspaceId, subject) + if (problem) return problem + return canMintWorkspaceTokens(token) ? 'mint_scope' : null +} + +function partnerNodeTokenProblem( + issued: IssuedToken, + workspaceId: string, + subject: string | null, + shortestLifetimeMs: number +): TokenProblem | null { + if (issued.expiresAt - shortestLifetimeMs <= Date.now()) return 'expires_too_soon' + return mintedTokenProblem(issued.token, workspaceId, subject) +} + +function mintFailureOf( + result: WorkspaceTokenMint, + problemOf: (issued: IssuedToken) => TokenProblem | null +): MintFailure | null { + if (result.kind === 'failed') return mintFailureReport(result) + const problem = problemOf(result) + return problem ? { reason: 'unusable_token', problem } : null +} + +function reportMint(failure: MintFailure | null): void { + if (failure) mainTelemetry.emit(VIEW_TOKEN_MINT_FAILED_EVENT, failure) + else mainTelemetry.emit(VIEW_TOKEN_MINTED_EVENT) +} + +function reportPartnerNodeToken(operation: 'mint' | 'renew', failure: MintFailure | null): void { + if (failure) mainTelemetry.emit(PARTNER_NODE_TOKEN_FAILED_EVENT, { operation, ...failure }) + else mainTelemetry.emit(PARTNER_NODE_TOKEN_ISSUED_EVENT, { operation }) } export interface GrantSession { @@ -95,6 +138,7 @@ export class ViewWorkspaceGrants { private seeded = false private desktopAllowed = new Set() private viewAllowed = new WeakMap>() + private refusedViews = new WeakSet() private minted = new Map() private partnerNodeMinted = new Map() private partnerNodeRetryAt = new Map() @@ -124,6 +168,7 @@ export class ViewWorkspaceGrants { this.seeded = true this.desktopAllowed = new Set() this.viewAllowed = new WeakMap() + this.refusedViews = new WeakSet() this.minted = new Map() this.partnerNodeMinted = new Map() this.partnerNodeRetryAt = new Map() @@ -150,12 +195,23 @@ export class ViewWorkspaceGrants { gate: Gate, read: (fence: MintFence) => Promise ): Promise { - if (!this.mayHold(view, workspaceId)) return null + if (!this.mayHold(view, workspaceId)) { + this.reportRefusal(view, workspaceId) + return null + } const fence = this.currentFence() if (!(await gate()) || !this.isCurrent(fence)) return null const held = await read(fence) if (!held || !this.isCurrent(fence) || !this.mayHold(view, workspaceId)) return null - return this.session.currentSubject() === held.subject ? held.token : null + if (this.session.currentSubject() !== held.subject) return null + this.refusedViews.delete(view) + return held.token + } + + private reportRefusal(view: WebContents, workspaceId: string): void { + if (this.session.currentWorkspaceId() !== workspaceId || this.refusedViews.has(view)) return + this.refusedViews.add(view) + mainTelemetry.emit(VIEW_TOKEN_REFUSED_EVENT) } private currentFence(): MintFence { @@ -213,7 +269,9 @@ export class ViewWorkspaceGrants { if (!allowed || !this.isCurrent(fence)) return null const result = await this.once(['workspace', key], fence, () => this.mint(accessToken, workspaceId).then((minted) => { - reportMint(minted) + reportMint( + mintFailureOf(minted, ({ token }) => workspaceTokenProblem(token, workspaceId, subject)) + ) this.endAnEndedGrant(accessToken, minted) if (minted.kind === 'failed' && minted.status === 401) this.onUnauthorized(accessToken) return minted @@ -221,12 +279,7 @@ export class ViewWorkspaceGrants { ) if (!this.isCurrent(fence) || this.session.currentSubject() !== subject) return null if (result.kind !== 'minted') return null - if ( - workspaceIdOf(result.token) !== workspaceId || - subjectOf(result.token) !== subject || - canMintWorkspaceTokens(result.token) - ) - return null + if (workspaceTokenProblem(result.token, workspaceId, subject)) return null this.minted.set(key, { token: result.token, expiresAt: result.expiresAt }) return { token: result.token, subject } } @@ -251,9 +304,7 @@ export class ViewWorkspaceGrants { ) const usable = result.kind === 'minted' && - workspaceIdOf(result.token) === workspaceId && - subjectOf(result.token) === subject && - result.expiresAt - MINT_REFRESH_SKEW_MS > Date.now() + !partnerNodeTokenProblem(result, workspaceId, subject, MINT_REFRESH_SKEW_MS) if (!usable && fence.epoch === this.epoch) { const retryAfterMs = result.kind === 'failed' && result.retryAfterSeconds !== undefined @@ -289,6 +340,17 @@ export class ViewWorkspaceGrants { fence: MintFence ): Promise { const renewed = await this.partnerNode.renew(partnerNodeToken) + reportPartnerNodeToken( + 'renew', + mintFailureOf(renewed, (issued) => + partnerNodeTokenProblem( + issued, + workspaceId, + subjectOf(accessToken), + PARTNER_NODE_RENEW_BEFORE_MS + ) + ) + ) if (renewed.kind === 'minted' && renewed.expiresAt - PARTNER_NODE_RENEW_BEFORE_MS > Date.now()) return renewed if (renewed.kind === 'failed' && !isRetryableFailure(renewed)) @@ -302,6 +364,12 @@ export class ViewWorkspaceGrants { workspaceId: string ): Promise { const minted = await this.partnerNode.mint(accessToken, workspaceId) + reportPartnerNodeToken( + 'mint', + mintFailureOf(minted, (issued) => + partnerNodeTokenProblem(issued, workspaceId, subjectOf(accessToken), MINT_REFRESH_SKEW_MS) + ) + ) this.endAnEndedGrant(accessToken, minted) return minted } diff --git a/src/shared/datadogMirroredEvents.test.ts b/src/shared/datadogMirroredEvents.test.ts index 1d2971bbb..7e9b5bd6a 100644 --- a/src/shared/datadogMirroredEvents.test.ts +++ b/src/shared/datadogMirroredEvents.test.ts @@ -34,6 +34,18 @@ describe('isDatadogMirroredEvent', () => { it.each([ ['comfy.desktop.auth.host_sign_in', true], ['comfy.desktop.auth.sign_in_failed', true], + ['comfy.desktop.auth.signed_out', true], + ['comfy.desktop.auth.expired_grant_rejected', true], + ['comfy.desktop.auth.grant_revocation', true], + ['comfy.desktop.auth.partner_node_token_failed', true], + ['comfy.desktop.auth.partner_node_token_issued', false], + ['comfy.desktop.auth.session_sign_in', true], + ['comfy.desktop.auth.grant_refresh_failed', true], + ['comfy.desktop.auth.workspace_token_mint_failed', true], + ['comfy.desktop.auth.stored_session_failed', true], + ['comfy.desktop.auth.stored_session_restored', false], + ['comfy.desktop.auth.embedded_session_disabled', true], + ['comfy.desktop.auth.view_token_refused', true], ['comfy.desktop.auth.sign_in_started', false] ])('mirrors %s: %s', (name, mirrored) => { expect(isDatadogMirroredEvent(name)).toBe(mirrored) diff --git a/src/shared/datadogMirroredEvents.ts b/src/shared/datadogMirroredEvents.ts index 7dd45fe03..66372343d 100644 --- a/src/shared/datadogMirroredEvents.ts +++ b/src/shared/datadogMirroredEvents.ts @@ -3,6 +3,16 @@ export const DATADOG_MIRRORED_EVENT_NAMES: ReadonlySet = new Set([ 'comfy.desktop.auth.workspace_switch_browser', 'comfy.desktop.auth.workspace_returned_to_anchor', 'comfy.desktop.auth.mint_scope_not_granted', + 'comfy.desktop.auth.signed_out', + 'comfy.desktop.auth.expired_grant_rejected', + 'comfy.desktop.auth.grant_revocation', + 'comfy.desktop.auth.partner_node_token_failed', + 'comfy.desktop.auth.session_sign_in', + 'comfy.desktop.auth.grant_refresh_failed', + 'comfy.desktop.auth.workspace_token_mint_failed', + 'comfy.desktop.auth.stored_session_failed', + 'comfy.desktop.auth.embedded_session_disabled', + 'comfy.desktop.auth.view_token_refused', 'comfy.desktop.execution.error', 'comfy.desktop.comfyui.exited',