diff --git a/browser_tests/fixtures/agentConsentTelemetryFixture.ts b/browser_tests/fixtures/agentConsentTelemetryFixture.ts new file mode 100644 index 00000000000..b590bcce285 --- /dev/null +++ b/browser_tests/fixtures/agentConsentTelemetryFixture.ts @@ -0,0 +1,88 @@ +import type { RemoteConfig } from '@/platform/remoteConfig/types' + +import { agentConsentTest } from '@e2e/fixtures/agentConsentFixture' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +export interface CapturedTelemetryEvent { + event: string + properties: Record +} + +/** + * Reads the consent surface's telemetry from inside a real browser. + * + * It captures at the **host bridge**, not over the network. `main.ts` registers + * the host sink after the cloud one when `window.__comfyDesktop2.Telemetry` + * exists, and `setTelemetryRegistry` keeps the last registry - so every + * `useTelemetry()` call in the app lands here synchronously, with no batching, + * no compression and no third-party host to decode. Cloud plus a host bridge is + * a configuration that exists in production (`main.ts` has an + * `isCloud && hasHostTelemetryBridge` branch of its own), not one fabricated for + * the test. + * + * What that does and does not prove: the browser proves which events a real + * interaction produces and with which properties, through the same + * `TelemetryRegistry` the PostHog provider is registered on. The mapping from a + * registry call to a PostHog capture is asserted by + * `PostHogTelemetryProvider.test.ts`, not here. + * + * PostHog still initializes, because the agent feature flag is bootstrapped + * through it; its ingest host is routed so nothing leaves the browser and the + * network-isolation fixture stays satisfied. + */ +export const agentConsentTelemetryTest = agentConsentTest.extend<{ + consentTelemetry: CapturedTelemetryEvent[] +}>({ + consentTelemetry: async ({ agentFlagEnabled: _agentFlagEnabled }, use) => { + await use([]) + }, + page: async ({ page, consentTelemetry }, use) => { + await page.exposeFunction( + '__captureHostTelemetry', + (captured: CapturedTelemetryEvent) => { + consentTelemetry.push(captured) + } + ) + await page.addInitScript(() => { + // Declared before the app boots so `hasHostTelemetryBridge` sees it. + Object.assign(window, { + __comfyDesktop2: { + isRemote: () => false, + Telemetry: { + capture: (event: string, properties: Record) => { + void ( + window as unknown as { + __captureHostTelemetry: ( + captured: CapturedTelemetryEvent + ) => Promise + } + ).__captureHostTelemetry({ event, properties }) + } + } + } + }) + }) + // Layered over the fixture's own `/api/features`, which the later route + // wins. Keep the authenticated allowlist grant from the base fixture: that + // grant is read directly from remote config once auth has loaded, not from + // PostHog's bootstrap flags. `enable_telemetry` gates the host sink. + const features: RemoteConfig = { + 'agent-in-app-experience': true, + enable_telemetry: true, + posthog_project_token: 'phc_e2e_agent_consent_outcome', + posthog_config: { + advanced_disable_flags: true, + bootstrap: { + featureFlags: { 'agent-in-app-experience': true } + } + } + } + await page.route('**/api/features', (route) => + route.fulfill(jsonRoute(features)) + ) + await page.route('**://t.comfy.org/**', (route) => + route.fulfill(jsonRoute({ status: 1 })) + ) + await use(page) + } +}) diff --git a/browser_tests/tests/agent/agentConsentOutcome.spec.ts b/browser_tests/tests/agent/agentConsentOutcome.spec.ts new file mode 100644 index 00000000000..f69ccb9564e --- /dev/null +++ b/browser_tests/tests/agent/agentConsentOutcome.spec.ts @@ -0,0 +1,197 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { AGENT_CONSENT_SETTING_ID } from '@/platform/settings/constants/agent' + +import { agentConsentTelemetryTest as test } from '@e2e/fixtures/agentConsentTelemetryFixture' + +/** + * A card that was shown and then went quiet used to cover three endings at + * once - dismissed, an acceptance whose save did not stick, and (signed out) an + * abandoned sign-in. `useAgentConsent.test.ts` pins all of them, but it arranges + * a dismissal by calling the dialog's `onClose` prop directly. This file is here + * because that is the one arrange a unit test cannot make faithfully: whether a + * real Escape key or a real overlay click reaches that handler at all depends on + * the dialog renderer, the dialog stack's escape bookkeeping and the overlay's + * own hit area, none of which a hand-called callback exercises. + */ +test.describe( + 'Automatic consent card outcome', + { tag: ['@cloud', '@ui'] }, + () => { + test.use({ agentConsentAccepted: false }) + + test('reports a dismissal when Escape closes the automatic card', async ({ + comfyPage, + agentPanel, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await test.step('The automatic offer reports exactly one impression', async () => { + await expect(dialog).toBeVisible() + await expect + .poll(() => + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_shown' + ) + ) + .toEqual([ + { + event: 'app:agent_consent_shown', + properties: expect.objectContaining({ trigger: 'first_load' }) + } + ]) + }) + + await test.step('Escape reports the dismissal as its own outcome', async () => { + expect( + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ).toEqual([]) + + await page.keyboard.press('Escape') + await expect(dialog).toHaveCount(0) + + await expect + .poll(() => + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ) + .toEqual([ + { + event: 'app:agent_consent_resolved', + properties: expect.objectContaining({ + decision: 'dismissed', + save_error_shown: false + }) + } + ]) + // Consent semantics are unchanged: a dismissal is still not a decision. + expect(agentConsentWrites).toHaveLength(0) + await expect(agentPanel.root).toHaveCount(0) + }) + + await test.step('One outcome per impression, not one per close path', async () => { + // The dialog closes through `closeDialog`, which fires `onClose`, and + // the promise settles separately. Both ran; only one event exists. + expect( + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ).toHaveLength(1) + }) + }) + + test('reports the same outcome when the overlay mask closes the card', async ({ + comfyPage, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await expect(dialog).toBeVisible() + await page + .getByTestId('dialog-overlay') + .click({ position: { x: 1, y: 1 } }) + await expect(dialog).toHaveCount(0) + + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_resolved') + .map((e) => e.properties.decision) + ) + .toEqual(['dismissed']) + expect(agentConsentWrites).toHaveLength(0) + }) + + test('reports an acceptance, not a dismissal, when the card is accepted', async ({ + comfyPage, + agentPanel, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await expect(dialog).toBeVisible() + await dialog + .getByRole('button', { name: enMessages.agent.consent.accept }) + .click() + await expect(dialog).toHaveCount(0) + await expect(agentPanel.root).toBeVisible() + + // `closeWith` settles before closing the dialog, so the accepted outcome + // must not be followed by a dismissal from the close it performs itself. + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_resolved') + .map((e) => e.properties.decision) + ) + .toEqual(['accepted']) + expect(agentConsentWrites).toEqual([true]) + }) + + test('names a failed consent read at the request, not just at the load', async ({ + comfyPage, + agentPanel, + consentTelemetry + }) => { + const page = comfyPage.page + // Every read fails. The automatic chain exits at the `load` stage without + // asking, which `gc-16`'s instrument already names; the user then clicks + // the entry button, and that attempt fails inside the request. This is the + // shape behind the 23 `agent_consent_setting_load_failure` events in + // Sentry that product analytics saw nothing of. + await page.route( + `**/api/global-settings/${AGENT_CONSENT_SETTING_ID}`, + (route) => route.fulfill({ status: 500, body: '{"code":"INTERNAL"}' }) + ) + await comfyPage.workflow.reloadAndWaitForApp() + + await expect(agentPanel.openButton).toBeEnabled() + await agentPanel.openButton.click() + + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_offer_exited') + .map((e) => [e.properties.stage, e.properties.exit]) + ) + .toContainEqual(['request', 'consent_read_failed']) + + // The same condition at two links, told apart by the pair rather than by + // the reason value - which is what `exit` plus `stage` is for. + const requestExits = consentTelemetry.filter( + (e) => + e.event === 'app:agent_consent_offer_exited' && + e.properties.stage === 'request' + ) + expect(requestExits.map((e) => e.properties)).toEqual([ + { + exit: 'consent_read_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ]) + await expect( + page.getByRole('dialog', { name: enMessages.agent.consent.title }) + ).toHaveCount(0) + await expect(agentPanel.root).toHaveCount(0) + }) + } +) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index 733f2019075..3dcb70d67c3 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -172,6 +172,12 @@ const notOffered = async () => .trackAgentConsentNotOffered ) +const offerExited = async () => + vi.mocked( + (await import('@/platform/telemetry')).useTelemetry()! + .trackAgentConsentOfferExited + ) + async function loadEntryAndSetup(): Promise { registerAgentPanelExtension() const ext = mocks.capturedExtensions.find( @@ -763,6 +769,303 @@ describe('AgentPanel extension flag gate', () => { }) }) + describe('names the exit when the offer ends without naming a surface', () => { + const rerunFlagGate = async (): Promise => { + agentFlagEnabled.value = false + await nextTick() + agentFlagEnabled.value = true + await nextTick() + } + + it.for([ + { + exit: 'consent_unresolved', + arrange: () => { + Object.assign(consentStore, { isChecking: true }) + } + }, + { + exit: 'consent_already_accepted', + arrange: () => { + Object.assign(consentStore, { accepted: true }) + } + }, + { + exit: 'workspace_unresolved', + arrange: () => { + Object.assign(workspaceStore, { activeWorkspaceId: null }) + } + }, + { + exit: 'workspace_switching', + arrange: () => { + Object.assign(workspaceStore, { isSwitching: true }) + } + }, + { + // The one-shot key silences `agent_consent_not_offered` outright, so + // this is the case that most needs its own event rather than a reason. + exit: 'already_offered', + arrange: () => { + localStorage.setItem(AUTO_SHOWN_KEY, 'true') + } + } + ] as const)( + 'reports $exit once however often the offer re-runs', + async ({ exit, arrange }) => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + arrange() + + await loadEntryAndSetup() + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit, + stage: 'offer', + retry_armed: false + }) + expect(useAgentConsent().withConsent).not.toHaveBeenCalled() + expect(await notOffered()).not.toHaveBeenCalled() + } + ) + + it('reports a signed-out offer separately from an unresolved account', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + const { useCurrentUser } = + await import('@/composables/auth/useCurrentUser') + vi.mocked(useCurrentUser()).isLoggedIn = computed(() => false) + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'signed_out', + stage: 'offer', + retry_armed: false + }) + }) + + it('reports an account that went missing after the consent read succeeded', async () => { + // `isLoggedIn` true with no resolved id is the API-key rail before its + // user is known - the only way these two conditions come apart, and the + // reason they are separate values rather than one. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + const { useCurrentUser } = + await import('@/composables/auth/useCurrentUser') + vi.mocked(useCurrentUser()).isLoggedIn = computed(() => true) + let decide = (_: boolean) => {} + startupDecision = new Promise((resolve) => { + decide = resolve + }) + + await loadEntryAndSetup() + await flush() + currentUser.value = null + decide(true) + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'account_unresolved', + stage: 'offer', + retry_armed: false + }) + }) + + it('reports an offer refused because another attempt is still in flight', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let finish = () => {} + const pending = new Promise((resolve) => { + finish = resolve + }) + vi.mocked(useAgentConsent().withConsent).mockImplementationOnce( + async () => { + await pending + } + ) + + await loadEntryAndSetup() + await vi.waitFor(() => + expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() + ) + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'offer_in_flight', + stage: 'offer', + retry_armed: false + }) + finish() + await pending + }) + + it('reports an offer skipped because the card has already been on screen', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + openDialog(CONSENT_DIALOG_KEY) + await flush() + closeDialog(CONSENT_DIALOG_KEY) + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledWith({ + exit: 'card_already_seen', + stage: 'offer', + retry_armed: false + }) + }) + + it.for([ + { + exit: 'consent_already_accepted', + stage: 'load', + arrange: () => { + vi.mocked(consentStore.load).mockResolvedValue(true) + } + }, + { + exit: 'consent_read_failed', + stage: 'load', + arrange: () => { + vi.mocked(consentStore.load).mockRejectedValue(new Error('offline')) + } + }, + { + exit: 'startup_probe_failed', + stage: 'startup', + arrange: () => { + const rejected = Promise.reject(new Error('startup blew up')) + // Marked handled here so the rejection is not unhandled during the + // window before the extension attaches its own catch. + rejected.catch(() => {}) + startupDecision = rejected + } + } + ] as const)( + 'reports $exit at the $stage stage', + async ({ exit, stage, arrange }) => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + arrange() + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit, + stage, + retry_armed: false + }) + } + ) + + it('reports why an undecided boot was also ineligible', async () => { + // `boot_undecided` is gated on eligibility, so this combination used to + // forfeit the offer and report nothing at all. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let decide = (_: boolean) => {} + startupDecision = new Promise((resolve) => { + decide = resolve + }) + + await loadEntryAndSetup() + await flush() + Object.assign(consentStore, { isChecking: true }) + decide(false) + await flush() + + expect(await notOffered()).not.toHaveBeenCalled() + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_unresolved', + stage: 'startup', + retry_armed: false + }) + }) + + it('marks the exit that loses a held offer as still having a retry armed', async () => { + // This is gc-17's silent loss, made visible: a dialog holds the offer, + // the screen clears, and the retry's consent read fails. Before #18975 + // the offer was gone for the page load; after it the hold survives - and + // either way the only telemetry was a first `dialog_open`, because the + // reason is deduplicated per page load. + agentFlagEnabled.value = true + openDialog() + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + await flush() + expect(await notOffered()).toHaveBeenCalledExactlyOnceWith({ + reason: 'dialog_open' + }) + expect(await offerExited()).not.toHaveBeenCalled() + + vi.mocked(consentStore.load).mockRejectedValueOnce(new Error('offline')) + closeDialog() + await flush() + await flush() + + expect(await notOffered()).toHaveBeenCalledOnce() + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_read_failed', + stage: 'load', + retry_armed: true + }) + }) + + it('reports nothing for an unflagged page load', async () => { + // Not because the reporter is gated on the flag - it is not - but because + // `loadConsentIfEligible` returns on an off flag before reaching any + // reported exit. That is the whole mechanism keeping this event off the + // population outside the rollout, so it is asserted rather than assumed. + agentFlagEnabled.value = false + Object.assign(consentStore, { accepted: false, isChecking: false }) + vi.mocked(consentStore.load).mockRejectedValue(new Error('offline')) + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).not.toHaveBeenCalled() + expect(consentStore.load).not.toHaveBeenCalled() + }) + + it('still reports why the offer ended when the flag goes off mid-flight', async () => { + // The flag was on when the offer started, so this attempt is already + // under way and its ending is real. A reporter gated on the current flag + // value would hide exactly this case, which is why it is not. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let fail = (_: Error) => {} + vi.mocked(consentStore.load).mockReturnValueOnce( + new Promise((_, reject) => { + fail = reject + }) + ) + + await loadEntryAndSetup() + await vi.waitFor(() => expect(consentStore.load).toHaveBeenCalled()) + agentFlagEnabled.value = false + await nextTick() + await flush() + fail(new Error('offline')) + await flush() + + expect(agentStore.enabled).toBe(false) + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_read_failed', + stage: 'load', + retry_armed: false + }) + }) + }) + it('offers in the same session once the dialog that held it closes', async () => { agentFlagEnabled.value = true openDialog() @@ -780,6 +1083,41 @@ describe('AgentPanel extension flag gate', () => { expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBe('true') }) + it('keeps the held offer when the retry cannot be made, and offers on the next clear screen', async () => { + agentFlagEnabled.value = true + openDialog() + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + await flush() + expect(await notOffered()).toHaveBeenCalledExactlyOnceWith({ + reason: 'dialog_open' + }) + + // The screen clears, but the retry's consent read fails - a cloud session + // whose auth is still settling is the ordinary way that happens. The + // release used to consume the hold before finding that out, which lost the + // offer for the rest of the page load and emitted nothing to say so: the + // reason is deduplicated per page load, so there is no second + // `agent_consent_not_offered` either. + vi.mocked(consentStore.load).mockRejectedValueOnce(new Error('offline')) + closeDialog() + await flush() + await flush() + expect(useAgentConsent().withConsent).not.toHaveBeenCalled() + expect(await notOffered()).toHaveBeenCalledOnce() + + // A later dialog comes and goes and the read works this time. The offer is + // still owed, so it lands. + openDialog() + await flush() + closeDialog() + await vi.waitFor(() => + expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() + ) + expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBe('true') + }) + it('keeps waiting when a tour ends while a dialog is still open', async () => { agentFlagEnabled.value = true activeTour.value = 'appMode' diff --git a/src/extensions/core/agentPanel.ts b/src/extensions/core/agentPanel.ts index 086ba8b6a84..7ad5d485926 100644 --- a/src/extensions/core/agentPanel.ts +++ b/src/extensions/core/agentPanel.ts @@ -11,7 +11,11 @@ import { } from '@/platform/remoteConfig/remoteConfig' import { useTelemetry } from '@/platform/telemetry' import { reportError } from '@/platform/telemetry/reportError' -import type { AgentConsentNotOfferedReason } from '@/platform/telemetry/types' +import type { + AgentConsentNotOfferedReason, + AgentConsentOfferExit, + AgentConsentOfferStage +} from '@/platform/telemetry/types' import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore' import { useFirstRunEntry } from '@/renderer/extensions/firstRunTour/gettingStarted/firstRunEntry' import { @@ -226,6 +230,57 @@ export function registerAgentPanelExtension(): void { withholdOffer(reason, userId, workspaceId) offerHeld.value = true } + /** + * Drops the hold. This is a claim that the offer no longer needs to be + * retried on this page load - it has just been made, or it has become + * moot - and it is deliberately *not* what the release watcher does. See + * the watcher for why those two have to be different things. + */ + const dropHold = (): void => { + offerHeld.value = false + } + + /** + * Names the endings that `agent_consent_not_offered` cannot: that event + * reports the surface a deferred offer is queued behind, so an attempt + * that stops for a reason of its own emits nothing at all today and is + * indistinguishable in telemetry from a user who was never offered. + * + * Three things this deliberately does not route through `withholdOffer`. + * It returns without emitting when either id is missing - and a missing + * id is one of the endings that has to be reportable. It returns without + * emitting once the auto-show key is burned, which would silence every + * later page load of every user who has seen the card. And its reason + * enum means "a surface is holding the offer", which none of these are. + * + * Dedup is per page load and per (stage, exit), so this reports the + * *presence* of an ending, never its frequency - `loadConsentIfEligible` + * is re-driven by the identity watcher, the flag gate, the release + * watcher and `withConsent`'s settlement, so an undeduplicated count + * would measure how long the tab was open. + * + * There is no "the flag is off" guard here, and that is deliberate rather + * than an omission. `loadConsentIfEligible` is the only way in, and it + * returns on an off flag before reaching any ending below - so the guard + * would have suppressed nothing except the one case worth seeing: a flag + * that flips off *mid-page-load*, abandoning an offer that was already + * under way. The unflagged population is kept out by the exit that is + * deliberately not reported at all, not by a gate here. + */ + const reportedExits = new Set() + const reportOfferExit = ( + exit: AgentConsentOfferExit, + stage: AgentConsentOfferStage + ): void => { + const key = `${stage}:${exit}` + if (reportedExits.has(key)) return + reportedExits.add(key) + useTelemetry()?.trackAgentConsentOfferExited({ + exit, + stage, + retry_armed: offerHeld.value + }) + } const consentScope = (): string | null => { const userId = resolvedUserInfo.value?.id @@ -246,13 +301,60 @@ export function registerAgentPanelExtension(): void { isLoggedIn.value && !consentStore.isChecking && !consentStore.accepted + /** + * Which of `offerEligible`'s conditions is unmet, in the order it checks + * them. Only meaningful when `offerEligible()` is false, and deliberately + * a separate expression from it: naming the cause must not be able to + * change the decision. + * + * `enabled` is not covered, so this is null when the flag being off is the + * only thing unmet: an off flag has no value on this event, for the reason + * on `AgentConsentOfferExit`. + */ + const offerIneligibility = (): AgentConsentOfferExit | null => + !isLoggedIn.value + ? 'signed_out' + : consentStore.isChecking + ? 'consent_unresolved' + : consentStore.accepted + ? 'consent_already_accepted' + : null + /** + * Why the offer has no consent scope to work with. A switch in progress is + * reported ahead of the ids it is moving, because it explains an absent + * workspace id and labelling that case `workspace_unresolved` would hide + * the more specific cause. + */ + const missingScopeExit = ( + userId: string | undefined + ): AgentConsentOfferExit => + workspaceStore.isSwitching + ? 'workspace_switching' + : !userId + ? 'account_unresolved' + : 'workspace_unresolved' let autoShowInFlight = false const offerConsentUnprompted = (): void => { - if (autoShowInFlight) return - if (!offerEligible()) return + // An exit that leaves the hold armed does so on purpose: the condition + // is transient, so the offer is still owed and the next clear screen + // has to retry it. `dropHold` marks the exits that are not transient. + if (autoShowInFlight) { + reportOfferExit('offer_in_flight', 'offer') + return + } + if (!offerEligible()) { + const ineligible = offerIneligibility() + if (ineligible) reportOfferExit(ineligible, 'offer') + if (consentStore.accepted) dropHold() + return + } const scope = consentScope() - if (scope && consentCardSeenIn.has(scope)) return + if (scope && consentCardSeenIn.has(scope)) { + reportOfferExit('card_already_seen', 'offer') + dropHold() + return + } // Must precede prepareAutoShow, which burns the one-shot key. const held = screenHolder() if (held) { @@ -262,14 +364,28 @@ export function registerAgentPanelExtension(): void { const userId = resolvedUserInfo.value?.id const workspaceId = workspaceStore.activeWorkspaceId - if (!userId || !workspaceId || workspaceStore.isSwitching) return + if (!userId || !workspaceId || workspaceStore.isSwitching) { + reportOfferExit(missingScopeExit(userId), 'offer') + return + } const key = `${CONSENT_AUTO_SHOWN_PREFIX}.${userId}.${workspaceId}` const autoShow = prepareAutoShow(key) - if (autoShow === 'storage_unavailable') withholdOffer(autoShow) - if (autoShow !== 'ready') return + if (autoShow !== 'ready') { + // `storage_unavailable` already has a reason on + // `agent_consent_not_offered`, so only the burned one-shot key needs + // naming here. + if (autoShow === 'storage_unavailable') withholdOffer(autoShow) + else reportOfferExit('already_offered', 'offer') + dropHold() + return + } const offeredIdentity = consentStore.identity autoShowInFlight = true + // The offer is being made now, so it is no longer owed. `canShow` can + // still re-arm the hold from under this if a surface takes the screen + // before the card mounts. + dropHold() agentPanelStore.suppressRestoredOpen() void withConsent( 'first_load', @@ -290,6 +406,13 @@ export function registerAgentPanelExtension(): void { ).finally(() => { autoShowInFlight = false if (consentStore.identity !== offeredIdentity) loadConsentIfEligible() + // A hold armed while this attempt was in flight was refused by the + // `autoShowInFlight` guard above, and the release watcher cannot + // help: the screen may have gone clear again before the guard + // dropped, and `whenever` only fires on a transition. Settling is + // the wake-up for that case. + else if (offerHeld.value && screenIsClear.value) + loadConsentIfEligible() }) } @@ -300,8 +423,16 @@ export function registerAgentPanelExtension(): void { .then((decided) => { if (decided) offerConsentUnprompted() else if (offerEligible()) withholdOffer('boot_undecided') + else { + // An undecided boot that is also ineligible reported nothing at + // all: `boot_undecided` is gated on eligibility, so the forfeited + // offer looked identical to one that was never owed. + const ineligible = offerIneligibility() + if (ineligible) reportOfferExit(ineligible, 'startup') + } }) .catch((error: unknown) => { + reportOfferExit('startup_probe_failed', 'startup') reportError(error, { errorType: 'agent_consent_auto_offer_failure' }) @@ -309,13 +440,27 @@ export function registerAgentPanelExtension(): void { } const loadConsentIfEligible = (): void => { + // Neither of these reports. An off flag is not this event's business + // (see `reportOfferExit`), and an account that has not resolved *yet* is + // not a lost offer: the watcher below re-drives this the moment + // `resolvedUserInfo` changes, so the exit has a guaranteed wake-up. It + // would otherwise fire on essentially every flagged page load, because + // the flag gate calls this before the auth rail settles - and a value + // that is present almost always discriminates nothing while looking + // like the dominant cause. `account_unresolved` is reported only at the + // `offer` stage, where the id went missing *after* a consent read had + // already succeeded with it. if (!agentPanelStore.enabled || !resolvedUserInfo.value) return void consentStore .load() .then((isAccepted) => { - if (!isAccepted) offerWhenStartupDecided() + if (isAccepted) { + reportOfferExit('consent_already_accepted', 'load') + dropHold() + } else offerWhenStartupDecided() }) .catch((error: unknown) => { + reportOfferExit('consent_read_failed', 'load') reportError(error, { errorType: 'agent_consent_setting_load_failure' }) @@ -326,10 +471,21 @@ export function registerAgentPanelExtension(): void { loadConsentIfEligible, { immediate: true } ) + /** + * Releasing a hold must not consume it. What the release triggers is + * asynchronous and exits early in several transient ways - a consent read + * that rejects, an offer already in flight, a workspace mid-switch, an + * account not resolved yet - and none of those exits re-arm. Dropping the + * hold here first turned any one of them into an offer lost for the rest + * of the page load, and lost *silently*: no card, and no second + * `agent_consent_not_offered`, because the reason is deduplicated per + * page load. The hold is dropped only by `dropHold`, at the points where + * the offer has actually been made or has become moot, so a retry that + * cannot be made is retried on the next clear screen instead. + */ whenever( () => offerHeld.value && screenIsClear.value, () => { - offerHeld.value = false loadConsentIfEligible() } ) diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index f79bb5b8143..8dfc1d5a3b3 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -304,7 +304,8 @@ describe('TelemetryRegistry', () => { trigger: 'button_click' } satisfies AgentConsentShownMetadata const consentResolvedMetadata = { - decision: 'accepted' + decision: 'dismissed', + save_error_shown: true } satisfies AgentConsentResolvedMetadata const onboardingStepMetadata = { step: 2, @@ -469,6 +470,36 @@ describe('TelemetryRegistry', () => { invoke: (registry) => registry.trackAgentConsentNotOffered({ reason: 'first_run_screen' }) }, + { + method: 'trackAgentConsentOfferExited', + expected: { + exit: 'already_offered', + stage: 'offer', + retry_armed: false + }, + invoke: (registry) => + registry.trackAgentConsentOfferExited({ + exit: 'already_offered', + stage: 'offer', + retry_armed: false + }) + }, + { + method: 'trackAgentConsentOfferExited', + expected: { + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + }, + invoke: (registry) => + registry.trackAgentConsentOfferExited({ + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + }) + }, { method: 'trackAgentOnboardingNotShown', expected: { reason: 'app_mode' }, diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index c92b6344ed8..1f27978371e 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -2,6 +2,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentPaywallCtaMetadata, @@ -469,6 +470,14 @@ export class TelemetryRegistry implements TelemetryDispatcher { ) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.dispatch((provider) => + provider.trackAgentConsentOfferExited?.(metadata) + ) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index 4bb635c90b7..35896361557 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -440,13 +440,19 @@ describe('PostHogTelemetryProvider', () => { await vi.dynamicImportSettled() provider.trackAgentConsentShown({ trigger: 'first_load' }) - provider.trackAgentConsentResolved({ decision: 'accepted' }) + provider.trackAgentConsentResolved({ + decision: 'dismissed', + save_error_shown: true + }) provider.trackAgentOnboardingShown() provider.trackAgentOnboardingStep({ step: 4, action: 'finish' }) expect(hoisted.mockCapture.mock.calls).toEqual([ [TelemetryEvents.AGENT_CONSENT_SHOWN, { trigger: 'first_load' }], - [TelemetryEvents.AGENT_CONSENT_RESOLVED, { decision: 'accepted' }], + [ + TelemetryEvents.AGENT_CONSENT_RESOLVED, + { decision: 'dismissed', save_error_shown: true } + ], [TelemetryEvents.AGENT_ONBOARDING_SHOWN, {}], [TelemetryEvents.AGENT_ONBOARDING_STEP, { step: 4, action: 'finish' }] ]) @@ -793,6 +799,36 @@ describe('PostHogTelemetryProvider', () => { provider.trackAgentConsentNotOffered({ reason: 'tour_active' }), properties: { reason: 'tour_active' } }, + { + event: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (provider: PostHogTelemetryProvider) => + provider.trackAgentConsentOfferExited({ + exit: 'consent_unresolved', + stage: 'load', + retry_armed: false + }), + properties: { + exit: 'consent_unresolved', + stage: 'load', + retry_armed: false + } + }, + { + event: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (provider: PostHogTelemetryProvider) => + provider.trackAgentConsentOfferExited({ + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + }), + properties: { + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + }, { event: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (provider: PostHogTelemetryProvider) => diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index 331cfff0fd5..eacd658390f 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -16,6 +16,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentEntryButtonClickedMetadata, @@ -830,6 +831,12 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_CONSENT_NOT_OFFERED, metadata) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.trackEvent(TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, metadata) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index 7a20a9d89f2..b17e5971852 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -317,8 +317,11 @@ describe('HostTelemetrySink', () => { { name: TelemetryEvents.AGENT_CONSENT_RESOLVED, track: (sink: HostTelemetrySink) => - sink.trackAgentConsentResolved({ decision: 'accepted' }), - properties: { decision: 'accepted' } + sink.trackAgentConsentResolved({ + decision: 'dismissed', + save_error_shown: true + }), + properties: { decision: 'dismissed', save_error_shown: true } }, { name: TelemetryEvents.AGENT_ONBOARDING_SHOWN, @@ -421,6 +424,36 @@ describe('HostTelemetrySink', () => { sink.trackAgentConsentNotOffered({ reason: 'tour_active' }), properties: { reason: 'tour_active' } }, + { + name: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (sink: HostTelemetrySink) => + sink.trackAgentConsentOfferExited({ + exit: 'workspace_switching', + stage: 'offer', + retry_armed: true + }), + properties: { + exit: 'workspace_switching', + stage: 'offer', + retry_armed: true + } + }, + { + name: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (sink: HostTelemetrySink) => + sink.trackAgentConsentOfferExited({ + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + }), + properties: { + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + }, { name: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.ts index 39fa501b585..53c50146411 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.ts @@ -6,6 +6,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentEntryButtonClickedMetadata, @@ -408,6 +409,12 @@ export class HostTelemetrySink implements TelemetryProvider { this.capture(TelemetryEvents.AGENT_CONSENT_NOT_OFFERED, metadata) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.capture(TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, metadata) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 3dc1b30bc51..47add42fef7 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -640,6 +640,132 @@ export interface AgentConsentNotOfferedMetadata extends Record< > { reason: AgentConsentNotOfferedReason } +/** + * Why an automatic consent offer ended without either making the offer or + * naming a surface that is holding it. + * + * `AgentConsentNotOfferedReason` covers the deferrals: a surface is in the way, + * it is named, and the offer is retried when that surface clears. Everything + * here is the other kind of ending - the attempt stopped for a reason of its + * own. Some of those endings are correct (the offer was not needed) and some + * are losses (it was owed and did not happen), so **a query over this event + * must split by `exit`; a total is not a quantity.** + * + * Correctly not needed: `consent_already_accepted`, `card_already_seen`, + * `already_offered`. + * Owed and not made: everything else. + * + * There is deliberately no value for "the agent flag is off". Every exit here + * is downstream of that check, so reporting it would emit once per page load + * for everyone outside the rollout - a count of exposure rather than of the + * mechanism - and the flag is already on every event as + * `$feature/agent-in-app-experience`. + * + * The `request` stage is the one place these endings are *countable*: see + * `AgentConsentOfferStage`. Every other stage reports the presence of an + * ending once per page load, so counts are comparable within one + * (`stage`, `exit`) pair and never across stages. + */ +export type AgentConsentOfferExit = + /** No Comfy account is signed in. */ + | 'signed_out' + /** Signed in, but the account has not resolved to a user id yet. */ + | 'account_unresolved' + /** No active workspace id yet, and no switch is in progress. */ + | 'workspace_unresolved' + /** A workspace switch is in progress, so the consent scope is moving. */ + | 'workspace_switching' + /** The stored consent read has not settled, so consent is unknown. */ + | 'consent_unresolved' + /** The stored consent read rejected. */ + | 'consent_read_failed' + /** Consent is already stored for this scope, so no card is needed. */ + | 'consent_already_accepted' + /** Another offer attempt for this page load has not finished. */ + | 'offer_in_flight' + /** The card has already been on screen for this scope this page load. */ + | 'card_already_seen' + /** The one-shot auto-show key for this scope is already burned. */ + | 'already_offered' + /** The first-run startup probe rejected. */ + | 'startup_probe_failed' + /** + * The consent scope moved while it was being resolved - a different account, + * a workspace transition, or a switch that started - so the read that would + * have decided whether to ask was never made. `request` stage only. + */ + | 'scope_changed_before_read' + /** + * Resolving the consent scope raised. The account or the workspace went away + * between the offer decision and the request. `request` stage only. + */ + | 'scope_probe_failed' + /** + * The consent scope moved while the stored-consent read was in flight, so the + * answer that came back belonged to a scope that is no longer current. + * `request` stage only. + */ + | 'scope_changed_after_read' + /** + * The card was asked for and the dialog closed before it rendered, so there + * is no `agent_consent_shown` and no outcome to record against one. `request` + * stage only. + */ + | 'card_closed_before_mount' +/** + * Which link in the offer chain exited. The same condition is checked at more + * than one of these - the pair (`exit`, `stage`) is what identifies a single + * exit in the code, so neither property is readable on its own. + */ +export type AgentConsentOfferStage = + /** `loadConsentIfEligible` - before the consent read, or on its result. */ + | 'load' + /** Waiting on the first-run startup decision. */ + | 'startup' + /** `offerConsentUnprompted` - the offer attempt itself. */ + | 'offer' + /** + * The consent request itself - `requestConsentForCurrentUser` and the card + * lifecycle in `showConsentDialog` - after the chain has decided to ask. + * + * The only stage that is **not** deduplicated, because it cannot inflate: + * the automatic path reaches it at most once per consent scope per page load + * (the one-shot auto-show key is burned first) and the button path reaches it + * once per click, so a repeat here is a repeated attempt rather than a + * measure of how long the tab was open. + */ + | 'request' +export interface AgentConsentOfferExitedMetadata extends Record< + string, + unknown +> { + exit: AgentConsentOfferExit + stage: AgentConsentOfferStage + /** + * Whether a hold was armed at the moment of the exit, i.e. whether this page + * load still has a queued retry. False on an owed-and-not-made exit means the + * offer is gone for this page load with nothing scheduled to bring it back. + * + * Two qualifications, both from the `request` stage. It is **structurally + * false** there: the offer drops the hold immediately before requesting, and + * the only thing that re-arms it is the `canShow` hook, which runs after + * every `request` exit. And the hold is not the only wake-up - `withConsent` + * settling re-drives the chain when the identity changed - so on the two + * `scope_changed_*` exits a retry does happen, by a mechanism this property + * does not describe. + */ + retry_armed: boolean + /** + * Which surface asked. Set **only** at the `request` stage, which is the one + * stage reachable from the topbar button as well as the automatic offer; + * every other stage is inside `offerConsentUnprompted` and automatic by + * construction. A query about automatic offers must therefore either restrict + * to `stage != 'request'` or filter `trigger = 'first_load'` - counting the + * `request` stage unsplit mixes a user-initiated click into the automatic + * denominator. + */ + trigger?: AgentConsentTrigger +} export type AgentOnboardingNotShownMetadata = | { reason: 'app_mode' | 'tour_active' } | { reason: 'target_missing'; step: number } @@ -658,16 +784,61 @@ export interface AgentConsentShownMetadata extends Record { trigger: AgentConsentTrigger } /** - * Only consent the user actually gave or refused resolves the card, so this - * never reports a decision that did not stick. An `agent_consent_shown` with - * no matching resolution is an *unresolved* offer, not a dismissal: it covers - * dismissing the card (Escape, overlay click), an acceptance whose save - * failed, and — signed out — accepting the card but abandoning the sign-in - * that has to follow. Splitting those three apart needs a signal this event - * does not carry. + * How a consent card that was on screen ended, plus the moment consent becomes + * stored. It used to carry only the two deciding values, which left a card that + * was shown and then went quiet covering three different endings at once - a + * dismissal, an acceptance whose save did not stick, and a signed-out + * acceptance whose sign-in was abandoned. Those are now named, and + * `save_error_shown` separates giving up after a failed save from walking away. + * + * **Only `accepted` means consent is stored.** That was this event's whole + * meaning before the other values existed, so any query that counted it as "a + * decision that stuck" must now filter `decision = 'accepted'`, and one that + * wants "the user answered" wants `decision in ('accepted', 'rejected')`. + * + * **The pairing with `agent_consent_shown`, exactly.** For a signed-in user - + * the whole cloud population - every impression ends in exactly one of these, + * so an impression with no outcome is a defect rather than a dismissal. In the + * signed-out (local) flow the card is only the first half: it ends at + * `accepted_pending_sign_in`, and `accepted` follows separately if the sign-in + * and the real save land. So `accepted` is the one value that is not always the + * card's own ending, which is how it keeps meaning "consent is stored". */ +export type AgentConsentDecision = + /** Accepted, and the acceptance is durably stored. */ + | 'accepted' + /** Declined on the card. Nothing is stored; the card can be offered again. */ + | 'rejected' + /** + * Closed without deciding - Escape, the overlay mask, or a programmatic close + * such as navigation. Distinguishing this from `accept_not_persisted` is the + * difference between the user walking away and the product failing them. + */ + | 'dismissed' + /** + * Accepted, and the write did not stick without raising: the scope moved + * mid-save or the workspace auth header was gone. The user believes they + * consented and no consent exists. + */ + | 'accept_not_persisted' + /** + * Accepted the card in the signed-out flow, where the card is only the first + * half: a sign-in and a real save still have to land before consent exists, + * and they report `accepted` themselves when they do. So this value with no + * later `accepted` is an abandoned sign-in, which is the third case this + * event could not previously name. + */ + | 'accepted_pending_sign_in' export interface AgentConsentResolvedMetadata extends Record { - decision: 'accepted' | 'rejected' + decision: AgentConsentDecision + /** + * Whether the card had already shown a save error when it reached this + * outcome. A raised save leaves the card open and retryable, so its ending is + * one of the values above rather than an outcome of its own - this is what + * tells "dismissed after the save failed" from "dismissed without trying", + * and marks an `accepted` that only landed on a retry. + */ + save_error_shown: boolean } export type AgentOnboardingAction = 'next' | 'finish' | 'skip' /** @@ -1541,6 +1712,7 @@ export interface TelemetryProvider { trackAgentRunModeChanged?(metadata: AgentRunModeChangedMetadata): void trackAgentThreadStarted?(metadata: AgentThreadStartedMetadata): void trackAgentConsentNotOffered?(metadata: AgentConsentNotOfferedMetadata): void + trackAgentConsentOfferExited?(metadata: AgentConsentOfferExitedMetadata): void trackAgentOnboardingNotShown?(metadata: AgentOnboardingNotShownMetadata): void // Right side panel widget favorite events @@ -1736,6 +1908,7 @@ export const TelemetryEvents = { AGENT_RUN_MODE_CHANGED: 'app:agent_run_mode_changed', AGENT_THREAD_STARTED: 'app:agent_thread_started', AGENT_CONSENT_NOT_OFFERED: 'app:agent_consent_not_offered', + AGENT_CONSENT_OFFER_EXITED: 'app:agent_consent_offer_exited', AGENT_ONBOARDING_NOT_SHOWN: 'app:agent_onboarding_not_shown', // Right Side Panel Widget Favorites diff --git a/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts index 3b6d8c4040f..d162ec68984 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts @@ -45,7 +45,8 @@ vi.mock(import('@/platform/telemetry/reportError'), () => ({ const telemetry = vi.hoisted(() => ({ trackAgentConsentShown: vi.fn(), - trackAgentConsentResolved: vi.fn() + trackAgentConsentResolved: vi.fn(), + trackAgentConsentOfferExited: vi.fn() })) vi.mock(import('@/platform/telemetry'), () => ({ useTelemetry: () => telemetry @@ -165,7 +166,7 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'rejected' }] + [{ decision: 'rejected', save_error_shown: false }] ]) expect(onOpen).not.toHaveBeenCalled() }) @@ -183,12 +184,12 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'accepted' }] + [{ decision: 'accepted', save_error_shown: false }] ]) expect(onOpen).toHaveBeenCalledOnce() }) - it('reports no decision when the acceptance fails to save', async () => { + it('reports no outcome while a failed save leaves the card open', async () => { fetchWithUnifiedRemint .mockResolvedValueOnce(settingResponse(false)) .mockRejectedValueOnce(new Error('offline')) @@ -202,10 +203,12 @@ describe('useAgentConsent', () => { ) }) + // A raised save is not an ending: the card is still on screen and + // retryable, so the outcome belongs to whatever the user does next. expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() }) - it('reports no decision when the card is dismissed instead of answered', async () => { + it('reports a dismissal as its own outcome, not as an unresolved card', async () => { const request = useAgentConsent().withConsent('button_click', vi.fn()) const dialog = await waitForConsentDialog() await renderConsentCard(dialog) @@ -213,7 +216,77 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentShown).toHaveBeenCalledOnce() + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'dismissed', save_error_shown: false }] + ]) + expect(telemetry.trackAgentConsentOfferExited).not.toHaveBeenCalled() + }) + + it('separates giving up after a failed save from walking away', async () => { + fetchWithUnifiedRemint + .mockResolvedValueOnce(settingResponse(false)) + .mockRejectedValueOnce(new Error('offline')) + const request = useAgentConsent().withConsent('button_click', vi.fn()) + const dialog = await waitForConsentDialog() + await renderConsentCard(dialog) + + await clickCardAction('accept') + await vi.waitFor(() => { + expect(dialog.contentProps.error).toBe( + i18n.global.t('agent.consent.saveError') + ) + }) + ;(dialog.dialogComponentProps.onClose as () => void)() + await request + + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'dismissed', save_error_shown: true }] + ]) + }) + + it('reports an acceptance that did not persist without raising', async () => { + const identity = ref('account-a') + useCurrentUser().resolvedUserInfo = computed(() => ({ id: identity.value })) + const onOpen = vi.fn() + const request = useAgentConsent().withConsent('button_click', onOpen) + const dialog = await waitForConsentDialog() + await renderConsentCard(dialog) + + // The scope moves under the open card, so `accept` resolves false: nothing + // was stored, nothing raised, and no error is shown. The user believes they + // consented. + identity.value = 'account-b' + await clickCardAction('accept') + await request + + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'accept_not_persisted', save_error_shown: false }] + ]) + expect(reportError).not.toHaveBeenCalled() + expect(onOpen).not.toHaveBeenCalled() + }) + + it('reports a card closed before it rendered as an exit, not a dismissal', async () => { + const request = useAgentConsent().withConsent('first_load', vi.fn()) + const dialog = await waitForConsentDialog() + + // No `renderConsentCard`: the async card chunk has not mounted, so there is + // no impression for an outcome to resolve. + ;(dialog.dialogComponentProps.onClose as () => void)() + await request + + expect(telemetry.trackAgentConsentShown).not.toHaveBeenCalled() expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) }) it('waits for the account setting to load before deciding whether to ask', async () => { @@ -258,6 +331,120 @@ describe('useAgentConsent', () => { detail: i18n.global.t('agent.consent.loadError') }) ) + // `reportError` has no product-analytics sink, so before this the funnel saw + // a first-run user who was simply never offered. + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'consent_read_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ] + ]) + }) + + it('separates a failed scope probe from a failed consent read', async () => { + // Signed in, but the account has not resolved to a user id, so `ensureScope` + // raises before any read is attempted. The message it raises is the one + // `load` also raises when its auth header is missing, which is why the + // reason comes from which call was in flight rather than from the error. + useCurrentUser().resolvedUserInfo = computed(() => null) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('button_click', onOpen) + await request + + expect(fetchWithUnifiedRemint).not.toHaveBeenCalled() + expect(useDialogStore().dialogStack).toHaveLength(0) + expect(onOpen).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ] + ]) + }) + + it('names the scope moving while it was still being resolved', async () => { + Object.assign(useTeamWorkspaceStore(), { activeWorkspaceId: null }) + vi.mocked(useTeamWorkspaceStore().initialize).mockImplementationOnce( + async () => { + Object.assign(useTeamWorkspaceStore(), { + activeWorkspaceId: 'workspace-b', + workspaceTransitionGeneration: + useTeamWorkspaceStore().workspaceTransitionGeneration + 1 + }) + } + ) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('first_load', onOpen) + await request + + expect(fetchWithUnifiedRemint).not.toHaveBeenCalled() + expect(onOpen).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_changed_before_read', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) + }) + + it('names the scope moving while the consent read was in flight', async () => { + const load = deferred() + fetchWithUnifiedRemint.mockReturnValueOnce(load.promise) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('first_load', onOpen) + await vi.waitFor(() => { + expect(fetchWithUnifiedRemint).toHaveBeenCalledOnce() + }) + Object.assign(useTeamWorkspaceStore(), { + activeWorkspaceId: 'workspace-b', + workspaceTransitionGeneration: + useTeamWorkspaceStore().workspaceTransitionGeneration + 1 + }) + load.resolve(settingResponse(false)) + await request + + expect(useDialogStore().dialogStack).toHaveLength(0) + expect(onOpen).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) + }) + + it('reports a repeated attempt each time rather than once per page load', async () => { + fetchWithUnifiedRemint.mockRejectedValue(new Error('offline')) + + await useAgentConsent().withConsent('button_click', vi.fn()) + await useAgentConsent().withConsent('button_click', vi.fn()) + + // Not deduplicated, deliberately: the `request` stage is only reachable once + // per consent scope per page load automatically, and once per click from the + // button, so a repeat is a repeated attempt rather than a measure of how + // long the tab was open. + expect(telemetry.trackAgentConsentOfferExited).toHaveBeenCalledTimes(2) }) it('configures the first-use card as an accessible dismissable dialog', async () => { @@ -520,8 +707,12 @@ describe('useAgentConsent', () => { false ) expect(onOpen).toHaveBeenCalledOnce() + // Two outcomes for one card, and that is the contract: the card's own + // ending is `accepted_pending_sign_in`, and `accepted` is the later moment + // consent actually became stored. expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'accepted' }] + [{ decision: 'accepted_pending_sign_in', save_error_shown: false }], + [{ decision: 'accepted', save_error_shown: false }] ]) }) @@ -542,8 +733,12 @@ describe('useAgentConsent', () => { expect(useToastStore().add).not.toHaveBeenCalled() // Accepting the card is only half of the signed-out flow. Consent was // never persisted, so reporting it accepted would put a decision the user - // did not complete into the funnel. - expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() + // did not complete into the funnel — but the card half did happen, and + // `accepted_pending_sign_in` with no later `accepted` is exactly how an + // abandoned sign-in is now readable. + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'accepted_pending_sign_in', save_error_shown: false }] + ]) }) it('reports sign-in loading failure without saving or opening and allows another attempt', async () => { diff --git a/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts b/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts index cd35487437e..f02c57a0b55 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts @@ -4,7 +4,11 @@ import { storeToRefs } from 'pinia' import { useCurrentUser } from '@/composables/auth/useCurrentUser' import { i18n } from '@/i18n' import { useTelemetry } from '@/platform/telemetry' -import type { AgentConsentTrigger } from '@/platform/telemetry/types' +import type { + AgentConsentDecision, + AgentConsentOfferExit, + AgentConsentTrigger +} from '@/platform/telemetry/types' import { reportError } from '@/platform/telemetry/reportError' import { useToastStore } from '@/platform/updates/common/toastStore' import { useDialogService } from '@/services/dialogService' @@ -38,6 +42,40 @@ export function useAgentConsent() { const { accepted, identity, isChecking } = storeToRefs(consentStore) const { t } = i18n.global + /** + * Names an ending of the consent request that produced neither a card on + * screen nor a decision. + * + * These are the last silent endings on the automatic consent chain. + * `agent_consent_not_offered` means a named surface is holding the offer and + * will retry it, and `agent_consent_offer_exited`'s other stages all stop + * before the consent scope is resolved - so an attempt that got as far as + * asking and still ended with nothing was indistinguishable in product + * analytics from a user who was never offered. The failing ones do reach + * Sentry and RUM through `reportError`, which is why they were easy to + * believe were covered; `reportError` has no PostHog sink, so none of them + * were in the funnel denominator `agent_consent_not_offered` is read against. + * + * `retry_armed` is reported false rather than read, and that is a code reading + * rather than a tested fact: `offerConsentUnprompted` calls `dropHold()` + * immediately before `withConsent`, the only caller that can re-arm is the + * `canShow` hook, and `canShow` runs after every exit reported here. A + * re-entrant offer cannot re-arm either - it exits on `autoShowInFlight` + * before reaching a hold. If that ordering changes, this becomes wrong + * silently, so the property's doc says which ordering it depends on. + */ + function reportRequestExit( + exit: AgentConsentOfferExit, + trigger: AgentConsentTrigger + ): void { + useTelemetry()?.trackAgentConsentOfferExited({ + exit, + stage: 'request', + retry_armed: false, + trigger + }) + } + function showConsentDialog( trigger: AgentConsentTrigger, persistOnAccept = true, @@ -51,6 +89,21 @@ export function useAgentConsent() { } let settled = false let saving = false + let shown = false + let saveErrorShown = false + + /** + * One outcome per card that was on screen, so this event and + * `agent_consent_shown` stay 1:1 and a shown card with no outcome is a + * defect rather than a dismissal. `accepted` and `rejected` keep exactly + * the meaning they already had. + */ + const reportOutcome = (decision: AgentConsentDecision): void => { + useTelemetry()?.trackAgentConsentResolved({ + decision, + save_error_shown: saveErrorShown + }) + } const closeWith = (result: boolean): void => { if (settled) return @@ -62,6 +115,10 @@ export function useAgentConsent() { function handleSaveFailure(error: unknown): void { if (settled) return saving = false + // Not an outcome: the card stays open, retryable, with the error on it. + // The ending it eventually reaches carries this flag instead, which is + // what separates giving up after a failed save from walking away. + saveErrorShown = true reportError(error, { errorType: 'agent_consent_setting_write_failure' }) @@ -94,12 +151,19 @@ export function useAgentConsent() { const saved = persistOnAccept ? await consentStore.accept(expectedIdentity) : true - // Only a persisted acceptance resolves here. Without `persistOnAccept` - // this card is the first half of the signed-out flow, which still has - // a sign-in and a real save to clear before consent exists, so that - // path reports its own acceptance once those land. - if (persistOnAccept && saved) - useTelemetry()?.trackAgentConsentResolved({ decision: 'accepted' }) + // Only a persisted acceptance is `accepted`. Without + // `persistOnAccept` this card is the first half of the signed-out + // flow, which still has a sign-in and a real save to clear before + // consent exists, so that path still reports its own `accepted` once + // those land - `accepted_pending_sign_in` records that the card half + // is done without claiming consent that does not exist yet. + // + // A save that resolved false neither stored consent nor raised, so it + // shows no error and closes the card: the user believes they + // consented. That is the ending this event could not tell from a + // dismissal, and the whole reason the two are named separately. + if (!persistOnAccept) reportOutcome('accepted_pending_sign_in') + else reportOutcome(saved ? 'accepted' : 'accept_not_persisted') closeWith(saved) } catch (error) { handleSaveFailure(error) @@ -120,13 +184,14 @@ export function useAgentConsent() { accepting: false, error: '', onVnodeMounted: () => { + shown = true useTelemetry()?.trackAgentConsentShown({ trigger }) if (expectedIdentity && identity.value !== expectedIdentity) return onShown?.() }, onAccept: () => void accept(), onReject: () => { - useTelemetry()?.trackAgentConsentResolved({ decision: 'rejected' }) + reportOutcome('rejected') closeWith(false) } }, @@ -142,6 +207,19 @@ export function useAgentConsent() { onClose: () => { if (settled) return settled = true + // Escape, the overlay mask, or a programmatic close. Reached only + // when nothing else settled first, so it is exactly the ending that + // used to emit `agent_consent_shown` and then nothing at all. + // + // Before the card renders there is no impression to resolve, and + // reporting a dismissal against one would break this event's 1:1 + // agreement with `agent_consent_shown` in the other direction - so + // that window is an ending of the request instead. It is narrow (the + // card is an async component, so it is the chunk's load time) and + // its production frequency is unknown rather than assumed to be + // zero. + if (shown) reportOutcome('dismissed') + else reportRequestExit('card_closed_before_mount', trigger) resolve(false) } } @@ -173,7 +251,14 @@ export function useAgentConsent() { const decisionIdentity = await consentStore.ensureScope() if (!decisionIdentity || !(await consentStore.accept(decisionIdentity))) return null - useTelemetry()?.trackAgentConsentResolved({ decision: 'accepted' }) + // The second half of the signed-out flow, so this is the moment consent + // becomes stored rather than the card's own ending - the card already + // reported `accepted_pending_sign_in`. No save error can have been shown + // on it: its accept ran with `persistOnAccept` false and never wrote. + useTelemetry()?.trackAgentConsentResolved({ + decision: 'accepted', + save_error_shown: false + }) return decisionIdentity } catch (error) { reportError(error, { @@ -193,11 +278,25 @@ export function useAgentConsent() { hooks: ConsentOfferHooks ): Promise { let decisionIdentity: string | null + // Which call is in flight, so the catch can tell a scope probe that failed + // from a consent read that failed. The error cannot carry that: both raise + // `AgentConsentAuthenticationError` with the same "account authentication is + // required" message, `ensureScope` from its own missing user and `load` from + // the auth header its read needs. + let reading = false try { decisionIdentity = await consentStore.ensureScope() - if (!decisionIdentity) return null + if (!decisionIdentity) { + reportRequestExit('scope_changed_before_read', trigger) + return null + } + reading = true await consentStore.load() } catch (error) { + reportRequestExit( + reading ? 'consent_read_failed' : 'scope_probe_failed', + trigger + ) reportError(error, { errorType: 'agent_consent_setting_load_failure' }) @@ -209,7 +308,19 @@ export function useAgentConsent() { return null } - if (identity.value !== decisionIdentity) return null + if (identity.value !== decisionIdentity) { + reportRequestExit('scope_changed_after_read', trigger) + return null + } + // Two endings report nothing here, and both are successes rather than + // silences. `showConsentDialog` returning false because `canShow` said no is + // already named by the surface that took the screen - the caller's hook + // holds the offer and `agent_consent_not_offered` carries the reason, so + // naming it again would double-count one deferral. And consent that is + // already stored skips the card entirely: this runs on to `onAccept`, which + // for the automatic offer is the only producer of + // `agent_panel_opened(source=automatic_consent)`. Reading the absence of a + // card as a failed offer is the error that mis-ranked gc-17. if ( !accepted.value && !(await showConsentDialog(trigger, true, decisionIdentity, hooks))