From 9419c1efc46804f35aa9aa25275324c178fed445 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 26 Sep 2026 01:13:46 -0700 Subject: [PATCH 1/5] fix(agent): keep a held consent offer when the retry cannot be made MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releasing a held consent offer also consumed the hold, before anything had checked whether the retry could actually happen. Everything the release triggers is asynchronous and exits early in several transient ways — the consent read rejecting, an offer already in flight, a workspace mid-switch, an account not resolved yet — and none of those exits re-armed the hold. Any one of them therefore lost the offer for the rest of the page load, silently: no card, and no second `agent_consent_not_offered` either, because the reason is deduplicated per page load. The hold is now dropped only where the offer has been made or has become moot (consent already accepted, the card already seen this page load, the one-shot key already burned). A transient exit leaves it armed, so the next clear screen retries. An attempt that was refused by the in-flight guard also gets a wake-up when that attempt settles, since `whenever` only fires on a transition and the screen may have gone clear again in the meantime. Stacked on #18966, which arms the hold for `first_run_screen` as well and so routes a fourth reason through this same release path. Consent semantics are untouched: same copy, same recorded decision, still required before the panel opens, and the panel is not auto-opened. --- src/extensions/core/agentPanel.test.ts | 35 +++++++++++++++++ src/extensions/core/agentPanel.ts | 54 +++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index 733f2019075..4ac038c7a71 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -780,6 +780,41 @@ describe('AgentPanel extension flag gate', () => { expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBe('true') }) + it('keeps the held offer when the retry cannot be made, and offers on the next clear screen', async () => { + agentFlagEnabled.value = true + openDialog() + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + await flush() + expect(await notOffered()).toHaveBeenCalledExactlyOnceWith({ + reason: 'dialog_open' + }) + + // The screen clears, but the retry's consent read fails - a cloud session + // whose auth is still settling is the ordinary way that happens. The + // release used to consume the hold before finding that out, which lost the + // offer for the rest of the page load and emitted nothing to say so: the + // reason is deduplicated per page load, so there is no second + // `agent_consent_not_offered` either. + vi.mocked(consentStore.load).mockRejectedValueOnce(new Error('offline')) + closeDialog() + await flush() + await flush() + expect(useAgentConsent().withConsent).not.toHaveBeenCalled() + expect(await notOffered()).toHaveBeenCalledOnce() + + // A later dialog comes and goes and the read works this time. The offer is + // still owed, so it lands. + openDialog() + await flush() + closeDialog() + await vi.waitFor(() => + expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() + ) + expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBe('true') + }) + it('keeps waiting when a tour ends while a dialog is still open', async () => { agentFlagEnabled.value = true activeTour.value = 'appMode' diff --git a/src/extensions/core/agentPanel.ts b/src/extensions/core/agentPanel.ts index 086ba8b6a84..1a4e55f09f7 100644 --- a/src/extensions/core/agentPanel.ts +++ b/src/extensions/core/agentPanel.ts @@ -226,6 +226,15 @@ export function registerAgentPanelExtension(): void { withholdOffer(reason, userId, workspaceId) offerHeld.value = true } + /** + * Drops the hold. This is a claim that the offer no longer needs to be + * retried on this page load - it has just been made, or it has become + * moot - and it is deliberately *not* what the release watcher does. See + * the watcher for why those two have to be different things. + */ + const dropHold = (): void => { + offerHeld.value = false + } const consentScope = (): string | null => { const userId = resolvedUserInfo.value?.id @@ -249,10 +258,19 @@ export function registerAgentPanelExtension(): void { let autoShowInFlight = false const offerConsentUnprompted = (): void => { + // An exit that leaves the hold armed does so on purpose: the condition + // is transient, so the offer is still owed and the next clear screen + // has to retry it. `dropHold` marks the exits that are not transient. if (autoShowInFlight) return - if (!offerEligible()) return + if (!offerEligible()) { + if (consentStore.accepted) dropHold() + return + } const scope = consentScope() - if (scope && consentCardSeenIn.has(scope)) return + if (scope && consentCardSeenIn.has(scope)) { + dropHold() + return + } // Must precede prepareAutoShow, which burns the one-shot key. const held = screenHolder() if (held) { @@ -266,10 +284,17 @@ export function registerAgentPanelExtension(): void { const key = `${CONSENT_AUTO_SHOWN_PREFIX}.${userId}.${workspaceId}` const autoShow = prepareAutoShow(key) if (autoShow === 'storage_unavailable') withholdOffer(autoShow) - if (autoShow !== 'ready') return + if (autoShow !== 'ready') { + dropHold() + return + } const offeredIdentity = consentStore.identity autoShowInFlight = true + // The offer is being made now, so it is no longer owed. `canShow` can + // still re-arm the hold from under this if a surface takes the screen + // before the card mounts. + dropHold() agentPanelStore.suppressRestoredOpen() void withConsent( 'first_load', @@ -290,6 +315,13 @@ export function registerAgentPanelExtension(): void { ).finally(() => { autoShowInFlight = false if (consentStore.identity !== offeredIdentity) loadConsentIfEligible() + // A hold armed while this attempt was in flight was refused by the + // `autoShowInFlight` guard above, and the release watcher cannot + // help: the screen may have gone clear again before the guard + // dropped, and `whenever` only fires on a transition. Settling is + // the wake-up for that case. + else if (offerHeld.value && screenIsClear.value) + loadConsentIfEligible() }) } @@ -313,7 +345,8 @@ export function registerAgentPanelExtension(): void { void consentStore .load() .then((isAccepted) => { - if (!isAccepted) offerWhenStartupDecided() + if (isAccepted) dropHold() + else offerWhenStartupDecided() }) .catch((error: unknown) => { reportError(error, { @@ -326,10 +359,21 @@ export function registerAgentPanelExtension(): void { loadConsentIfEligible, { immediate: true } ) + /** + * Releasing a hold must not consume it. What the release triggers is + * asynchronous and exits early in several transient ways - a consent read + * that rejects, an offer already in flight, a workspace mid-switch, an + * account not resolved yet - and none of those exits re-arm. Dropping the + * hold here first turned any one of them into an offer lost for the rest + * of the page load, and lost *silently*: no card, and no second + * `agent_consent_not_offered`, because the reason is deduplicated per + * page load. The hold is dropped only by `dropHold`, at the points where + * the offer has actually been made or has become moot, so a retry that + * cannot be made is retried on the next clear screen instead. + */ whenever( () => offerHeld.value && screenIsClear.value, () => { - offerHeld.value = false loadConsentIfEligible() } ) From 77c48d22d895a7abe87236b75399d115d4b906bd Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 26 Sep 2026 02:15:22 -0700 Subject: [PATCH 2/5] feat(agent): name the exit when an automatic consent offer ends silently `agent_consent_not_offered` reports the surface a deferred consent offer is queued behind. An attempt that stops for a reason of its own reports nothing at all, so a user whose offer was lost is indistinguishable in telemetry from one who was never offered - 307 of 554 users held on `first_run_screen` return and emit neither a card nor a second suppression, and only 45 of those can be shown to have had consent already. Adds `app:agent_consent_offer_exited`, carrying `exit`, `stage` and `retry_armed`. The pair (`exit`, `stage`) identifies one exit in the code, because the same condition is checked at more than one link of the chain: `loadConsentIfEligible`, the startup probe, and `offerConsentUnprompted`. Three values mean the offer was correctly not needed (`consent_already_accepted`, `card_already_seen`, `already_offered`) and the rest mean it was owed and did not happen, so a query over this event must split by `exit`; a total is not a quantity. The type's doc comment says so. Deliberately a new event rather than new `reason` values, for three reasons. `withholdOffer` returns without emitting when either id is missing - and a missing id is one of the endings that has to be reportable. It also returns without emitting once the one-shot auto-show key is burned, which would silence every later page load of every user who has seen the card. And its reason enum means "a named surface is holding the offer", which none of these endings are; reusing it would redefine an event several published figures are counted against. Two exits stay unreported on purpose. An off feature flag would emit once per page load for the whole population outside the rollout - a count of exposure rather than of the mechanism - and it is already on every event as a `$feature` property. An account that has not resolved yet is not a lost offer: the identity watcher re-drives the chain the moment it does, so that exit has a guaranteed wake-up, and it would have been present on nearly every flagged page load while discriminating nothing. No change to the offer's behaviour. `offerEligible` is unchanged and `offerIneligibility` is a separate expression from it so that naming a cause cannot alter a decision; the two split `if`s are split for naming only. All 71 pre-existing cases pass untouched. --- src/extensions/core/agentPanel.test.ts | 324 ++++++++++++++++++ src/extensions/core/agentPanel.ts | 119 ++++++- .../telemetry/TelemetryRegistry.test.ts | 14 + src/platform/telemetry/TelemetryRegistry.ts | 9 + .../cloud/PostHogTelemetryProvider.test.ts | 14 + .../cloud/PostHogTelemetryProvider.ts | 7 + .../providers/host/HostTelemetrySink.test.ts | 14 + .../providers/host/HostTelemetrySink.ts | 7 + src/platform/telemetry/types.ts | 71 ++++ 9 files changed, 574 insertions(+), 5 deletions(-) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index 4ac038c7a71..e5ff4f57ffe 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -172,6 +172,12 @@ const notOffered = async () => .trackAgentConsentNotOffered ) +const offerExited = async () => + vi.mocked( + (await import('@/platform/telemetry')).useTelemetry()! + .trackAgentConsentOfferExited + ) + async function loadEntryAndSetup(): Promise { registerAgentPanelExtension() const ext = mocks.capturedExtensions.find( @@ -763,6 +769,324 @@ describe('AgentPanel extension flag gate', () => { }) }) + describe('names the exit when the offer ends without naming a surface', () => { + const rerunFlagGate = async (): Promise => { + agentFlagEnabled.value = false + await nextTick() + agentFlagEnabled.value = true + await nextTick() + } + + it.for([ + { + exit: 'consent_unresolved', + arrange: () => { + Object.assign(consentStore, { isChecking: true }) + } + }, + { + exit: 'consent_already_accepted', + arrange: () => { + Object.assign(consentStore, { accepted: true }) + } + }, + { + exit: 'workspace_unresolved', + arrange: () => { + Object.assign(workspaceStore, { activeWorkspaceId: null }) + } + }, + { + exit: 'workspace_switching', + arrange: () => { + Object.assign(workspaceStore, { isSwitching: true }) + } + }, + { + // The one-shot key silences `agent_consent_not_offered` outright, so + // this is the case that most needs its own event rather than a reason. + exit: 'already_offered', + arrange: () => { + localStorage.setItem(AUTO_SHOWN_KEY, 'true') + } + } + ] as const)( + 'reports $exit once however often the offer re-runs', + async ({ exit, arrange }) => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + arrange() + + await loadEntryAndSetup() + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit, + stage: 'offer', + retry_armed: false + }) + expect(useAgentConsent().withConsent).not.toHaveBeenCalled() + expect(await notOffered()).not.toHaveBeenCalled() + } + ) + + it('reports a signed-out offer separately from an unresolved account', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + const { useCurrentUser } = + await import('@/composables/auth/useCurrentUser') + vi.mocked(useCurrentUser()).isLoggedIn = computed(() => false) + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'signed_out', + stage: 'offer', + retry_armed: false + }) + }) + + it('reports an account that went missing after the consent read succeeded', async () => { + // `isLoggedIn` true with no resolved id is the API-key rail before its + // user is known - the only way these two conditions come apart, and the + // reason they are separate values rather than one. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + const { useCurrentUser } = + await import('@/composables/auth/useCurrentUser') + vi.mocked(useCurrentUser()).isLoggedIn = computed(() => true) + let decide = (_: boolean) => {} + startupDecision = new Promise((resolve) => { + decide = resolve + }) + + await loadEntryAndSetup() + await flush() + currentUser.value = null + decide(true) + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'account_unresolved', + stage: 'offer', + retry_armed: false + }) + }) + + it('reports an offer refused because another attempt is still in flight', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let finish = () => {} + const pending = new Promise((resolve) => { + finish = resolve + }) + vi.mocked(useAgentConsent().withConsent).mockImplementationOnce( + async () => { + await pending + } + ) + + await loadEntryAndSetup() + await vi.waitFor(() => + expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() + ) + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'offer_in_flight', + stage: 'offer', + retry_armed: false + }) + finish() + await pending + }) + + it('reports an offer skipped because the card has already been on screen', async () => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + openDialog(CONSENT_DIALOG_KEY) + await flush() + closeDialog(CONSENT_DIALOG_KEY) + await rerunFlagGate() + await flush() + + expect(await offerExited()).toHaveBeenCalledWith({ + exit: 'card_already_seen', + stage: 'offer', + retry_armed: false + }) + }) + + it.for([ + { + exit: 'consent_already_accepted', + stage: 'load', + arrange: () => { + vi.mocked(consentStore.load).mockResolvedValue(true) + } + }, + { + exit: 'consent_read_failed', + stage: 'load', + arrange: () => { + vi.mocked(consentStore.load).mockRejectedValue(new Error('offline')) + } + }, + { + exit: 'startup_probe_failed', + stage: 'startup', + arrange: () => { + const rejected = Promise.reject(new Error('startup blew up')) + // Marked handled here so the rejection is not unhandled during the + // window before the extension attaches its own catch. + rejected.catch(() => {}) + startupDecision = rejected + } + } + ] as const)( + 'reports $exit at the $stage stage', + async ({ exit, stage, arrange }) => { + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + arrange() + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit, + stage, + retry_armed: false + }) + } + ) + + it('reports why an undecided boot was also ineligible', async () => { + // `boot_undecided` is gated on eligibility, so this combination used to + // forfeit the offer and report nothing at all. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let decide = (_: boolean) => {} + startupDecision = new Promise((resolve) => { + decide = resolve + }) + + await loadEntryAndSetup() + await flush() + Object.assign(consentStore, { isChecking: true }) + decide(false) + await flush() + + expect(await notOffered()).not.toHaveBeenCalled() + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_unresolved', + stage: 'startup', + retry_armed: false + }) + }) + + it('marks the exit that loses a held offer as still having a retry armed', async () => { + // This is gc-17's silent loss, made visible: a dialog holds the offer, + // the screen clears, and the retry's consent read fails. Before #18975 + // the offer was gone for the page load; after it the hold survives - and + // either way the only telemetry was a first `dialog_open`, because the + // reason is deduplicated per page load. + agentFlagEnabled.value = true + openDialog() + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + await flush() + expect(await notOffered()).toHaveBeenCalledExactlyOnceWith({ + reason: 'dialog_open' + }) + expect(await offerExited()).not.toHaveBeenCalled() + + vi.mocked(consentStore.load).mockRejectedValueOnce(new Error('offline')) + closeDialog() + await flush() + await flush() + + expect(await notOffered()).toHaveBeenCalledOnce() + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_read_failed', + stage: 'load', + retry_armed: true + }) + }) + + it('reports nothing for an unflagged page load', async () => { + // Not because the reporter is gated on the flag - it is not - but because + // `loadConsentIfEligible` returns on an off flag before reaching any + // reported exit. That is the whole mechanism keeping this event off the + // population outside the rollout, so it is asserted rather than assumed. + agentFlagEnabled.value = false + Object.assign(consentStore, { accepted: false, isChecking: false }) + vi.mocked(consentStore.load).mockRejectedValue(new Error('offline')) + + await loadEntryAndSetup() + await flush() + + expect(await offerExited()).not.toHaveBeenCalled() + expect(consentStore.load).not.toHaveBeenCalled() + }) + + it('still reports why the offer ended when the flag goes off mid-flight', async () => { + // The flag was on when the offer started, so this attempt is already + // under way and its ending is real. A reporter gated on the current flag + // value would hide exactly this case, which is why it is not. + agentFlagEnabled.value = true + Object.assign(consentStore, { accepted: false, isChecking: false }) + let fail = (_: Error) => {} + vi.mocked(consentStore.load).mockReturnValueOnce( + new Promise((_, reject) => { + fail = reject + }) + ) + + await loadEntryAndSetup() + await vi.waitFor(() => expect(consentStore.load).toHaveBeenCalled()) + agentFlagEnabled.value = false + await nextTick() + await flush() + fail(new Error('offline')) + await flush() + + expect(agentStore.enabled).toBe(false) + expect(await offerExited()).toHaveBeenCalledExactlyOnceWith({ + exit: 'consent_read_failed', + stage: 'load', + retry_armed: false + }) + }) + }) + + it('keeps waiting when a tour ends while Getting Started is still up', async () => { + agentFlagEnabled.value = true + screenShown() + activeTour.value = 'appMode' + Object.assign(consentStore, { accepted: false, isChecking: false }) + + await loadEntryAndSetup() + await nextTick() + await flush() + activeTour.value = null + await flush() + + expect(useAgentConsent().withConsent).not.toHaveBeenCalled() + expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBeNull() + + screenClosed() + await vi.waitFor(() => + expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() + ) + }) + it('offers in the same session once the dialog that held it closes', async () => { agentFlagEnabled.value = true openDialog() diff --git a/src/extensions/core/agentPanel.ts b/src/extensions/core/agentPanel.ts index 1a4e55f09f7..60917504007 100644 --- a/src/extensions/core/agentPanel.ts +++ b/src/extensions/core/agentPanel.ts @@ -11,7 +11,11 @@ import { } from '@/platform/remoteConfig/remoteConfig' import { useTelemetry } from '@/platform/telemetry' import { reportError } from '@/platform/telemetry/reportError' -import type { AgentConsentNotOfferedReason } from '@/platform/telemetry/types' +import type { + AgentConsentNotOfferedReason, + AgentConsentOfferExit, + AgentConsentOfferStage +} from '@/platform/telemetry/types' import { useTeamWorkspaceStore } from '@/platform/workspace/stores/teamWorkspaceStore' import { useFirstRunEntry } from '@/renderer/extensions/firstRunTour/gettingStarted/firstRunEntry' import { @@ -236,6 +240,48 @@ export function registerAgentPanelExtension(): void { offerHeld.value = false } + /** + * Names the endings that `agent_consent_not_offered` cannot: that event + * reports the surface a deferred offer is queued behind, so an attempt + * that stops for a reason of its own emits nothing at all today and is + * indistinguishable in telemetry from a user who was never offered. + * + * Three things this deliberately does not route through `withholdOffer`. + * It returns without emitting when either id is missing - and a missing + * id is one of the endings that has to be reportable. It returns without + * emitting once the auto-show key is burned, which would silence every + * later page load of every user who has seen the card. And its reason + * enum means "a surface is holding the offer", which none of these are. + * + * Dedup is per page load and per (stage, exit), so this reports the + * *presence* of an ending, never its frequency - `loadConsentIfEligible` + * is re-driven by the identity watcher, the flag gate, the release + * watcher and `withConsent`'s settlement, so an undeduplicated count + * would measure how long the tab was open. + * + * There is no "the flag is off" guard here, and that is deliberate rather + * than an omission. `loadConsentIfEligible` is the only way in, and it + * returns on an off flag before reaching any ending below - so the guard + * would have suppressed nothing except the one case worth seeing: a flag + * that flips off *mid-page-load*, abandoning an offer that was already + * under way. The unflagged population is kept out by the exit that is + * deliberately not reported at all, not by a gate here. + */ + const reportedExits = new Set() + const reportOfferExit = ( + exit: AgentConsentOfferExit, + stage: AgentConsentOfferStage + ): void => { + const key = `${stage}:${exit}` + if (reportedExits.has(key)) return + reportedExits.add(key) + useTelemetry()?.trackAgentConsentOfferExited({ + exit, + stage, + retry_armed: offerHeld.value + }) + } + const consentScope = (): string | null => { const userId = resolvedUserInfo.value?.id const workspaceId = workspaceStore.activeWorkspaceId @@ -255,19 +301,43 @@ export function registerAgentPanelExtension(): void { isLoggedIn.value && !consentStore.isChecking && !consentStore.accepted + /** + * Which of `offerEligible`'s conditions is unmet, in the order it checks + * them. Only meaningful when `offerEligible()` is false, and deliberately + * a separate expression from it: naming the cause must not be able to + * change the decision. + * + * `enabled` is not covered, so this is null when the flag being off is the + * only thing unmet: an off flag has no value on this event, for the reason + * on `AgentConsentOfferExit`. + */ + const offerIneligibility = (): AgentConsentOfferExit | null => + !isLoggedIn.value + ? 'signed_out' + : consentStore.isChecking + ? 'consent_unresolved' + : consentStore.accepted + ? 'consent_already_accepted' + : null let autoShowInFlight = false const offerConsentUnprompted = (): void => { // An exit that leaves the hold armed does so on purpose: the condition // is transient, so the offer is still owed and the next clear screen // has to retry it. `dropHold` marks the exits that are not transient. - if (autoShowInFlight) return + if (autoShowInFlight) { + reportOfferExit('offer_in_flight', 'offer') + return + } if (!offerEligible()) { + const ineligible = offerIneligibility() + if (ineligible) reportOfferExit(ineligible, 'offer') if (consentStore.accepted) dropHold() return } const scope = consentScope() if (scope && consentCardSeenIn.has(scope)) { + reportOfferExit('card_already_seen', 'offer') dropHold() return } @@ -280,11 +350,29 @@ export function registerAgentPanelExtension(): void { const userId = resolvedUserInfo.value?.id const workspaceId = workspaceStore.activeWorkspaceId - if (!userId || !workspaceId || workspaceStore.isSwitching) return + if (!userId || !workspaceId || workspaceStore.isSwitching) { + // A switch in progress is reported ahead of the ids it is moving: it + // explains an absent workspace id, so labelling that case + // `workspace_unresolved` would hide the more specific cause. The + // condition above is unchanged either way - only the label differs. + reportOfferExit( + workspaceStore.isSwitching + ? 'workspace_switching' + : !userId + ? 'account_unresolved' + : 'workspace_unresolved', + 'offer' + ) + return + } const key = `${CONSENT_AUTO_SHOWN_PREFIX}.${userId}.${workspaceId}` const autoShow = prepareAutoShow(key) + // `storage_unavailable` already has a reason on + // `agent_consent_not_offered`, so it is not reported twice. if (autoShow === 'storage_unavailable') withholdOffer(autoShow) if (autoShow !== 'ready') { + if (autoShow === 'already_offered') + reportOfferExit('already_offered', 'offer') dropHold() return } @@ -332,8 +420,16 @@ export function registerAgentPanelExtension(): void { .then((decided) => { if (decided) offerConsentUnprompted() else if (offerEligible()) withholdOffer('boot_undecided') + else { + // An undecided boot that is also ineligible reported nothing at + // all: `boot_undecided` is gated on eligibility, so the forfeited + // offer looked identical to one that was never owed. + const ineligible = offerIneligibility() + if (ineligible) reportOfferExit(ineligible, 'startup') + } }) .catch((error: unknown) => { + reportOfferExit('startup_probe_failed', 'startup') reportError(error, { errorType: 'agent_consent_auto_offer_failure' }) @@ -341,14 +437,27 @@ export function registerAgentPanelExtension(): void { } const loadConsentIfEligible = (): void => { + // Neither of these reports. An off flag is not this event's business + // (see `reportOfferExit`), and an account that has not resolved *yet* is + // not a lost offer: the watcher below re-drives this the moment + // `resolvedUserInfo` changes, so the exit has a guaranteed wake-up. It + // would otherwise fire on essentially every flagged page load, because + // the flag gate calls this before the auth rail settles - and a value + // that is present almost always discriminates nothing while looking + // like the dominant cause. `account_unresolved` is reported only at the + // `offer` stage, where the id went missing *after* a consent read had + // already succeeded with it. if (!agentPanelStore.enabled || !resolvedUserInfo.value) return void consentStore .load() .then((isAccepted) => { - if (isAccepted) dropHold() - else offerWhenStartupDecided() + if (isAccepted) { + reportOfferExit('consent_already_accepted', 'load') + dropHold() + } else offerWhenStartupDecided() }) .catch((error: unknown) => { + reportOfferExit('consent_read_failed', 'load') reportError(error, { errorType: 'agent_consent_setting_load_failure' }) diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index f79bb5b8143..641155058b3 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -469,6 +469,20 @@ describe('TelemetryRegistry', () => { invoke: (registry) => registry.trackAgentConsentNotOffered({ reason: 'first_run_screen' }) }, + { + method: 'trackAgentConsentOfferExited', + expected: { + exit: 'already_offered', + stage: 'offer', + retry_armed: false + }, + invoke: (registry) => + registry.trackAgentConsentOfferExited({ + exit: 'already_offered', + stage: 'offer', + retry_armed: false + }) + }, { method: 'trackAgentOnboardingNotShown', expected: { reason: 'app_mode' }, diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index c92b6344ed8..1f27978371e 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -2,6 +2,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentPaywallCtaMetadata, @@ -469,6 +470,14 @@ export class TelemetryRegistry implements TelemetryDispatcher { ) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.dispatch((provider) => + provider.trackAgentConsentOfferExited?.(metadata) + ) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index 4bb635c90b7..53e249922d4 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -793,6 +793,20 @@ describe('PostHogTelemetryProvider', () => { provider.trackAgentConsentNotOffered({ reason: 'tour_active' }), properties: { reason: 'tour_active' } }, + { + event: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (provider: PostHogTelemetryProvider) => + provider.trackAgentConsentOfferExited({ + exit: 'consent_unresolved', + stage: 'load', + retry_armed: false + }), + properties: { + exit: 'consent_unresolved', + stage: 'load', + retry_armed: false + } + }, { event: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (provider: PostHogTelemetryProvider) => diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index 331cfff0fd5..eacd658390f 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -16,6 +16,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentEntryButtonClickedMetadata, @@ -830,6 +831,12 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_CONSENT_NOT_OFFERED, metadata) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.trackEvent(TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, metadata) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index 7a20a9d89f2..d572cd212ff 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -421,6 +421,20 @@ describe('HostTelemetrySink', () => { sink.trackAgentConsentNotOffered({ reason: 'tour_active' }), properties: { reason: 'tour_active' } }, + { + name: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (sink: HostTelemetrySink) => + sink.trackAgentConsentOfferExited({ + exit: 'workspace_switching', + stage: 'offer', + retry_armed: true + }), + properties: { + exit: 'workspace_switching', + stage: 'offer', + retry_armed: true + } + }, { name: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.ts index 39fa501b585..53c50146411 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.ts @@ -6,6 +6,7 @@ import type { AddCreditsClickMetadata, AgentAttachButtonClickedMetadata, AgentConsentNotOfferedMetadata, + AgentConsentOfferExitedMetadata, AgentConsentResolvedMetadata, AgentConsentShownMetadata, AgentEntryButtonClickedMetadata, @@ -408,6 +409,12 @@ export class HostTelemetrySink implements TelemetryProvider { this.capture(TelemetryEvents.AGENT_CONSENT_NOT_OFFERED, metadata) } + trackAgentConsentOfferExited( + metadata: AgentConsentOfferExitedMetadata + ): void { + this.capture(TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, metadata) + } + trackAgentOnboardingNotShown( metadata: AgentOnboardingNotShownMetadata ): void { diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 3dc1b30bc51..232748c0636 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -640,6 +640,75 @@ export interface AgentConsentNotOfferedMetadata extends Record< > { reason: AgentConsentNotOfferedReason } +/** + * Why an automatic consent offer ended without either making the offer or + * naming a surface that is holding it. + * + * `AgentConsentNotOfferedReason` covers the deferrals: a surface is in the way, + * it is named, and the offer is retried when that surface clears. Everything + * here is the other kind of ending - the attempt stopped for a reason of its + * own. Some of those endings are correct (the offer was not needed) and some + * are losses (it was owed and did not happen), so **a query over this event + * must split by `exit`; a total is not a quantity.** + * + * Correctly not needed: `consent_already_accepted`, `card_already_seen`, + * `already_offered`. + * Owed and not made: everything else. + * + * There is deliberately no value for "the agent flag is off". Every exit here + * is downstream of that check, so reporting it would emit once per page load + * for everyone outside the rollout - a count of exposure rather than of the + * mechanism - and the flag is already on every event as + * `$feature/agent-in-app-experience`. + */ +export type AgentConsentOfferExit = + /** No Comfy account is signed in. */ + | 'signed_out' + /** Signed in, but the account has not resolved to a user id yet. */ + | 'account_unresolved' + /** No active workspace id yet, and no switch is in progress. */ + | 'workspace_unresolved' + /** A workspace switch is in progress, so the consent scope is moving. */ + | 'workspace_switching' + /** The stored consent read has not settled, so consent is unknown. */ + | 'consent_unresolved' + /** The stored consent read rejected. */ + | 'consent_read_failed' + /** Consent is already stored for this scope, so no card is needed. */ + | 'consent_already_accepted' + /** Another offer attempt for this page load has not finished. */ + | 'offer_in_flight' + /** The card has already been on screen for this scope this page load. */ + | 'card_already_seen' + /** The one-shot auto-show key for this scope is already burned. */ + | 'already_offered' + /** The first-run startup probe rejected. */ + | 'startup_probe_failed' +/** + * Which link in the offer chain exited. The same condition is checked at more + * than one of these - the pair (`exit`, `stage`) is what identifies a single + * exit in the code, so neither property is readable on its own. + */ +export type AgentConsentOfferStage = + /** `loadConsentIfEligible` - before the consent read, or on its result. */ + | 'load' + /** Waiting on the first-run startup decision. */ + | 'startup' + /** `offerConsentUnprompted` - the offer attempt itself. */ + | 'offer' +export interface AgentConsentOfferExitedMetadata extends Record< + string, + unknown +> { + exit: AgentConsentOfferExit + stage: AgentConsentOfferStage + /** + * Whether a hold was armed at the moment of the exit, i.e. whether this page + * load still has a queued retry. False on an owed-and-not-made exit means the + * offer is gone for this page load with nothing scheduled to bring it back. + */ + retry_armed: boolean +} export type AgentOnboardingNotShownMetadata = | { reason: 'app_mode' | 'tour_active' } | { reason: 'target_missing'; step: number } @@ -1541,6 +1610,7 @@ export interface TelemetryProvider { trackAgentRunModeChanged?(metadata: AgentRunModeChangedMetadata): void trackAgentThreadStarted?(metadata: AgentThreadStartedMetadata): void trackAgentConsentNotOffered?(metadata: AgentConsentNotOfferedMetadata): void + trackAgentConsentOfferExited?(metadata: AgentConsentOfferExitedMetadata): void trackAgentOnboardingNotShown?(metadata: AgentOnboardingNotShownMetadata): void // Right side panel widget favorite events @@ -1736,6 +1806,7 @@ export const TelemetryEvents = { AGENT_RUN_MODE_CHANGED: 'app:agent_run_mode_changed', AGENT_THREAD_STARTED: 'app:agent_thread_started', AGENT_CONSENT_NOT_OFFERED: 'app:agent_consent_not_offered', + AGENT_CONSENT_OFFER_EXITED: 'app:agent_consent_offer_exited', AGENT_ONBOARDING_NOT_SHOWN: 'app:agent_onboarding_not_shown', // Right Side Panel Widget Favorites From 614bc9de68370ece633b45cb9ca34b4bdd6341c9 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sat, 26 Sep 2026 02:20:25 -0700 Subject: [PATCH 3/5] refactor(agent): lift the exit-naming decisions out of the offer body Naming each silent exit put three more branches inside `offerConsentUnprompted`, which took its cognitive complexity over the threshold - `fallow audit` fails on newly introduced findings and this was one. No behaviour change. The missing-scope label moves to `missingScopeExit`, and the two non-`ready` auto-show outcomes become one if/else instead of three separate conditions, which is the shape the code had before the reporting was added. --- src/extensions/core/agentPanel.test.ts | 21 --------------- src/extensions/core/agentPanel.ts | 37 ++++++++++++++------------ 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/src/extensions/core/agentPanel.test.ts b/src/extensions/core/agentPanel.test.ts index e5ff4f57ffe..3dcb70d67c3 100644 --- a/src/extensions/core/agentPanel.test.ts +++ b/src/extensions/core/agentPanel.test.ts @@ -1066,27 +1066,6 @@ describe('AgentPanel extension flag gate', () => { }) }) - it('keeps waiting when a tour ends while Getting Started is still up', async () => { - agentFlagEnabled.value = true - screenShown() - activeTour.value = 'appMode' - Object.assign(consentStore, { accepted: false, isChecking: false }) - - await loadEntryAndSetup() - await nextTick() - await flush() - activeTour.value = null - await flush() - - expect(useAgentConsent().withConsent).not.toHaveBeenCalled() - expect(localStorage.getItem(AUTO_SHOWN_KEY)).toBeNull() - - screenClosed() - await vi.waitFor(() => - expect(useAgentConsent().withConsent).toHaveBeenCalledOnce() - ) - }) - it('offers in the same session once the dialog that held it closes', async () => { agentFlagEnabled.value = true openDialog() diff --git a/src/extensions/core/agentPanel.ts b/src/extensions/core/agentPanel.ts index 60917504007..7ad5d485926 100644 --- a/src/extensions/core/agentPanel.ts +++ b/src/extensions/core/agentPanel.ts @@ -319,6 +319,20 @@ export function registerAgentPanelExtension(): void { : consentStore.accepted ? 'consent_already_accepted' : null + /** + * Why the offer has no consent scope to work with. A switch in progress is + * reported ahead of the ids it is moving, because it explains an absent + * workspace id and labelling that case `workspace_unresolved` would hide + * the more specific cause. + */ + const missingScopeExit = ( + userId: string | undefined + ): AgentConsentOfferExit => + workspaceStore.isSwitching + ? 'workspace_switching' + : !userId + ? 'account_unresolved' + : 'workspace_unresolved' let autoShowInFlight = false const offerConsentUnprompted = (): void => { @@ -351,28 +365,17 @@ export function registerAgentPanelExtension(): void { const userId = resolvedUserInfo.value?.id const workspaceId = workspaceStore.activeWorkspaceId if (!userId || !workspaceId || workspaceStore.isSwitching) { - // A switch in progress is reported ahead of the ids it is moving: it - // explains an absent workspace id, so labelling that case - // `workspace_unresolved` would hide the more specific cause. The - // condition above is unchanged either way - only the label differs. - reportOfferExit( - workspaceStore.isSwitching - ? 'workspace_switching' - : !userId - ? 'account_unresolved' - : 'workspace_unresolved', - 'offer' - ) + reportOfferExit(missingScopeExit(userId), 'offer') return } const key = `${CONSENT_AUTO_SHOWN_PREFIX}.${userId}.${workspaceId}` const autoShow = prepareAutoShow(key) - // `storage_unavailable` already has a reason on - // `agent_consent_not_offered`, so it is not reported twice. - if (autoShow === 'storage_unavailable') withholdOffer(autoShow) if (autoShow !== 'ready') { - if (autoShow === 'already_offered') - reportOfferExit('already_offered', 'offer') + // `storage_unavailable` already has a reason on + // `agent_consent_not_offered`, so only the burned one-shot key needs + // naming here. + if (autoShow === 'storage_unavailable') withholdOffer(autoShow) + else reportOfferExit('already_offered', 'offer') dropHold() return } From 0b19c9dd3f45cf01d7db086a615e727edede6150 Mon Sep 17 00:00:00 2001 From: christian-byrne Date: Sat, 26 Sep 2026 04:14:50 -0700 Subject: [PATCH 4/5] feat(agent): name every ending of a consent request and every outcome of a shown card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `requestConsentForCurrentUser` returned null without emitting on three paths, and a card dismissed with Escape or the overlay mask emitted `agent_consent_shown` and then nothing — so "shown and walked away" was indistinguishable from "shown and the accept failed". In the last three days 5 of 26 card impressions ended with no outcome at all, and the silent catch fired 23 times for 9 people, visible in Sentry and invisible to product analytics because `reportError` has no PostHog sink. `app:agent_consent_offer_exited` gains a fourth stage, `request`, with four endings: `scope_changed_before_read`, `scope_probe_failed`, `scope_changed_after_read`, and `card_closed_before_mount`, plus the existing `consent_read_failed` reused for the same condition one link later. The catch tracks which call was in flight because it cannot be read from the error: `ensureScope` and the read's `requireAuthHeader` raise the same `AgentConsentAuthenticationError` message, and 6 of those 23 production events are that message. A `trigger` property is set only at this stage, which is the only one reachable from the topbar button as well as the automatic offer. `app:agent_consent_resolved` becomes total over shown cards: `dismissed`, `accept_not_persisted` (the acceptance resolved false without raising, so no error is shown and the user believes they consented) and `accepted_pending_sign_in` join the two deciding values, with `save_error_shown` separating a give-up after a failed save from a walk-away. Only `accepted` still means consent is stored, and the type says so: a query that counted this event as a decision that stuck must now filter on it. Consent semantics are unchanged — same copy, same stored decision, still required before the panel opens. --- .../fixtures/agentConsentTelemetryFixture.ts | 86 +++++++ .../tests/agent/agentConsentOutcome.spec.ts | 197 ++++++++++++++++ .../telemetry/TelemetryRegistry.test.ts | 19 +- .../cloud/PostHogTelemetryProvider.test.ts | 26 ++- .../providers/host/HostTelemetrySink.test.ts | 23 +- src/platform/telemetry/types.ts | 118 +++++++++- .../composables/agent/useAgentConsent.test.ts | 211 +++++++++++++++++- .../composables/agent/useAgentConsent.ts | 133 ++++++++++- 8 files changed, 781 insertions(+), 32 deletions(-) create mode 100644 browser_tests/fixtures/agentConsentTelemetryFixture.ts create mode 100644 browser_tests/tests/agent/agentConsentOutcome.spec.ts diff --git a/browser_tests/fixtures/agentConsentTelemetryFixture.ts b/browser_tests/fixtures/agentConsentTelemetryFixture.ts new file mode 100644 index 00000000000..7402e0eb362 --- /dev/null +++ b/browser_tests/fixtures/agentConsentTelemetryFixture.ts @@ -0,0 +1,86 @@ +import type { RemoteConfig } from '@/platform/remoteConfig/types' + +import { agentConsentTest } from '@e2e/fixtures/agentConsentFixture' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +export interface CapturedTelemetryEvent { + event: string + properties: Record +} + +/** + * Reads the consent surface's telemetry from inside a real browser. + * + * It captures at the **host bridge**, not over the network. `main.ts` registers + * the host sink after the cloud one when `window.__comfyDesktop2.Telemetry` + * exists, and `setTelemetryRegistry` keeps the last registry - so every + * `useTelemetry()` call in the app lands here synchronously, with no batching, + * no compression and no third-party host to decode. Cloud plus a host bridge is + * a configuration that exists in production (`main.ts` has an + * `isCloud && hasHostTelemetryBridge` branch of its own), not one fabricated for + * the test. + * + * What that does and does not prove: the browser proves which events a real + * interaction produces and with which properties, through the same + * `TelemetryRegistry` the PostHog provider is registered on. The mapping from a + * registry call to a PostHog capture is asserted by + * `PostHogTelemetryProvider.test.ts`, not here. + * + * PostHog still initializes, because the agent feature flag is bootstrapped + * through it; its ingest host is routed so nothing leaves the browser and the + * network-isolation fixture stays satisfied. + */ +export const agentConsentTelemetryTest = agentConsentTest.extend<{ + consentTelemetry: CapturedTelemetryEvent[] +}>({ + consentTelemetry: async ({ agentFlagEnabled: _agentFlagEnabled }, use) => { + await use([]) + }, + page: async ({ page, consentTelemetry }, use) => { + await page.exposeFunction( + '__captureHostTelemetry', + (captured: CapturedTelemetryEvent) => { + consentTelemetry.push(captured) + } + ) + await page.addInitScript(() => { + // Declared before the app boots so `hasHostTelemetryBridge` sees it. + Object.assign(window, { + __comfyDesktop2: { + isRemote: () => false, + Telemetry: { + capture: (event: string, properties: Record) => { + void ( + window as unknown as { + __captureHostTelemetry: ( + captured: CapturedTelemetryEvent + ) => Promise + } + ).__captureHostTelemetry({ event, properties }) + } + } + } + }) + }) + // Layered over the fixture's own `/api/features`, which the later route + // wins. `enable_telemetry` is what gates the host sink; the PostHog values + // repeat the fixture's because the agent flag is bootstrapped through them. + const features: RemoteConfig = { + enable_telemetry: true, + posthog_project_token: 'phc_e2e_agent_consent_outcome', + posthog_config: { + advanced_disable_flags: true, + bootstrap: { + featureFlags: { 'agent-in-app-experience': true } + } + } + } + await page.route('**/api/features', (route) => + route.fulfill(jsonRoute(features)) + ) + await page.route('**://t.comfy.org/**', (route) => + route.fulfill(jsonRoute({ status: 1 })) + ) + await use(page) + } +}) diff --git a/browser_tests/tests/agent/agentConsentOutcome.spec.ts b/browser_tests/tests/agent/agentConsentOutcome.spec.ts new file mode 100644 index 00000000000..f69ccb9564e --- /dev/null +++ b/browser_tests/tests/agent/agentConsentOutcome.spec.ts @@ -0,0 +1,197 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { AGENT_CONSENT_SETTING_ID } from '@/platform/settings/constants/agent' + +import { agentConsentTelemetryTest as test } from '@e2e/fixtures/agentConsentTelemetryFixture' + +/** + * A card that was shown and then went quiet used to cover three endings at + * once - dismissed, an acceptance whose save did not stick, and (signed out) an + * abandoned sign-in. `useAgentConsent.test.ts` pins all of them, but it arranges + * a dismissal by calling the dialog's `onClose` prop directly. This file is here + * because that is the one arrange a unit test cannot make faithfully: whether a + * real Escape key or a real overlay click reaches that handler at all depends on + * the dialog renderer, the dialog stack's escape bookkeeping and the overlay's + * own hit area, none of which a hand-called callback exercises. + */ +test.describe( + 'Automatic consent card outcome', + { tag: ['@cloud', '@ui'] }, + () => { + test.use({ agentConsentAccepted: false }) + + test('reports a dismissal when Escape closes the automatic card', async ({ + comfyPage, + agentPanel, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await test.step('The automatic offer reports exactly one impression', async () => { + await expect(dialog).toBeVisible() + await expect + .poll(() => + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_shown' + ) + ) + .toEqual([ + { + event: 'app:agent_consent_shown', + properties: expect.objectContaining({ trigger: 'first_load' }) + } + ]) + }) + + await test.step('Escape reports the dismissal as its own outcome', async () => { + expect( + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ).toEqual([]) + + await page.keyboard.press('Escape') + await expect(dialog).toHaveCount(0) + + await expect + .poll(() => + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ) + .toEqual([ + { + event: 'app:agent_consent_resolved', + properties: expect.objectContaining({ + decision: 'dismissed', + save_error_shown: false + }) + } + ]) + // Consent semantics are unchanged: a dismissal is still not a decision. + expect(agentConsentWrites).toHaveLength(0) + await expect(agentPanel.root).toHaveCount(0) + }) + + await test.step('One outcome per impression, not one per close path', async () => { + // The dialog closes through `closeDialog`, which fires `onClose`, and + // the promise settles separately. Both ran; only one event exists. + expect( + consentTelemetry.filter( + (e) => e.event === 'app:agent_consent_resolved' + ) + ).toHaveLength(1) + }) + }) + + test('reports the same outcome when the overlay mask closes the card', async ({ + comfyPage, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await expect(dialog).toBeVisible() + await page + .getByTestId('dialog-overlay') + .click({ position: { x: 1, y: 1 } }) + await expect(dialog).toHaveCount(0) + + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_resolved') + .map((e) => e.properties.decision) + ) + .toEqual(['dismissed']) + expect(agentConsentWrites).toHaveLength(0) + }) + + test('reports an acceptance, not a dismissal, when the card is accepted', async ({ + comfyPage, + agentPanel, + agentConsentWrites, + consentTelemetry + }) => { + const page = comfyPage.page + const dialog = page.getByRole('dialog', { + name: enMessages.agent.consent.title + }) + + await expect(dialog).toBeVisible() + await dialog + .getByRole('button', { name: enMessages.agent.consent.accept }) + .click() + await expect(dialog).toHaveCount(0) + await expect(agentPanel.root).toBeVisible() + + // `closeWith` settles before closing the dialog, so the accepted outcome + // must not be followed by a dismissal from the close it performs itself. + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_resolved') + .map((e) => e.properties.decision) + ) + .toEqual(['accepted']) + expect(agentConsentWrites).toEqual([true]) + }) + + test('names a failed consent read at the request, not just at the load', async ({ + comfyPage, + agentPanel, + consentTelemetry + }) => { + const page = comfyPage.page + // Every read fails. The automatic chain exits at the `load` stage without + // asking, which `gc-16`'s instrument already names; the user then clicks + // the entry button, and that attempt fails inside the request. This is the + // shape behind the 23 `agent_consent_setting_load_failure` events in + // Sentry that product analytics saw nothing of. + await page.route( + `**/api/global-settings/${AGENT_CONSENT_SETTING_ID}`, + (route) => route.fulfill({ status: 500, body: '{"code":"INTERNAL"}' }) + ) + await comfyPage.workflow.reloadAndWaitForApp() + + await expect(agentPanel.openButton).toBeEnabled() + await agentPanel.openButton.click() + + await expect + .poll(() => + consentTelemetry + .filter((e) => e.event === 'app:agent_consent_offer_exited') + .map((e) => [e.properties.stage, e.properties.exit]) + ) + .toContainEqual(['request', 'consent_read_failed']) + + // The same condition at two links, told apart by the pair rather than by + // the reason value - which is what `exit` plus `stage` is for. + const requestExits = consentTelemetry.filter( + (e) => + e.event === 'app:agent_consent_offer_exited' && + e.properties.stage === 'request' + ) + expect(requestExits.map((e) => e.properties)).toEqual([ + { + exit: 'consent_read_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ]) + await expect( + page.getByRole('dialog', { name: enMessages.agent.consent.title }) + ).toHaveCount(0) + await expect(agentPanel.root).toHaveCount(0) + }) + } +) diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index 641155058b3..8dfc1d5a3b3 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -304,7 +304,8 @@ describe('TelemetryRegistry', () => { trigger: 'button_click' } satisfies AgentConsentShownMetadata const consentResolvedMetadata = { - decision: 'accepted' + decision: 'dismissed', + save_error_shown: true } satisfies AgentConsentResolvedMetadata const onboardingStepMetadata = { step: 2, @@ -483,6 +484,22 @@ describe('TelemetryRegistry', () => { retry_armed: false }) }, + { + method: 'trackAgentConsentOfferExited', + expected: { + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + }, + invoke: (registry) => + registry.trackAgentConsentOfferExited({ + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + }) + }, { method: 'trackAgentOnboardingNotShown', expected: { reason: 'app_mode' }, diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index 53e249922d4..35896361557 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -440,13 +440,19 @@ describe('PostHogTelemetryProvider', () => { await vi.dynamicImportSettled() provider.trackAgentConsentShown({ trigger: 'first_load' }) - provider.trackAgentConsentResolved({ decision: 'accepted' }) + provider.trackAgentConsentResolved({ + decision: 'dismissed', + save_error_shown: true + }) provider.trackAgentOnboardingShown() provider.trackAgentOnboardingStep({ step: 4, action: 'finish' }) expect(hoisted.mockCapture.mock.calls).toEqual([ [TelemetryEvents.AGENT_CONSENT_SHOWN, { trigger: 'first_load' }], - [TelemetryEvents.AGENT_CONSENT_RESOLVED, { decision: 'accepted' }], + [ + TelemetryEvents.AGENT_CONSENT_RESOLVED, + { decision: 'dismissed', save_error_shown: true } + ], [TelemetryEvents.AGENT_ONBOARDING_SHOWN, {}], [TelemetryEvents.AGENT_ONBOARDING_STEP, { step: 4, action: 'finish' }] ]) @@ -807,6 +813,22 @@ describe('PostHogTelemetryProvider', () => { retry_armed: false } }, + { + event: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (provider: PostHogTelemetryProvider) => + provider.trackAgentConsentOfferExited({ + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + }), + properties: { + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + }, { event: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (provider: PostHogTelemetryProvider) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index d572cd212ff..b17e5971852 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -317,8 +317,11 @@ describe('HostTelemetrySink', () => { { name: TelemetryEvents.AGENT_CONSENT_RESOLVED, track: (sink: HostTelemetrySink) => - sink.trackAgentConsentResolved({ decision: 'accepted' }), - properties: { decision: 'accepted' } + sink.trackAgentConsentResolved({ + decision: 'dismissed', + save_error_shown: true + }), + properties: { decision: 'dismissed', save_error_shown: true } }, { name: TelemetryEvents.AGENT_ONBOARDING_SHOWN, @@ -435,6 +438,22 @@ describe('HostTelemetrySink', () => { retry_armed: true } }, + { + name: TelemetryEvents.AGENT_CONSENT_OFFER_EXITED, + track: (sink: HostTelemetrySink) => + sink.trackAgentConsentOfferExited({ + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + }), + properties: { + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + }, { name: TelemetryEvents.AGENT_ONBOARDING_NOT_SHOWN, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 232748c0636..47add42fef7 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -660,6 +660,11 @@ export interface AgentConsentNotOfferedMetadata extends Record< * for everyone outside the rollout - a count of exposure rather than of the * mechanism - and the flag is already on every event as * `$feature/agent-in-app-experience`. + * + * The `request` stage is the one place these endings are *countable*: see + * `AgentConsentOfferStage`. Every other stage reports the presence of an + * ending once per page load, so counts are comparable within one + * (`stage`, `exit`) pair and never across stages. */ export type AgentConsentOfferExit = /** No Comfy account is signed in. */ @@ -684,6 +689,29 @@ export type AgentConsentOfferExit = | 'already_offered' /** The first-run startup probe rejected. */ | 'startup_probe_failed' + /** + * The consent scope moved while it was being resolved - a different account, + * a workspace transition, or a switch that started - so the read that would + * have decided whether to ask was never made. `request` stage only. + */ + | 'scope_changed_before_read' + /** + * Resolving the consent scope raised. The account or the workspace went away + * between the offer decision and the request. `request` stage only. + */ + | 'scope_probe_failed' + /** + * The consent scope moved while the stored-consent read was in flight, so the + * answer that came back belonged to a scope that is no longer current. + * `request` stage only. + */ + | 'scope_changed_after_read' + /** + * The card was asked for and the dialog closed before it rendered, so there + * is no `agent_consent_shown` and no outcome to record against one. `request` + * stage only. + */ + | 'card_closed_before_mount' /** * Which link in the offer chain exited. The same condition is checked at more * than one of these - the pair (`exit`, `stage`) is what identifies a single @@ -696,6 +724,17 @@ export type AgentConsentOfferStage = | 'startup' /** `offerConsentUnprompted` - the offer attempt itself. */ | 'offer' + /** + * The consent request itself - `requestConsentForCurrentUser` and the card + * lifecycle in `showConsentDialog` - after the chain has decided to ask. + * + * The only stage that is **not** deduplicated, because it cannot inflate: + * the automatic path reaches it at most once per consent scope per page load + * (the one-shot auto-show key is burned first) and the button path reaches it + * once per click, so a repeat here is a repeated attempt rather than a + * measure of how long the tab was open. + */ + | 'request' export interface AgentConsentOfferExitedMetadata extends Record< string, unknown @@ -706,8 +745,26 @@ export interface AgentConsentOfferExitedMetadata extends Record< * Whether a hold was armed at the moment of the exit, i.e. whether this page * load still has a queued retry. False on an owed-and-not-made exit means the * offer is gone for this page load with nothing scheduled to bring it back. + * + * Two qualifications, both from the `request` stage. It is **structurally + * false** there: the offer drops the hold immediately before requesting, and + * the only thing that re-arms it is the `canShow` hook, which runs after + * every `request` exit. And the hold is not the only wake-up - `withConsent` + * settling re-drives the chain when the identity changed - so on the two + * `scope_changed_*` exits a retry does happen, by a mechanism this property + * does not describe. */ retry_armed: boolean + /** + * Which surface asked. Set **only** at the `request` stage, which is the one + * stage reachable from the topbar button as well as the automatic offer; + * every other stage is inside `offerConsentUnprompted` and automatic by + * construction. A query about automatic offers must therefore either restrict + * to `stage != 'request'` or filter `trigger = 'first_load'` - counting the + * `request` stage unsplit mixes a user-initiated click into the automatic + * denominator. + */ + trigger?: AgentConsentTrigger } export type AgentOnboardingNotShownMetadata = | { reason: 'app_mode' | 'tour_active' } @@ -727,16 +784,61 @@ export interface AgentConsentShownMetadata extends Record { trigger: AgentConsentTrigger } /** - * Only consent the user actually gave or refused resolves the card, so this - * never reports a decision that did not stick. An `agent_consent_shown` with - * no matching resolution is an *unresolved* offer, not a dismissal: it covers - * dismissing the card (Escape, overlay click), an acceptance whose save - * failed, and — signed out — accepting the card but abandoning the sign-in - * that has to follow. Splitting those three apart needs a signal this event - * does not carry. + * How a consent card that was on screen ended, plus the moment consent becomes + * stored. It used to carry only the two deciding values, which left a card that + * was shown and then went quiet covering three different endings at once - a + * dismissal, an acceptance whose save did not stick, and a signed-out + * acceptance whose sign-in was abandoned. Those are now named, and + * `save_error_shown` separates giving up after a failed save from walking away. + * + * **Only `accepted` means consent is stored.** That was this event's whole + * meaning before the other values existed, so any query that counted it as "a + * decision that stuck" must now filter `decision = 'accepted'`, and one that + * wants "the user answered" wants `decision in ('accepted', 'rejected')`. + * + * **The pairing with `agent_consent_shown`, exactly.** For a signed-in user - + * the whole cloud population - every impression ends in exactly one of these, + * so an impression with no outcome is a defect rather than a dismissal. In the + * signed-out (local) flow the card is only the first half: it ends at + * `accepted_pending_sign_in`, and `accepted` follows separately if the sign-in + * and the real save land. So `accepted` is the one value that is not always the + * card's own ending, which is how it keeps meaning "consent is stored". */ +export type AgentConsentDecision = + /** Accepted, and the acceptance is durably stored. */ + | 'accepted' + /** Declined on the card. Nothing is stored; the card can be offered again. */ + | 'rejected' + /** + * Closed without deciding - Escape, the overlay mask, or a programmatic close + * such as navigation. Distinguishing this from `accept_not_persisted` is the + * difference between the user walking away and the product failing them. + */ + | 'dismissed' + /** + * Accepted, and the write did not stick without raising: the scope moved + * mid-save or the workspace auth header was gone. The user believes they + * consented and no consent exists. + */ + | 'accept_not_persisted' + /** + * Accepted the card in the signed-out flow, where the card is only the first + * half: a sign-in and a real save still have to land before consent exists, + * and they report `accepted` themselves when they do. So this value with no + * later `accepted` is an abandoned sign-in, which is the third case this + * event could not previously name. + */ + | 'accepted_pending_sign_in' export interface AgentConsentResolvedMetadata extends Record { - decision: 'accepted' | 'rejected' + decision: AgentConsentDecision + /** + * Whether the card had already shown a save error when it reached this + * outcome. A raised save leaves the card open and retryable, so its ending is + * one of the values above rather than an outcome of its own - this is what + * tells "dismissed after the save failed" from "dismissed without trying", + * and marks an `accepted` that only landed on a retry. + */ + save_error_shown: boolean } export type AgentOnboardingAction = 'next' | 'finish' | 'skip' /** diff --git a/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts index 3b6d8c4040f..d162ec68984 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentConsent.test.ts @@ -45,7 +45,8 @@ vi.mock(import('@/platform/telemetry/reportError'), () => ({ const telemetry = vi.hoisted(() => ({ trackAgentConsentShown: vi.fn(), - trackAgentConsentResolved: vi.fn() + trackAgentConsentResolved: vi.fn(), + trackAgentConsentOfferExited: vi.fn() })) vi.mock(import('@/platform/telemetry'), () => ({ useTelemetry: () => telemetry @@ -165,7 +166,7 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'rejected' }] + [{ decision: 'rejected', save_error_shown: false }] ]) expect(onOpen).not.toHaveBeenCalled() }) @@ -183,12 +184,12 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'accepted' }] + [{ decision: 'accepted', save_error_shown: false }] ]) expect(onOpen).toHaveBeenCalledOnce() }) - it('reports no decision when the acceptance fails to save', async () => { + it('reports no outcome while a failed save leaves the card open', async () => { fetchWithUnifiedRemint .mockResolvedValueOnce(settingResponse(false)) .mockRejectedValueOnce(new Error('offline')) @@ -202,10 +203,12 @@ describe('useAgentConsent', () => { ) }) + // A raised save is not an ending: the card is still on screen and + // retryable, so the outcome belongs to whatever the user does next. expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() }) - it('reports no decision when the card is dismissed instead of answered', async () => { + it('reports a dismissal as its own outcome, not as an unresolved card', async () => { const request = useAgentConsent().withConsent('button_click', vi.fn()) const dialog = await waitForConsentDialog() await renderConsentCard(dialog) @@ -213,7 +216,77 @@ describe('useAgentConsent', () => { await request expect(telemetry.trackAgentConsentShown).toHaveBeenCalledOnce() + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'dismissed', save_error_shown: false }] + ]) + expect(telemetry.trackAgentConsentOfferExited).not.toHaveBeenCalled() + }) + + it('separates giving up after a failed save from walking away', async () => { + fetchWithUnifiedRemint + .mockResolvedValueOnce(settingResponse(false)) + .mockRejectedValueOnce(new Error('offline')) + const request = useAgentConsent().withConsent('button_click', vi.fn()) + const dialog = await waitForConsentDialog() + await renderConsentCard(dialog) + + await clickCardAction('accept') + await vi.waitFor(() => { + expect(dialog.contentProps.error).toBe( + i18n.global.t('agent.consent.saveError') + ) + }) + ;(dialog.dialogComponentProps.onClose as () => void)() + await request + + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'dismissed', save_error_shown: true }] + ]) + }) + + it('reports an acceptance that did not persist without raising', async () => { + const identity = ref('account-a') + useCurrentUser().resolvedUserInfo = computed(() => ({ id: identity.value })) + const onOpen = vi.fn() + const request = useAgentConsent().withConsent('button_click', onOpen) + const dialog = await waitForConsentDialog() + await renderConsentCard(dialog) + + // The scope moves under the open card, so `accept` resolves false: nothing + // was stored, nothing raised, and no error is shown. The user believes they + // consented. + identity.value = 'account-b' + await clickCardAction('accept') + await request + + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'accept_not_persisted', save_error_shown: false }] + ]) + expect(reportError).not.toHaveBeenCalled() + expect(onOpen).not.toHaveBeenCalled() + }) + + it('reports a card closed before it rendered as an exit, not a dismissal', async () => { + const request = useAgentConsent().withConsent('first_load', vi.fn()) + const dialog = await waitForConsentDialog() + + // No `renderConsentCard`: the async card chunk has not mounted, so there is + // no impression for an outcome to resolve. + ;(dialog.dialogComponentProps.onClose as () => void)() + await request + + expect(telemetry.trackAgentConsentShown).not.toHaveBeenCalled() expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'card_closed_before_mount', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) }) it('waits for the account setting to load before deciding whether to ask', async () => { @@ -258,6 +331,120 @@ describe('useAgentConsent', () => { detail: i18n.global.t('agent.consent.loadError') }) ) + // `reportError` has no product-analytics sink, so before this the funnel saw + // a first-run user who was simply never offered. + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'consent_read_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ] + ]) + }) + + it('separates a failed scope probe from a failed consent read', async () => { + // Signed in, but the account has not resolved to a user id, so `ensureScope` + // raises before any read is attempted. The message it raises is the one + // `load` also raises when its auth header is missing, which is why the + // reason comes from which call was in flight rather than from the error. + useCurrentUser().resolvedUserInfo = computed(() => null) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('button_click', onOpen) + await request + + expect(fetchWithUnifiedRemint).not.toHaveBeenCalled() + expect(useDialogStore().dialogStack).toHaveLength(0) + expect(onOpen).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_probe_failed', + stage: 'request', + retry_armed: false, + trigger: 'button_click' + } + ] + ]) + }) + + it('names the scope moving while it was still being resolved', async () => { + Object.assign(useTeamWorkspaceStore(), { activeWorkspaceId: null }) + vi.mocked(useTeamWorkspaceStore().initialize).mockImplementationOnce( + async () => { + Object.assign(useTeamWorkspaceStore(), { + activeWorkspaceId: 'workspace-b', + workspaceTransitionGeneration: + useTeamWorkspaceStore().workspaceTransitionGeneration + 1 + }) + } + ) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('first_load', onOpen) + await request + + expect(fetchWithUnifiedRemint).not.toHaveBeenCalled() + expect(onOpen).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_changed_before_read', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) + }) + + it('names the scope moving while the consent read was in flight', async () => { + const load = deferred() + fetchWithUnifiedRemint.mockReturnValueOnce(load.promise) + const onOpen = vi.fn() + + const request = useAgentConsent().withConsent('first_load', onOpen) + await vi.waitFor(() => { + expect(fetchWithUnifiedRemint).toHaveBeenCalledOnce() + }) + Object.assign(useTeamWorkspaceStore(), { + activeWorkspaceId: 'workspace-b', + workspaceTransitionGeneration: + useTeamWorkspaceStore().workspaceTransitionGeneration + 1 + }) + load.resolve(settingResponse(false)) + await request + + expect(useDialogStore().dialogStack).toHaveLength(0) + expect(onOpen).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(telemetry.trackAgentConsentOfferExited.mock.calls).toEqual([ + [ + { + exit: 'scope_changed_after_read', + stage: 'request', + retry_armed: false, + trigger: 'first_load' + } + ] + ]) + }) + + it('reports a repeated attempt each time rather than once per page load', async () => { + fetchWithUnifiedRemint.mockRejectedValue(new Error('offline')) + + await useAgentConsent().withConsent('button_click', vi.fn()) + await useAgentConsent().withConsent('button_click', vi.fn()) + + // Not deduplicated, deliberately: the `request` stage is only reachable once + // per consent scope per page load automatically, and once per click from the + // button, so a repeat is a repeated attempt rather than a measure of how + // long the tab was open. + expect(telemetry.trackAgentConsentOfferExited).toHaveBeenCalledTimes(2) }) it('configures the first-use card as an accessible dismissable dialog', async () => { @@ -520,8 +707,12 @@ describe('useAgentConsent', () => { false ) expect(onOpen).toHaveBeenCalledOnce() + // Two outcomes for one card, and that is the contract: the card's own + // ending is `accepted_pending_sign_in`, and `accepted` is the later moment + // consent actually became stored. expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ - [{ decision: 'accepted' }] + [{ decision: 'accepted_pending_sign_in', save_error_shown: false }], + [{ decision: 'accepted', save_error_shown: false }] ]) }) @@ -542,8 +733,12 @@ describe('useAgentConsent', () => { expect(useToastStore().add).not.toHaveBeenCalled() // Accepting the card is only half of the signed-out flow. Consent was // never persisted, so reporting it accepted would put a decision the user - // did not complete into the funnel. - expect(telemetry.trackAgentConsentResolved).not.toHaveBeenCalled() + // did not complete into the funnel — but the card half did happen, and + // `accepted_pending_sign_in` with no later `accepted` is exactly how an + // abandoned sign-in is now readable. + expect(telemetry.trackAgentConsentResolved.mock.calls).toEqual([ + [{ decision: 'accepted_pending_sign_in', save_error_shown: false }] + ]) }) it('reports sign-in loading failure without saving or opening and allows another attempt', async () => { diff --git a/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts b/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts index cd35487437e..f02c57a0b55 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentConsent.ts @@ -4,7 +4,11 @@ import { storeToRefs } from 'pinia' import { useCurrentUser } from '@/composables/auth/useCurrentUser' import { i18n } from '@/i18n' import { useTelemetry } from '@/platform/telemetry' -import type { AgentConsentTrigger } from '@/platform/telemetry/types' +import type { + AgentConsentDecision, + AgentConsentOfferExit, + AgentConsentTrigger +} from '@/platform/telemetry/types' import { reportError } from '@/platform/telemetry/reportError' import { useToastStore } from '@/platform/updates/common/toastStore' import { useDialogService } from '@/services/dialogService' @@ -38,6 +42,40 @@ export function useAgentConsent() { const { accepted, identity, isChecking } = storeToRefs(consentStore) const { t } = i18n.global + /** + * Names an ending of the consent request that produced neither a card on + * screen nor a decision. + * + * These are the last silent endings on the automatic consent chain. + * `agent_consent_not_offered` means a named surface is holding the offer and + * will retry it, and `agent_consent_offer_exited`'s other stages all stop + * before the consent scope is resolved - so an attempt that got as far as + * asking and still ended with nothing was indistinguishable in product + * analytics from a user who was never offered. The failing ones do reach + * Sentry and RUM through `reportError`, which is why they were easy to + * believe were covered; `reportError` has no PostHog sink, so none of them + * were in the funnel denominator `agent_consent_not_offered` is read against. + * + * `retry_armed` is reported false rather than read, and that is a code reading + * rather than a tested fact: `offerConsentUnprompted` calls `dropHold()` + * immediately before `withConsent`, the only caller that can re-arm is the + * `canShow` hook, and `canShow` runs after every exit reported here. A + * re-entrant offer cannot re-arm either - it exits on `autoShowInFlight` + * before reaching a hold. If that ordering changes, this becomes wrong + * silently, so the property's doc says which ordering it depends on. + */ + function reportRequestExit( + exit: AgentConsentOfferExit, + trigger: AgentConsentTrigger + ): void { + useTelemetry()?.trackAgentConsentOfferExited({ + exit, + stage: 'request', + retry_armed: false, + trigger + }) + } + function showConsentDialog( trigger: AgentConsentTrigger, persistOnAccept = true, @@ -51,6 +89,21 @@ export function useAgentConsent() { } let settled = false let saving = false + let shown = false + let saveErrorShown = false + + /** + * One outcome per card that was on screen, so this event and + * `agent_consent_shown` stay 1:1 and a shown card with no outcome is a + * defect rather than a dismissal. `accepted` and `rejected` keep exactly + * the meaning they already had. + */ + const reportOutcome = (decision: AgentConsentDecision): void => { + useTelemetry()?.trackAgentConsentResolved({ + decision, + save_error_shown: saveErrorShown + }) + } const closeWith = (result: boolean): void => { if (settled) return @@ -62,6 +115,10 @@ export function useAgentConsent() { function handleSaveFailure(error: unknown): void { if (settled) return saving = false + // Not an outcome: the card stays open, retryable, with the error on it. + // The ending it eventually reaches carries this flag instead, which is + // what separates giving up after a failed save from walking away. + saveErrorShown = true reportError(error, { errorType: 'agent_consent_setting_write_failure' }) @@ -94,12 +151,19 @@ export function useAgentConsent() { const saved = persistOnAccept ? await consentStore.accept(expectedIdentity) : true - // Only a persisted acceptance resolves here. Without `persistOnAccept` - // this card is the first half of the signed-out flow, which still has - // a sign-in and a real save to clear before consent exists, so that - // path reports its own acceptance once those land. - if (persistOnAccept && saved) - useTelemetry()?.trackAgentConsentResolved({ decision: 'accepted' }) + // Only a persisted acceptance is `accepted`. Without + // `persistOnAccept` this card is the first half of the signed-out + // flow, which still has a sign-in and a real save to clear before + // consent exists, so that path still reports its own `accepted` once + // those land - `accepted_pending_sign_in` records that the card half + // is done without claiming consent that does not exist yet. + // + // A save that resolved false neither stored consent nor raised, so it + // shows no error and closes the card: the user believes they + // consented. That is the ending this event could not tell from a + // dismissal, and the whole reason the two are named separately. + if (!persistOnAccept) reportOutcome('accepted_pending_sign_in') + else reportOutcome(saved ? 'accepted' : 'accept_not_persisted') closeWith(saved) } catch (error) { handleSaveFailure(error) @@ -120,13 +184,14 @@ export function useAgentConsent() { accepting: false, error: '', onVnodeMounted: () => { + shown = true useTelemetry()?.trackAgentConsentShown({ trigger }) if (expectedIdentity && identity.value !== expectedIdentity) return onShown?.() }, onAccept: () => void accept(), onReject: () => { - useTelemetry()?.trackAgentConsentResolved({ decision: 'rejected' }) + reportOutcome('rejected') closeWith(false) } }, @@ -142,6 +207,19 @@ export function useAgentConsent() { onClose: () => { if (settled) return settled = true + // Escape, the overlay mask, or a programmatic close. Reached only + // when nothing else settled first, so it is exactly the ending that + // used to emit `agent_consent_shown` and then nothing at all. + // + // Before the card renders there is no impression to resolve, and + // reporting a dismissal against one would break this event's 1:1 + // agreement with `agent_consent_shown` in the other direction - so + // that window is an ending of the request instead. It is narrow (the + // card is an async component, so it is the chunk's load time) and + // its production frequency is unknown rather than assumed to be + // zero. + if (shown) reportOutcome('dismissed') + else reportRequestExit('card_closed_before_mount', trigger) resolve(false) } } @@ -173,7 +251,14 @@ export function useAgentConsent() { const decisionIdentity = await consentStore.ensureScope() if (!decisionIdentity || !(await consentStore.accept(decisionIdentity))) return null - useTelemetry()?.trackAgentConsentResolved({ decision: 'accepted' }) + // The second half of the signed-out flow, so this is the moment consent + // becomes stored rather than the card's own ending - the card already + // reported `accepted_pending_sign_in`. No save error can have been shown + // on it: its accept ran with `persistOnAccept` false and never wrote. + useTelemetry()?.trackAgentConsentResolved({ + decision: 'accepted', + save_error_shown: false + }) return decisionIdentity } catch (error) { reportError(error, { @@ -193,11 +278,25 @@ export function useAgentConsent() { hooks: ConsentOfferHooks ): Promise { let decisionIdentity: string | null + // Which call is in flight, so the catch can tell a scope probe that failed + // from a consent read that failed. The error cannot carry that: both raise + // `AgentConsentAuthenticationError` with the same "account authentication is + // required" message, `ensureScope` from its own missing user and `load` from + // the auth header its read needs. + let reading = false try { decisionIdentity = await consentStore.ensureScope() - if (!decisionIdentity) return null + if (!decisionIdentity) { + reportRequestExit('scope_changed_before_read', trigger) + return null + } + reading = true await consentStore.load() } catch (error) { + reportRequestExit( + reading ? 'consent_read_failed' : 'scope_probe_failed', + trigger + ) reportError(error, { errorType: 'agent_consent_setting_load_failure' }) @@ -209,7 +308,19 @@ export function useAgentConsent() { return null } - if (identity.value !== decisionIdentity) return null + if (identity.value !== decisionIdentity) { + reportRequestExit('scope_changed_after_read', trigger) + return null + } + // Two endings report nothing here, and both are successes rather than + // silences. `showConsentDialog` returning false because `canShow` said no is + // already named by the surface that took the screen - the caller's hook + // holds the offer and `agent_consent_not_offered` carries the reason, so + // naming it again would double-count one deferral. And consent that is + // already stored skips the card entirely: this runs on to `onAccept`, which + // for the automatic offer is the only producer of + // `agent_panel_opened(source=automatic_consent)`. Reading the absence of a + // card as a failed offer is the error that mis-ranked gc-17. if ( !accepted.value && !(await showConsentDialog(trigger, true, decisionIdentity, hooks)) From c4a04b5900bf04f9d7df28f9d914d6083dbee824 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Tue, 29 Sep 2026 00:38:42 -0700 Subject: [PATCH 5/5] test(agent): preserve consent fixture allowlist grant --- browser_tests/fixtures/agentConsentTelemetryFixture.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/browser_tests/fixtures/agentConsentTelemetryFixture.ts b/browser_tests/fixtures/agentConsentTelemetryFixture.ts index 7402e0eb362..b590bcce285 100644 --- a/browser_tests/fixtures/agentConsentTelemetryFixture.ts +++ b/browser_tests/fixtures/agentConsentTelemetryFixture.ts @@ -63,9 +63,11 @@ export const agentConsentTelemetryTest = agentConsentTest.extend<{ }) }) // Layered over the fixture's own `/api/features`, which the later route - // wins. `enable_telemetry` is what gates the host sink; the PostHog values - // repeat the fixture's because the agent flag is bootstrapped through them. + // wins. Keep the authenticated allowlist grant from the base fixture: that + // grant is read directly from remote config once auth has loaded, not from + // PostHog's bootstrap flags. `enable_telemetry` gates the host sink. const features: RemoteConfig = { + 'agent-in-app-experience': true, enable_telemetry: true, posthog_project_token: 'phc_e2e_agent_consent_outcome', posthog_config: {