Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions browser_tests/fixtures/agentConsentTelemetryFixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import type { RemoteConfig } from '@/platform/remoteConfig/types'

import { agentConsentTest } from '@e2e/fixtures/agentConsentFixture'
import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute'

export interface CapturedTelemetryEvent {
event: string
properties: Record<string, unknown>
}

/**
* Reads the consent surface's telemetry from inside a real browser.
*
* It captures at the **host bridge**, not over the network. `main.ts` registers
* the host sink after the cloud one when `window.__comfyDesktop2.Telemetry`
* exists, and `setTelemetryRegistry` keeps the last registry - so every
* `useTelemetry()` call in the app lands here synchronously, with no batching,
* no compression and no third-party host to decode. Cloud plus a host bridge is
* a configuration that exists in production (`main.ts` has an
* `isCloud && hasHostTelemetryBridge` branch of its own), not one fabricated for
* the test.
*
* What that does and does not prove: the browser proves which events a real
* interaction produces and with which properties, through the same
* `TelemetryRegistry` the PostHog provider is registered on. The mapping from a
* registry call to a PostHog capture is asserted by
* `PostHogTelemetryProvider.test.ts`, not here.
*
* PostHog still initializes, because the agent feature flag is bootstrapped
* through it; its ingest host is routed so nothing leaves the browser and the
* network-isolation fixture stays satisfied.
*/
export const agentConsentTelemetryTest = agentConsentTest.extend<{
consentTelemetry: CapturedTelemetryEvent[]
}>({
consentTelemetry: async ({ agentFlagEnabled: _agentFlagEnabled }, use) => {
await use([])
},
page: async ({ page, consentTelemetry }, use) => {
await page.exposeFunction(
'__captureHostTelemetry',
(captured: CapturedTelemetryEvent) => {
consentTelemetry.push(captured)
}
)
await page.addInitScript(() => {
// Declared before the app boots so `hasHostTelemetryBridge` sees it.
Object.assign(window, {
__comfyDesktop2: {
isRemote: () => false,
Telemetry: {
capture: (event: string, properties: Record<string, unknown>) => {
void (
window as unknown as {
__captureHostTelemetry: (
captured: CapturedTelemetryEvent
) => Promise<void>
}
).__captureHostTelemetry({ event, properties })
}
}
}
})
})
// Layered over the fixture's own `/api/features`, which the later route
// wins. Keep the authenticated allowlist grant from the base fixture: that
// grant is read directly from remote config once auth has loaded, not from
// PostHog's bootstrap flags. `enable_telemetry` gates the host sink.
const features: RemoteConfig = {
'agent-in-app-experience': true,
enable_telemetry: true,
posthog_project_token: 'phc_e2e_agent_consent_outcome',
posthog_config: {
advanced_disable_flags: true,
bootstrap: {
featureFlags: { 'agent-in-app-experience': true }
}
}
}
await page.route('**/api/features', (route) =>
route.fulfill(jsonRoute(features))
)
await page.route('**://t.comfy.org/**', (route) =>
route.fulfill(jsonRoute({ status: 1 }))
)
await use(page)
}
})
197 changes: 197 additions & 0 deletions browser_tests/tests/agent/agentConsentOutcome.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
import { expect } from '@playwright/test'

import enMessages from '@/locales/en/main.json' with { type: 'json' }
import { AGENT_CONSENT_SETTING_ID } from '@/platform/settings/constants/agent'

import { agentConsentTelemetryTest as test } from '@e2e/fixtures/agentConsentTelemetryFixture'

/**
* A card that was shown and then went quiet used to cover three endings at
* once - dismissed, an acceptance whose save did not stick, and (signed out) an
* abandoned sign-in. `useAgentConsent.test.ts` pins all of them, but it arranges
* a dismissal by calling the dialog's `onClose` prop directly. This file is here
* because that is the one arrange a unit test cannot make faithfully: whether a
* real Escape key or a real overlay click reaches that handler at all depends on
* the dialog renderer, the dialog stack's escape bookkeeping and the overlay's
* own hit area, none of which a hand-called callback exercises.
*/
test.describe(
'Automatic consent card outcome',
{ tag: ['@cloud', '@ui'] },
() => {
test.use({ agentConsentAccepted: false })

test('reports a dismissal when Escape closes the automatic card', async ({
comfyPage,
agentPanel,
agentConsentWrites,
consentTelemetry
}) => {
const page = comfyPage.page
const dialog = page.getByRole('dialog', {
name: enMessages.agent.consent.title
})

await test.step('The automatic offer reports exactly one impression', async () => {
await expect(dialog).toBeVisible()
await expect
.poll(() =>
consentTelemetry.filter(
(e) => e.event === 'app:agent_consent_shown'
)
)
.toEqual([
{
event: 'app:agent_consent_shown',
properties: expect.objectContaining({ trigger: 'first_load' })
}
])
})

await test.step('Escape reports the dismissal as its own outcome', async () => {
expect(
consentTelemetry.filter(
(e) => e.event === 'app:agent_consent_resolved'
)
).toEqual([])

await page.keyboard.press('Escape')
await expect(dialog).toHaveCount(0)

await expect
.poll(() =>
consentTelemetry.filter(
(e) => e.event === 'app:agent_consent_resolved'
)
)
.toEqual([
{
event: 'app:agent_consent_resolved',
properties: expect.objectContaining({
decision: 'dismissed',
save_error_shown: false
})
}
])
// Consent semantics are unchanged: a dismissal is still not a decision.
expect(agentConsentWrites).toHaveLength(0)
await expect(agentPanel.root).toHaveCount(0)
})

await test.step('One outcome per impression, not one per close path', async () => {
// The dialog closes through `closeDialog`, which fires `onClose`, and
// the promise settles separately. Both ran; only one event exists.
expect(
consentTelemetry.filter(
(e) => e.event === 'app:agent_consent_resolved'
)
).toHaveLength(1)
})
})

test('reports the same outcome when the overlay mask closes the card', async ({
comfyPage,
agentConsentWrites,
consentTelemetry
}) => {
const page = comfyPage.page
const dialog = page.getByRole('dialog', {
name: enMessages.agent.consent.title
})

await expect(dialog).toBeVisible()
await page
.getByTestId('dialog-overlay')
.click({ position: { x: 1, y: 1 } })
await expect(dialog).toHaveCount(0)

await expect
.poll(() =>
consentTelemetry
.filter((e) => e.event === 'app:agent_consent_resolved')
.map((e) => e.properties.decision)
)
.toEqual(['dismissed'])
expect(agentConsentWrites).toHaveLength(0)
})

test('reports an acceptance, not a dismissal, when the card is accepted', async ({
comfyPage,
agentPanel,
agentConsentWrites,
consentTelemetry
}) => {
const page = comfyPage.page
const dialog = page.getByRole('dialog', {
name: enMessages.agent.consent.title
})

await expect(dialog).toBeVisible()
await dialog
.getByRole('button', { name: enMessages.agent.consent.accept })
.click()
await expect(dialog).toHaveCount(0)
await expect(agentPanel.root).toBeVisible()

// `closeWith` settles before closing the dialog, so the accepted outcome
// must not be followed by a dismissal from the close it performs itself.
await expect
.poll(() =>
consentTelemetry
.filter((e) => e.event === 'app:agent_consent_resolved')
.map((e) => e.properties.decision)
)
.toEqual(['accepted'])
expect(agentConsentWrites).toEqual([true])
})

test('names a failed consent read at the request, not just at the load', async ({
comfyPage,
agentPanel,
consentTelemetry
}) => {
const page = comfyPage.page
// Every read fails. The automatic chain exits at the `load` stage without
// asking, which `gc-16`'s instrument already names; the user then clicks
// the entry button, and that attempt fails inside the request. This is the
// shape behind the 23 `agent_consent_setting_load_failure` events in
// Sentry that product analytics saw nothing of.
await page.route(
`**/api/global-settings/${AGENT_CONSENT_SETTING_ID}`,
(route) => route.fulfill({ status: 500, body: '{"code":"INTERNAL"}' })
)
await comfyPage.workflow.reloadAndWaitForApp()

await expect(agentPanel.openButton).toBeEnabled()
await agentPanel.openButton.click()

await expect
.poll(() =>
consentTelemetry
.filter((e) => e.event === 'app:agent_consent_offer_exited')
.map((e) => [e.properties.stage, e.properties.exit])
)
.toContainEqual(['request', 'consent_read_failed'])

// The same condition at two links, told apart by the pair rather than by
// the reason value - which is what `exit` plus `stage` is for.
const requestExits = consentTelemetry.filter(
(e) =>
e.event === 'app:agent_consent_offer_exited' &&
e.properties.stage === 'request'
)
expect(requestExits.map((e) => e.properties)).toEqual([
{
exit: 'consent_read_failed',
stage: 'request',
retry_armed: false,
trigger: 'button_click'
}
])
await expect(
page.getByRole('dialog', { name: enMessages.agent.consent.title })
).toHaveCount(0)
await expect(agentPanel.root).toHaveCount(0)
})
}
)
Loading
Loading