Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion browser_tests/tests/agent/agentPanelMocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ async function mockAgentBoot(
agentConsentWrites,
agentAutoShownReadProbe,
agentFlagEnabled,
starterPromptSet,
agentPanelInitiallyOpen,
agentOnboardingCompleted,
agentBilling,
Expand Down Expand Up @@ -311,7 +312,13 @@ async function mockAgentBoot(
)

await mockCloudBoot(page, {
features: { ...agentFeatures(agentFlagEnabled), ...initialFeatureFlags },
features: {
...agentFeatures(agentFlagEnabled),
...(starterPromptSet !== undefined && {
'agent-starter-prompt-set': starterPromptSet
}),
...initialFeatureFlags
},
settings: {
'Comfy.TutorialCompleted': true,
'Comfy.RightSidePanel.ShowErrorsTab': false,
Expand Down Expand Up @@ -492,6 +499,7 @@ type AgentFixtures = {
agentConsentSave: { status: number; pending?: Promise<void> }
agentConsentWrites: boolean[]
agentFlagEnabled: boolean
starterPromptSet: RemoteConfig['agent-starter-prompt-set']
agentPanel: AgentPanel
agentPanelInitiallyOpen: boolean
agentOnboardingCompleted: boolean
Expand Down Expand Up @@ -520,6 +528,7 @@ export const agentTest = comfyPageFixture.extend<AgentFixtures>({
await use([])
},
agentFlagEnabled: [true, { option: true }],
starterPromptSet: [undefined, { option: true }],
agentPanel: async ({ comfyPage }, use) => {
await use(new AgentPanel(comfyPage.page))
},
Expand All @@ -535,6 +544,7 @@ export const agentTest = comfyPageFixture.extend<AgentFixtures>({
agentConsentSave,
agentConsentWrites,
agentFlagEnabled,
starterPromptSet,
agentPanelInitiallyOpen,
agentOnboardingCompleted,
agentBilling,
Expand All @@ -555,6 +565,7 @@ export const agentTest = comfyPageFixture.extend<AgentFixtures>({
agentConsentSave,
agentConsentWrites,
agentFlagEnabled,
starterPromptSet,
agentPanelInitiallyOpen,
agentOnboardingCompleted,
agentBilling,
Expand Down
56 changes: 56 additions & 0 deletions browser_tests/tests/agent/agentStarterPromptExperiment.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { expect, mergeTests } from '@playwright/test'

import { hostTelemetryFixture } from '@e2e/fixtures/hostTelemetryFixture'
import { webSocketFixture } from '@e2e/fixtures/ws'

import enMessages from '@/locales/en/main.json' with { type: 'json' }

import { agentTest } from '@e2e/tests/agent/agentPanelMocks'

const test = mergeTests(agentTest, hostTelemetryFixture, webSocketFixture)

test.describe('Agent starter prompt experiment', { tag: '@cloud' }, () => {
test.use({
connectWebSocketToServer: false,
initialFeatureFlags: { enable_telemetry: true },
starterPromptSet: 'test'
})

test('reports the treatment exposure and carries the arm onto the send', async ({
agentPanel,
hostTelemetry
}) => {
const treatmentPrompt = enMessages.agent.suggestedPrompts.treatment.cloud[0]

await test.step('open the panel on the treatment arm', async () => {
await agentPanel.open()
await agentPanel.selectWorkflow()
await expect
.poll(() =>
hostTelemetry.find(
({ event }) => event === 'app:agent_starter_prompt_exposure'
)
)
.toEqual({
event: 'app:agent_starter_prompt_exposure',
properties: { '$feature/agent-starter-prompt-set': 'test' }
})
})

await test.step('send a treatment starter prompt', async () => {
await agentPanel.root
.getByRole('button', { name: treatmentPrompt })
.click()
await expect(agentPanel.composer).toHaveText(treatmentPrompt)
await agentPanel.sendButton.click()
await expect
.poll(
() =>
hostTelemetry.find(
({ event }) => event === 'app:agent_message_sent'
)?.properties['$feature/agent-starter-prompt-set']
)
.toBe('test')
})
})
})
11 changes: 10 additions & 1 deletion src/locales/en/main.json
Original file line number Diff line number Diff line change
Expand Up @@ -5255,7 +5255,16 @@
"Explain the selected nodes",
"Help me install the missing nodes for this workflow",
"Fix the errors in this workflow"
]
],
"treatment": {
"cloud": [
"Create a product image from a text prompt",
"Build an image-to-video workflow",
"Explain how a ComfyUI workflow works",
"Find the best workflow for a specific task",
"Diagnose and fix errors in a workflow"
]
}
},
"attach": "Attach a file",
"attachFiles": "Upload images or files",
Expand Down
2 changes: 2 additions & 0 deletions src/platform/remoteConfig/types.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { GetFeaturesResponses } from '@comfyorg/ingest-types'
import type { PostHogConfig } from 'posthog-js'

import type { TelemetryEventName } from '@/platform/telemetry/types'
Expand Down Expand Up @@ -109,6 +110,7 @@ export type RemoteConfig = {
manager_survey_url?: string
linear_toggle_enabled?: boolean
'agent-in-app-experience'?: boolean
'agent-starter-prompt-set'?: GetFeaturesResponses[200]['agent-starter-prompt-set']
'agent-free-use-message-placement'?: string
partner_node_governance_enabled?: boolean
/** Kill switch for the local partner-nodes run gate; defaults on client-side. */
Expand Down
12 changes: 12 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type {
AgentPanelClosedMetadata,
AgentPanelOpenedMetadata,
AgentStarterPromptClickedMetadata,
AgentStarterPromptExposureMetadata,
AgentWorkflowAppliedMetadata,
BillingTelemetryEvent,
CheckoutJourneyTelemetryEvent,
Expand Down Expand Up @@ -309,6 +310,9 @@ describe('TelemetryRegistry', () => {
click_id: 'click-1',
draft_was_empty: true
} satisfies AgentStarterPromptClickedMetadata
const starterPromptExposureMetadata = {
'$feature/agent-starter-prompt-set': 'test'
} satisfies AgentStarterPromptExposureMetadata
const consentShownMetadata = {
trigger: 'button_click'
} satisfies AgentConsentShownMetadata
Expand Down Expand Up @@ -374,6 +378,14 @@ describe('TelemetryRegistry', () => {
invoke: (registry) =>
registry.trackAgentStarterPromptClicked(starterPromptClickedMetadata)
},
{
method: 'trackAgentStarterPromptExposure',
expected: { ...starterPromptExposureMetadata },
invoke: (registry) =>
registry.trackAgentStarterPromptExposure(
starterPromptExposureMetadata
)
},
{
method: 'trackAgentConsentShown',
expected: { ...consentShownMetadata },
Expand Down
9 changes: 9 additions & 0 deletions src/platform/telemetry/TelemetryRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import type {
AgentPanelClosedMetadata,
AgentPanelOpenedMetadata,
AgentStarterPromptClickedMetadata,
AgentStarterPromptExposureMetadata,
AgentWorkflowAppliedMetadata,
AuthErrorMetadata,
AuthMetadata,
Expand Down Expand Up @@ -425,6 +426,14 @@ export class TelemetryRegistry implements TelemetryDispatcher {
)
}

trackAgentStarterPromptExposure(
metadata: AgentStarterPromptExposureMetadata
): void {
this.dispatch((provider) =>
provider.trackAgentStarterPromptExposure?.(metadata)
)
}

// fallow-ignore-next-line unused-class-member
trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void {
this.dispatch((provider) => provider.trackAgentFreeUseNotice?.(metadata))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,20 @@ describe('PostHogTelemetryProvider', () => {
)
})

it('captures starter prompt exposure with its assignment', async () => {
const provider = createProvider()
await vi.dynamicImportSettled()

provider.trackAgentStarterPromptExposure({
'$feature/agent-starter-prompt-set': 'test'
})

expect(hoisted.mockCapture).toHaveBeenCalledWith(
TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE,
{ '$feature/agent-starter-prompt-set': 'test' }
)
})

it('captures free-use notice interactions with their placement', async () => {
const provider = createProvider()
await vi.dynamicImportSettled()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import type {
AgentPanelClosedMetadata,
AgentPanelOpenedMetadata,
AgentStarterPromptClickedMetadata,
AgentStarterPromptExposureMetadata,
AgentWorkflowAppliedMetadata,
AuthErrorMetadata,
AuthMetadata,
Expand Down Expand Up @@ -747,6 +748,12 @@ export class PostHogTelemetryProvider implements TelemetryProvider {
this.trackEvent(TelemetryEvents.AGENT_STARTER_PROMPT_CLICKED, metadata)
}

trackAgentStarterPromptExposure(
metadata: AgentStarterPromptExposureMetadata
): void {
this.trackEvent(TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, metadata)
}

trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void {
this.trackEvent(TelemetryEvents.AGENT_FREE_USE_NOTICE, metadata)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,14 @@ describe('HostTelemetrySink', () => {
draft_was_empty: false
}
},
{
name: TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE,
track: (sink: HostTelemetrySink) =>
sink.trackAgentStarterPromptExposure({
'$feature/agent-starter-prompt-set': 'test'
}),
properties: { '$feature/agent-starter-prompt-set': 'test' }
},
{
name: TelemetryEvents.AGENT_FREE_USE_NOTICE,
track: (sink: HostTelemetrySink) =>
Expand Down
7 changes: 7 additions & 0 deletions src/platform/telemetry/providers/host/HostTelemetrySink.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type {
AgentPaywallCtaMetadata,
AgentPaywallShownMetadata,
AgentStarterPromptClickedMetadata,
AgentStarterPromptExposureMetadata,
AgentWorkflowAppliedMetadata,
AgentOnboardingStepMetadata,
AuthMetadata,
Expand Down Expand Up @@ -360,6 +361,12 @@ export class HostTelemetrySink implements TelemetryProvider {
this.capture(TelemetryEvents.AGENT_STARTER_PROMPT_CLICKED, metadata)
}

trackAgentStarterPromptExposure(
metadata: AgentStarterPromptExposureMetadata
): void {
this.capture(TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, metadata)
}

trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void {
this.capture(TelemetryEvents.AGENT_FREE_USE_NOTICE, metadata)
}
Expand Down
32 changes: 27 additions & 5 deletions src/platform/telemetry/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
* 3. Check dist/assets/*.js files contain no tracking code
*/

import type { GetFeaturesResponses } from '@comfyorg/ingest-types'

import type { TierKey } from '@/platform/cloud/subscription/constants/tierPricing'
import type { BillingCycle } from '@/platform/cloud/subscription/utils/subscriptionTierRank'
import type { AppMode } from '@/utils/appMode'
Expand Down Expand Up @@ -828,6 +830,17 @@ export type AgentStarterPromptId =
| 'slot_4'
| 'slot_5'
| 'unregistered'
export type AgentStarterPromptAssignment = NonNullable<
GetFeaturesResponses[200]['agent-starter-prompt-set']
>
interface AgentStarterPromptExperimentMetadata {
/**
* Assignment of the prompt set that rendered. Optional consumers omit it
* for QA overrides, unsupported locales, and surfaces rendered before
* authenticated config.
*/
'$feature/agent-starter-prompt-set': AgentStarterPromptAssignment
}
/**
* Where the free-use notice was placed, for the DES-1221 placement experiment.
*
Expand Down Expand Up @@ -856,10 +869,10 @@ export interface AgentFreeUseNoticeMetadata extends Record<string, unknown> {
action: 'shown' | 'dismissed' | 'learn_more_clicked'
placement: AgentFreeUsePlacement
}
export interface AgentStarterPromptClickedMetadata extends Record<
string,
unknown
> {
export interface AgentStarterPromptClickedMetadata
extends
Record<string, unknown>,
Partial<AgentStarterPromptExperimentMetadata> {
prompt_id: AgentStarterPromptId
/** Slot position, so a reorder is visible rather than silently re-labelling. */
prompt_index: number
Expand Down Expand Up @@ -889,7 +902,12 @@ export interface AgentStarterPromptClickedMetadata extends Record<
*/
draft_was_empty: boolean
}
export interface AgentMessageSentMetadata extends Record<string, unknown> {
export interface AgentStarterPromptExposureMetadata
extends Record<string, unknown>, AgentStarterPromptExperimentMetadata {}
export interface AgentMessageSentMetadata
extends
Record<string, unknown>,
Partial<AgentStarterPromptExperimentMetadata> {
attachment_count: number
node_tag_count: number
/**
Expand Down Expand Up @@ -1686,6 +1704,9 @@ export interface TelemetryProvider {
trackAgentStarterPromptClicked?(
metadata: AgentStarterPromptClickedMetadata
): void
trackAgentStarterPromptExposure?(
metadata: AgentStarterPromptExposureMetadata
): void
trackAgentFreeUseNotice?(metadata: AgentFreeUseNoticeMetadata): void
trackAgentFreeUseExposure?(metadata: AgentFreeUseExposureMetadata): void
trackAgentNodeTagged?(metadata: AgentNodeTaggedMetadata): void
Expand Down Expand Up @@ -1867,6 +1888,7 @@ export const TelemetryEvents = {
AGENT_ONBOARDING_STEP: 'app:agent_onboarding_step',
AGENT_MESSAGE_SENT: 'app:agent_message_sent',
AGENT_STARTER_PROMPT_CLICKED: 'app:agent_starter_prompt_clicked',
AGENT_STARTER_PROMPT_EXPOSURE: 'app:agent_starter_prompt_exposure',
AGENT_FREE_USE_NOTICE: 'app:agent_free_use_notice',
AGENT_FREE_USE_EXPOSURE: 'app:agent_free_use_exposure',
AGENT_NODE_TAGGED: 'app:agent_node_tagged',
Expand Down
16 changes: 15 additions & 1 deletion src/workbench/extensions/agent/AgentPanelRoot.vue
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ import {
import { useWorkspaceUI } from '@/platform/workspace/composables/useWorkspaceUI'

import { useFreeUsePlacement } from './experiments/freeUsePlacement'
import { useStarterPromptSet } from './experiments/starterPromptSet'
import AgentPanel from './components/agent/AgentPanel.vue'
import { agentBoundWorkflowIdKey } from './components/agent/agentBoundWorkflowId'
import OnboardingCoach from './components/agent/OnboardingCoach.vue'
Expand Down Expand Up @@ -1208,6 +1209,11 @@ const { copy } = useClipboard({ legacy: true })
* denominator — to the panel openers the hypothesis is about.
*/
const { variant: freeUsePlacement } = useFreeUsePlacement()
const {
assignment: starterPromptAssignment,
attributeExperiment: attributeStarterPromptExperiment,
expose: exposeStarterPromptSet
} = useStarterPromptSet()

function onFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void {
useTelemetry()?.trackAgentFreeUseNotice(metadata)
Expand Down Expand Up @@ -1371,7 +1377,12 @@ const { submit: onSend } = useAgentDraftSubmission({
client_message_id: meta.clientMessageId,
input_method: meta.inputMethod,
starter_prompt_id: meta.starterPrompt?.id ?? null,
starter_prompt_click_id: meta.starterPrompt?.clickId ?? null
starter_prompt_click_id: meta.starterPrompt?.clickId ?? null,
...(meta.starterPrompt?.assignment
? {
'$feature/agent-starter-prompt-set': meta.starterPrompt.assignment
}
: {})
},
origin:
originContext === undefined ? null : { tabPath: originContext.tabPath }
Expand Down Expand Up @@ -1857,6 +1868,9 @@ async function onPanelDrop(event: DragEvent): Promise<void> {
:paywall-presentation="paywallPresentation"
:credits-exhausted="showStandingPaywall"
:free-use-placement="freeUsePlacement"
:starter-prompt-assignment="starterPromptAssignment"
:attribute-starter-prompt-experiment="attributeStarterPromptExperiment"
@starter-prompt-rendered="exposeStarterPromptSet"
@free-use-notice="onFreeUseNotice"
@send="onSend"
@stop="onStop"
Expand Down
Loading
Loading