diff --git a/browser_tests/tests/agent/agentPanelMocks.ts b/browser_tests/tests/agent/agentPanelMocks.ts index 2de4b252eea..4182ccaa0fe 100644 --- a/browser_tests/tests/agent/agentPanelMocks.ts +++ b/browser_tests/tests/agent/agentPanelMocks.ts @@ -231,6 +231,7 @@ async function mockAgentBoot( agentConsentWrites, agentAutoShownReadProbe, agentFlagEnabled, + starterPromptSet, agentPanelInitiallyOpen, agentOnboardingCompleted, agentBilling, @@ -311,7 +312,13 @@ async function mockAgentBoot( ) await mockCloudBoot(page, { - features: { ...agentFeatures(agentFlagEnabled), ...initialFeatureFlags }, + features: { + ...agentFeatures(agentFlagEnabled), + ...(starterPromptSet !== undefined && { + 'agent-starter-prompt-set': starterPromptSet + }), + ...initialFeatureFlags + }, settings: { 'Comfy.TutorialCompleted': true, 'Comfy.RightSidePanel.ShowErrorsTab': false, @@ -492,6 +499,7 @@ type AgentFixtures = { agentConsentSave: { status: number; pending?: Promise } agentConsentWrites: boolean[] agentFlagEnabled: boolean + starterPromptSet: RemoteConfig['agent-starter-prompt-set'] agentPanel: AgentPanel agentPanelInitiallyOpen: boolean agentOnboardingCompleted: boolean @@ -520,6 +528,7 @@ export const agentTest = comfyPageFixture.extend({ await use([]) }, agentFlagEnabled: [true, { option: true }], + starterPromptSet: [undefined, { option: true }], agentPanel: async ({ comfyPage }, use) => { await use(new AgentPanel(comfyPage.page)) }, @@ -535,6 +544,7 @@ export const agentTest = comfyPageFixture.extend({ agentConsentSave, agentConsentWrites, agentFlagEnabled, + starterPromptSet, agentPanelInitiallyOpen, agentOnboardingCompleted, agentBilling, @@ -555,6 +565,7 @@ export const agentTest = comfyPageFixture.extend({ agentConsentSave, agentConsentWrites, agentFlagEnabled, + starterPromptSet, agentPanelInitiallyOpen, agentOnboardingCompleted, agentBilling, diff --git a/browser_tests/tests/agent/agentStarterPromptExperiment.spec.ts b/browser_tests/tests/agent/agentStarterPromptExperiment.spec.ts new file mode 100644 index 00000000000..e935a0bf7ed --- /dev/null +++ b/browser_tests/tests/agent/agentStarterPromptExperiment.spec.ts @@ -0,0 +1,56 @@ +import { expect, mergeTests } from '@playwright/test' + +import { hostTelemetryFixture } from '@e2e/fixtures/hostTelemetryFixture' +import { webSocketFixture } from '@e2e/fixtures/ws' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } + +import { agentTest } from '@e2e/tests/agent/agentPanelMocks' + +const test = mergeTests(agentTest, hostTelemetryFixture, webSocketFixture) + +test.describe('Agent starter prompt experiment', { tag: '@cloud' }, () => { + test.use({ + connectWebSocketToServer: false, + initialFeatureFlags: { enable_telemetry: true }, + starterPromptSet: 'test' + }) + + test('reports the treatment exposure and carries the arm onto the send', async ({ + agentPanel, + hostTelemetry + }) => { + const treatmentPrompt = enMessages.agent.suggestedPrompts.treatment.cloud[0] + + await test.step('open the panel on the treatment arm', async () => { + await agentPanel.open() + await agentPanel.selectWorkflow() + await expect + .poll(() => + hostTelemetry.find( + ({ event }) => event === 'app:agent_starter_prompt_exposure' + ) + ) + .toEqual({ + event: 'app:agent_starter_prompt_exposure', + properties: { '$feature/agent-starter-prompt-set': 'test' } + }) + }) + + await test.step('send a treatment starter prompt', async () => { + await agentPanel.root + .getByRole('button', { name: treatmentPrompt }) + .click() + await expect(agentPanel.composer).toHaveText(treatmentPrompt) + await agentPanel.sendButton.click() + await expect + .poll( + () => + hostTelemetry.find( + ({ event }) => event === 'app:agent_message_sent' + )?.properties['$feature/agent-starter-prompt-set'] + ) + .toBe('test') + }) + }) +}) diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 2f45ff43e2b..230ea4ce202 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5255,7 +5255,16 @@ "Explain the selected nodes", "Help me install the missing nodes for this workflow", "Fix the errors in this workflow" - ] + ], + "treatment": { + "cloud": [ + "Create a product image from a text prompt", + "Build an image-to-video workflow", + "Explain how a ComfyUI workflow works", + "Find the best workflow for a specific task", + "Diagnose and fix errors in a workflow" + ] + } }, "attach": "Attach a file", "attachFiles": "Upload images or files", diff --git a/src/platform/remoteConfig/types.ts b/src/platform/remoteConfig/types.ts index dcf9a63ed09..351a6c2eca3 100644 --- a/src/platform/remoteConfig/types.ts +++ b/src/platform/remoteConfig/types.ts @@ -1,3 +1,4 @@ +import type { GetFeaturesResponses } from '@comfyorg/ingest-types' import type { PostHogConfig } from 'posthog-js' import type { TelemetryEventName } from '@/platform/telemetry/types' @@ -109,6 +110,7 @@ export type RemoteConfig = { manager_survey_url?: string linear_toggle_enabled?: boolean 'agent-in-app-experience'?: boolean + 'agent-starter-prompt-set'?: GetFeaturesResponses[200]['agent-starter-prompt-set'] 'agent-free-use-message-placement'?: string partner_node_governance_enabled?: boolean /** Kill switch for the local partner-nodes run gate; defaults on client-side. */ diff --git a/src/platform/telemetry/TelemetryRegistry.test.ts b/src/platform/telemetry/TelemetryRegistry.test.ts index cff06937264..1a7cbb609e8 100644 --- a/src/platform/telemetry/TelemetryRegistry.test.ts +++ b/src/platform/telemetry/TelemetryRegistry.test.ts @@ -12,6 +12,7 @@ import type { AgentPanelClosedMetadata, AgentPanelOpenedMetadata, AgentStarterPromptClickedMetadata, + AgentStarterPromptExposureMetadata, AgentWorkflowAppliedMetadata, BillingTelemetryEvent, CheckoutJourneyTelemetryEvent, @@ -309,6 +310,9 @@ describe('TelemetryRegistry', () => { click_id: 'click-1', draft_was_empty: true } satisfies AgentStarterPromptClickedMetadata + const starterPromptExposureMetadata = { + '$feature/agent-starter-prompt-set': 'test' + } satisfies AgentStarterPromptExposureMetadata const consentShownMetadata = { trigger: 'button_click' } satisfies AgentConsentShownMetadata @@ -374,6 +378,14 @@ describe('TelemetryRegistry', () => { invoke: (registry) => registry.trackAgentStarterPromptClicked(starterPromptClickedMetadata) }, + { + method: 'trackAgentStarterPromptExposure', + expected: { ...starterPromptExposureMetadata }, + invoke: (registry) => + registry.trackAgentStarterPromptExposure( + starterPromptExposureMetadata + ) + }, { method: 'trackAgentConsentShown', expected: { ...consentShownMetadata }, diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index bdc2d9688e9..28afc734bf9 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -18,6 +18,7 @@ import type { AgentPanelClosedMetadata, AgentPanelOpenedMetadata, AgentStarterPromptClickedMetadata, + AgentStarterPromptExposureMetadata, AgentWorkflowAppliedMetadata, AuthErrorMetadata, AuthMetadata, @@ -425,6 +426,14 @@ export class TelemetryRegistry implements TelemetryDispatcher { ) } + trackAgentStarterPromptExposure( + metadata: AgentStarterPromptExposureMetadata + ): void { + this.dispatch((provider) => + provider.trackAgentStarterPromptExposure?.(metadata) + ) + } + // fallow-ignore-next-line unused-class-member trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void { this.dispatch((provider) => provider.trackAgentFreeUseNotice?.(metadata)) diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts index d37be823df1..658ea43f644 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.test.ts @@ -483,6 +483,20 @@ describe('PostHogTelemetryProvider', () => { ) }) + it('captures starter prompt exposure with its assignment', async () => { + const provider = createProvider() + await vi.dynamicImportSettled() + + provider.trackAgentStarterPromptExposure({ + '$feature/agent-starter-prompt-set': 'test' + }) + + expect(hoisted.mockCapture).toHaveBeenCalledWith( + TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, + { '$feature/agent-starter-prompt-set': 'test' } + ) + }) + it('captures free-use notice interactions with their placement', async () => { const provider = createProvider() await vi.dynamicImportSettled() diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index b574afbd5ef..bdd0068130a 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -30,6 +30,7 @@ import type { AgentPanelClosedMetadata, AgentPanelOpenedMetadata, AgentStarterPromptClickedMetadata, + AgentStarterPromptExposureMetadata, AgentWorkflowAppliedMetadata, AuthErrorMetadata, AuthMetadata, @@ -747,6 +748,12 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_STARTER_PROMPT_CLICKED, metadata) } + trackAgentStarterPromptExposure( + metadata: AgentStarterPromptExposureMetadata + ): void { + this.trackEvent(TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, metadata) + } + trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void { this.trackEvent(TelemetryEvents.AGENT_FREE_USE_NOTICE, metadata) } diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts index bbe2513b260..1bc50f40442 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.test.ts @@ -290,6 +290,14 @@ describe('HostTelemetrySink', () => { draft_was_empty: false } }, + { + name: TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, + track: (sink: HostTelemetrySink) => + sink.trackAgentStarterPromptExposure({ + '$feature/agent-starter-prompt-set': 'test' + }), + properties: { '$feature/agent-starter-prompt-set': 'test' } + }, { name: TelemetryEvents.AGENT_FREE_USE_NOTICE, track: (sink: HostTelemetrySink) => diff --git a/src/platform/telemetry/providers/host/HostTelemetrySink.ts b/src/platform/telemetry/providers/host/HostTelemetrySink.ts index afa0081e73c..bb59127528a 100644 --- a/src/platform/telemetry/providers/host/HostTelemetrySink.ts +++ b/src/platform/telemetry/providers/host/HostTelemetrySink.ts @@ -21,6 +21,7 @@ import type { AgentPaywallCtaMetadata, AgentPaywallShownMetadata, AgentStarterPromptClickedMetadata, + AgentStarterPromptExposureMetadata, AgentWorkflowAppliedMetadata, AgentOnboardingStepMetadata, AuthMetadata, @@ -360,6 +361,12 @@ export class HostTelemetrySink implements TelemetryProvider { this.capture(TelemetryEvents.AGENT_STARTER_PROMPT_CLICKED, metadata) } + trackAgentStarterPromptExposure( + metadata: AgentStarterPromptExposureMetadata + ): void { + this.capture(TelemetryEvents.AGENT_STARTER_PROMPT_EXPOSURE, metadata) + } + trackAgentFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void { this.capture(TelemetryEvents.AGENT_FREE_USE_NOTICE, metadata) } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 2131eaec5a2..573016da69a 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -12,6 +12,8 @@ * 3. Check dist/assets/*.js files contain no tracking code */ +import type { GetFeaturesResponses } from '@comfyorg/ingest-types' + import type { TierKey } from '@/platform/cloud/subscription/constants/tierPricing' import type { BillingCycle } from '@/platform/cloud/subscription/utils/subscriptionTierRank' import type { AppMode } from '@/utils/appMode' @@ -828,6 +830,17 @@ export type AgentStarterPromptId = | 'slot_4' | 'slot_5' | 'unregistered' +export type AgentStarterPromptAssignment = NonNullable< + GetFeaturesResponses[200]['agent-starter-prompt-set'] +> +interface AgentStarterPromptExperimentMetadata { + /** + * Assignment of the prompt set that rendered. Optional consumers omit it + * for QA overrides, unsupported locales, and surfaces rendered before + * authenticated config. + */ + '$feature/agent-starter-prompt-set': AgentStarterPromptAssignment +} /** * Where the free-use notice was placed, for the DES-1221 placement experiment. * @@ -856,10 +869,10 @@ export interface AgentFreeUseNoticeMetadata extends Record { action: 'shown' | 'dismissed' | 'learn_more_clicked' placement: AgentFreeUsePlacement } -export interface AgentStarterPromptClickedMetadata extends Record< - string, - unknown -> { +export interface AgentStarterPromptClickedMetadata + extends + Record, + Partial { prompt_id: AgentStarterPromptId /** Slot position, so a reorder is visible rather than silently re-labelling. */ prompt_index: number @@ -889,7 +902,12 @@ export interface AgentStarterPromptClickedMetadata extends Record< */ draft_was_empty: boolean } -export interface AgentMessageSentMetadata extends Record { +export interface AgentStarterPromptExposureMetadata + extends Record, AgentStarterPromptExperimentMetadata {} +export interface AgentMessageSentMetadata + extends + Record, + Partial { attachment_count: number node_tag_count: number /** @@ -1686,6 +1704,9 @@ export interface TelemetryProvider { trackAgentStarterPromptClicked?( metadata: AgentStarterPromptClickedMetadata ): void + trackAgentStarterPromptExposure?( + metadata: AgentStarterPromptExposureMetadata + ): void trackAgentFreeUseNotice?(metadata: AgentFreeUseNoticeMetadata): void trackAgentFreeUseExposure?(metadata: AgentFreeUseExposureMetadata): void trackAgentNodeTagged?(metadata: AgentNodeTaggedMetadata): void @@ -1867,6 +1888,7 @@ export const TelemetryEvents = { AGENT_ONBOARDING_STEP: 'app:agent_onboarding_step', AGENT_MESSAGE_SENT: 'app:agent_message_sent', AGENT_STARTER_PROMPT_CLICKED: 'app:agent_starter_prompt_clicked', + AGENT_STARTER_PROMPT_EXPOSURE: 'app:agent_starter_prompt_exposure', AGENT_FREE_USE_NOTICE: 'app:agent_free_use_notice', AGENT_FREE_USE_EXPOSURE: 'app:agent_free_use_exposure', AGENT_NODE_TAGGED: 'app:agent_node_tagged', diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 875926fa633..70ad0d05f78 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -71,6 +71,7 @@ import { import { useWorkspaceUI } from '@/platform/workspace/composables/useWorkspaceUI' import { useFreeUsePlacement } from './experiments/freeUsePlacement' +import { useStarterPromptSet } from './experiments/starterPromptSet' import AgentPanel from './components/agent/AgentPanel.vue' import { agentBoundWorkflowIdKey } from './components/agent/agentBoundWorkflowId' import OnboardingCoach from './components/agent/OnboardingCoach.vue' @@ -1208,6 +1209,11 @@ const { copy } = useClipboard({ legacy: true }) * denominator — to the panel openers the hypothesis is about. */ const { variant: freeUsePlacement } = useFreeUsePlacement() +const { + assignment: starterPromptAssignment, + attributeExperiment: attributeStarterPromptExperiment, + expose: exposeStarterPromptSet +} = useStarterPromptSet() function onFreeUseNotice(metadata: AgentFreeUseNoticeMetadata): void { useTelemetry()?.trackAgentFreeUseNotice(metadata) @@ -1371,7 +1377,12 @@ const { submit: onSend } = useAgentDraftSubmission({ client_message_id: meta.clientMessageId, input_method: meta.inputMethod, starter_prompt_id: meta.starterPrompt?.id ?? null, - starter_prompt_click_id: meta.starterPrompt?.clickId ?? null + starter_prompt_click_id: meta.starterPrompt?.clickId ?? null, + ...(meta.starterPrompt?.assignment + ? { + '$feature/agent-starter-prompt-set': meta.starterPrompt.assignment + } + : {}) }, origin: originContext === undefined ? null : { tabPath: originContext.tabPath } @@ -1857,6 +1868,9 @@ async function onPanelDrop(event: DragEvent): Promise { :paywall-presentation="paywallPresentation" :credits-exhausted="showStandingPaywall" :free-use-placement="freeUsePlacement" + :starter-prompt-assignment="starterPromptAssignment" + :attribute-starter-prompt-experiment="attributeStarterPromptExperiment" + @starter-prompt-rendered="exposeStarterPromptSet" @free-use-notice="onFreeUseNotice" @send="onSend" @stop="onStop" diff --git a/src/workbench/extensions/agent/components/agent/AgentPanel.test.ts b/src/workbench/extensions/agent/components/agent/AgentPanel.test.ts index 556e8c04c45..68ea76522ac 100644 --- a/src/workbench/extensions/agent/components/agent/AgentPanel.test.ts +++ b/src/workbench/extensions/agent/components/agent/AgentPanel.test.ts @@ -3,6 +3,14 @@ import userEvent from '@testing-library/user-event' import { createPinia, setActivePinia } from 'pinia' import { beforeEach, describe, expect, it } from 'vitest' +const distribution = vi.hoisted(() => ({ isCloud: false })) + +vi.mock(import('@/platform/distribution/types'), () => ({ + get isCloud() { + return distribution.isCloud + } +})) + import { i18n } from '@/i18n' import type { TurnId } from '../../schemas/agentApiSchema' @@ -31,9 +39,34 @@ function mount(isMaximized = false) { describe('AgentPanel', () => { beforeEach(() => { + distribution.isCloud = false localStorage.clear() }) + it('renders and forwards the treatment starter prompt exposure', () => { + distribution.isCloud = true + const { emitted } = render(AgentPanel, { + props: { + entries: [], + historyGroups, + starterPromptAssignment: 'test', + attributeStarterPromptExperiment: true + }, + global: { + plugins: [i18n], + directives: { tooltip: {} }, + stubs: { WorkflowSelectorChip: true } + } + }) + + expect( + screen.getByRole('button', { + name: i18n.global.t('agent.suggestedPrompts.treatment.cloud.0') + }) + ).toBeVisible() + expect(emitted('starterPromptRendered')).toEqual([['test']]) + }) + it('shows the minimized run notice and disclaimer by default', () => { mount() diff --git a/src/workbench/extensions/agent/components/agent/AgentPanel.vue b/src/workbench/extensions/agent/components/agent/AgentPanel.vue index 7e7a14cc98c..14167a513c8 100644 --- a/src/workbench/extensions/agent/components/agent/AgentPanel.vue +++ b/src/workbench/extensions/agent/components/agent/AgentPanel.vue @@ -15,7 +15,8 @@ import Input from '@/components/ui/input/Input.vue' import { buildTooltipConfig } from '@/composables/useTooltipConfig' import type { AgentFreeUseNoticeMetadata, - AgentPaywallSurface + AgentPaywallSurface, + AgentStarterPromptAssignment } from '@/platform/telemetry/types' import type { FreeUseVariant } from '../../experiments/freeUsePlacement' @@ -76,7 +77,9 @@ const { historyGroups, editableTurnId = null, answeringAskIds = new Set(), - freeUsePlacement = 'control' + freeUsePlacement = 'control', + starterPromptAssignment = 'control', + attributeStarterPromptExperiment = false } = defineProps<{ entries: ConversationEntry[] userName?: string @@ -114,6 +117,8 @@ const { editableTurnId?: TurnId | null answeringAskIds?: ReadonlySet freeUsePlacement?: FreeUseVariant + starterPromptAssignment?: AgentStarterPromptAssignment + attributeStarterPromptExperiment?: boolean }>() const emit = defineEmits<{ send: [ @@ -146,6 +151,7 @@ const emit = defineEmits<{ openReferenceWorkflow: [workflowId: string, workflowName: string] showTarget: [] freeUseNotice: [metadata: AgentFreeUseNoticeMetadata] + starterPromptRendered: [assignment: AgentStarterPromptAssignment] }>() const targetNotice = computed(() => { @@ -385,6 +391,9 @@ defineExpose({ addAttachment, updateAttachment, removeAttachment }) -import { computed } from 'vue' +import { computed, onMounted } from 'vue' import { useI18n } from 'vue-i18n' import { cn } from '@comfyorg/tailwind-utils' import Button from '@/components/ui/button/Button.vue' import { FALLBACK_LOCALE } from '@/i18n' import { isCloud } from '@/platform/distribution/types' +import type { AgentStarterPromptAssignment } from '@/platform/telemetry/types' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' import { starterPromptAttribution } from '../../utils/starterPrompts' -const { userName } = defineProps<{ userName?: string }>() +const { + userName, + assignment = 'control', + attributeExperiment = false +} = defineProps<{ + userName?: string + assignment?: AgentStarterPromptAssignment + attributeExperiment?: boolean +}>() const emit = defineEmits<{ insert: [text: string, prompt: AgentStarterPromptAttribution] + rendered: [assignment: AgentStarterPromptAssignment] }>() const { t, te, tm, locale } = useI18n() @@ -20,10 +30,38 @@ const { t, te, tm, locale } = useI18n() const promptKey = isCloud ? 'agent.suggestedPrompts.cloud' : 'agent.suggestedPrompts.local' -const prompts = computed(() => tm(promptKey) as string[]) -const promptLocale = computed(() => - te(`${promptKey}.0`, locale.value) ? locale.value : FALLBACK_LOCALE +const treatmentPromptKey = 'agent.suggestedPrompts.treatment.cloud' +const hasTreatmentCopy = computed( + () => isCloud && te(`${treatmentPromptKey}.0`, locale.value) +) +const renderedAssignment: AgentStarterPromptAssignment = + assignment === 'test' && hasTreatmentCopy.value ? 'test' : 'control' +const selectedPromptKey = computed(() => + renderedAssignment === 'test' && hasTreatmentCopy.value + ? treatmentPromptKey + : promptKey +) +const prompts = computed(() => { + return tm(selectedPromptKey.value) as string[] +}) +const effectiveAssignment = computed(() => + selectedPromptKey.value === treatmentPromptKey ? 'test' : 'control' ) +const promptLocale = computed(() => { + return te(`${selectedPromptKey.value}.0`, locale.value) + ? locale.value + : FALLBACK_LOCALE +}) + +const eligibleAtMount = attributeExperiment && hasTreatmentCopy.value +const shouldAttributeExperiment = computed( + () => eligibleAtMount && hasTreatmentCopy.value +) + +onMounted(() => { + if (shouldAttributeExperiment.value) + emit('rendered', effectiveAssignment.value) +}) /** * One emit per click, carrying the slot's stable id rather than its text. Fires @@ -37,7 +75,8 @@ function onPromptClick(prompt: string, index: number): void { prompt, index, prompts.value.length, - promptLocale.value + promptLocale.value, + shouldAttributeExperiment.value ? effectiveAssignment.value : undefined ) ) } diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts index 6380fd0b19e..314ca750705 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts @@ -4,6 +4,7 @@ import { useTelemetry } from '@/platform/telemetry' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' +import { starterPromptAttribution } from '../../utils/starterPrompts' import type { ComposerAttachment } from '../../types/composerAttachment' import { useComposer } from './useComposer' @@ -17,7 +18,8 @@ const CHIP: AgentStarterPromptAttribution = { promptIndex: 1, promptCount: 5, promptTextHash: 'deadbeef', - locale: 'en' + locale: 'en', + assignment: 'control' } function setup(streaming = false) { @@ -174,13 +176,15 @@ describe('useComposer', () => { prompt_text_hash: 'deadbeef', locale: 'zh', click_id: expect.any(String), - draft_was_empty: true + draft_was_empty: true, + '$feature/agent-starter-prompt-set': 'control' }) // The id on the event is the id the send will be attributed with. const [[event]] = telemetry.trackAgentStarterPromptClicked.mock.calls expect(store.starterPrompt).toEqual({ id: 'slot_2', - clickId: event.click_id + clickId: event.click_id, + assignment: 'control' }) }) @@ -203,6 +207,21 @@ describe('useComposer', () => { ) }) + it('omits experiment properties when prompt attribution has no assignment', () => { + const { composer } = setup() + + composer.insert( + 'QA treatment', + starterPromptAttribution('QA treatment', 1, 5, 'en') + ) + + const [[event]] = telemetry.trackAgentStarterPromptClicked.mock.calls + expect(event).not.toHaveProperty('$feature/agent-starter-prompt-set') + expect(useAgentComposerStore().starterPrompt).not.toHaveProperty( + 'assignment' + ) + }) + it('treats a whitespace-only draft as empty for clean prompt attribution', () => { const { composer } = setup() composer.setText(' ') diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.ts b/src/workbench/extensions/agent/composables/agent/useComposer.ts index 407c582fd84..395d5fd6f4b 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.ts @@ -62,7 +62,13 @@ export function useComposer(options: UseComposerOptions) { return } const clickId = uuidv4() - store.markSuggestedPrompt({ id: starterPrompt.promptId, clickId }) + store.markSuggestedPrompt({ + id: starterPrompt.promptId, + clickId, + ...(starterPrompt.assignment + ? { assignment: starterPrompt.assignment } + : {}) + }) useTelemetry()?.trackAgentStarterPromptClicked({ prompt_id: starterPrompt.promptId, prompt_index: starterPrompt.promptIndex, @@ -70,7 +76,12 @@ export function useComposer(options: UseComposerOptions) { prompt_text_hash: starterPrompt.promptTextHash, locale: starterPrompt.locale, click_id: clickId, - draft_was_empty: draftWasEmpty + draft_was_empty: draftWasEmpty, + ...(starterPrompt.assignment + ? { + '$feature/agent-starter-prompt-set': starterPrompt.assignment + } + : {}) }) } diff --git a/src/workbench/extensions/agent/experiments/starterPromptSet.test.ts b/src/workbench/extensions/agent/experiments/starterPromptSet.test.ts new file mode 100644 index 00000000000..694c08f3d44 --- /dev/null +++ b/src/workbench/extensions/agent/experiments/starterPromptSet.test.ts @@ -0,0 +1,199 @@ +import { nextTick } from 'vue' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + authenticatedRemoteConfigState, + remoteConfig +} from '@/platform/remoteConfig/remoteConfig' +import { useTelemetry } from '@/platform/telemetry' +import { getDevOverride } from '@/utils/devFeatureFlagOverride' +import { getSessionOverride } from '@/utils/sessionFeatureFlagOverride' + +import { + STARTER_PROMPT_SET_FLAG, + useStarterPromptSet +} from './starterPromptSet' + +vi.mock(import('@/platform/telemetry')) +vi.mock(import('@/utils/devFeatureFlagOverride')) +vi.mock(import('@/utils/sessionFeatureFlagOverride')) + +describe('useStarterPromptSet', () => { + beforeEach(() => { + remoteConfig.value = {} + authenticatedRemoteConfigState.value = 'unloaded' + }) + + it('selects test but waits for the rendered surface before exposure', () => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + const { assignment, attributeExperiment, expose } = useStarterPromptSet() + + expect(assignment.value).toBe('control') + expect(attributeExperiment.value).toBe(false) + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + authenticatedRemoteConfigState.value = 'authenticated' + expect(assignment.value).toBe('test') + expect(attributeExperiment.value).toBe(true) + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenCalledWith({ + [`$feature/${STARTER_PROMPT_SET_FLAG}`]: 'test' + }) + }) + + it('does not expose a surface rendered before authentication', async () => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + const { assignment, expose } = useStarterPromptSet() + + expose('control') + authenticatedRemoteConfigState.value = 'authenticated' + await nextTick() + + expect(assignment.value).toBe('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + }) + + it('fails closed to control for an unknown assignment', () => { + Reflect.set(remoteConfig.value, STARTER_PROMPT_SET_FLAG, 'unexpected') + authenticatedRemoteConfigState.value = 'authenticated' + const { assignment } = useStarterPromptSet() + + expect(assignment.value).toBe('control') + }) + + it('does not expose an assignment that differs from the rendered surface', () => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + authenticatedRemoteConfigState.value = 'authenticated' + const { expose } = useStarterPromptSet() + + expose('control') + + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + }) + + it.for(['unloaded', 'error'] as const)( + 'does not carry a pre-auth %s surface into the experiment', + (configState) => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + authenticatedRemoteConfigState.value = configState + const { expose } = useStarterPromptSet() + + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + authenticatedRemoteConfigState.value = 'authenticated' + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenCalledWith({ + [`$feature/${STARTER_PROMPT_SET_FLAG}`]: 'test' + }) + } + ) + + it('requires a new rendered surface after authenticated config recovers', () => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + authenticatedRemoteConfigState.value = 'authenticated' + const { expose } = useStarterPromptSet() + + authenticatedRemoteConfigState.value = 'error' + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + authenticatedRemoteConfigState.value = 'authenticated' + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenCalledWith({ + [`$feature/${STARTER_PROMPT_SET_FLAG}`]: 'test' + }) + }) + + it('tracks each rendered surface with its current assignment', () => { + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'control' } + authenticatedRemoteConfigState.value = 'authenticated' + const { assignment, expose } = useStarterPromptSet() + + expose('control') + expect(assignment.value).toBe('control') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenCalledTimes(1) + + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + expose('test') + + expect(assignment.value).toBe('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenLastCalledWith({ + [`$feature/${STARTER_PROMPT_SET_FLAG}`]: 'test' + }) + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).toHaveBeenCalledTimes(2) + }) + + it.for(['authenticated', 'unloaded'] as const)( + 'renders QA overrides without attributing experiment events while %s', + (configState) => { + authenticatedRemoteConfigState.value = configState + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'control' } + vi.mocked(getSessionOverride).mockReturnValue('test') + + const { assignment, attributeExperiment, expose } = useStarterPromptSet() + + expect(assignment.value).toBe('test') + expect(attributeExperiment.value).toBe(false) + expose('test') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + } + ) + + it.for([ + { source: 'session', readOverride: getSessionOverride }, + { source: 'dev', readOverride: getDevOverride } + ])( + 'treats a null $source override as an unattributed control render', + ({ readOverride }) => { + authenticatedRemoteConfigState.value = 'authenticated' + remoteConfig.value = { [STARTER_PROMPT_SET_FLAG]: 'test' } + vi.mocked(readOverride).mockReturnValue(null) + + const { assignment, attributeExperiment, expose } = useStarterPromptSet() + + expect(assignment.value).toBe('control') + expect(attributeExperiment.value).toBe(false) + expose('control') + expect( + useTelemetry()?.trackAgentStarterPromptExposure + ).not.toHaveBeenCalled() + } + ) +}) diff --git a/src/workbench/extensions/agent/experiments/starterPromptSet.ts b/src/workbench/extensions/agent/experiments/starterPromptSet.ts new file mode 100644 index 00000000000..6ee79e1f817 --- /dev/null +++ b/src/workbench/extensions/agent/experiments/starterPromptSet.ts @@ -0,0 +1,62 @@ +import { computed } from 'vue' + +import { zGetFeaturesResponse } from '@comfyorg/ingest-types/zod' + +import { + isAuthenticatedConfigLoaded, + remoteConfig +} from '@/platform/remoteConfig/remoteConfig' +import { useTelemetry } from '@/platform/telemetry' +import type { AgentStarterPromptAssignment } from '@/platform/telemetry/types' +import { getDevOverride } from '@/utils/devFeatureFlagOverride' +import { getSessionOverride } from '@/utils/sessionFeatureFlagOverride' + +export const STARTER_PROMPT_SET_FLAG = 'agent-starter-prompt-set' +const starterPromptAssignmentSchema = + zGetFeaturesResponse.shape[STARTER_PROMPT_SET_FLAG].catch('control') + +function resolveAssignment(): { + assignment: AgentStarterPromptAssignment + hasQaOverride: boolean +} { + const sessionOverride = getSessionOverride(STARTER_PROMPT_SET_FLAG) + const override = + sessionOverride !== undefined + ? sessionOverride + : getDevOverride(STARTER_PROMPT_SET_FLAG) + const candidate = + override !== undefined + ? override + : remoteConfig.value[STARTER_PROMPT_SET_FLAG] + return { + assignment: starterPromptAssignmentSchema.parse(candidate), + hasQaOverride: override !== undefined + } +} + +export function useStarterPromptSet() { + const assignment = computed(() => { + const resolved = resolveAssignment() + return isAuthenticatedConfigLoaded.value || resolved.hasQaOverride + ? resolved.assignment + : 'control' + }) + const attributeExperiment = computed(() => { + const resolved = resolveAssignment() + return isAuthenticatedConfigLoaded.value && !resolved.hasQaOverride + }) + + const expose = (renderedAssignment: AgentStarterPromptAssignment) => { + if (!attributeExperiment.value || renderedAssignment !== assignment.value) + return + useTelemetry()?.trackAgentStarterPromptExposure({ + [`$feature/${STARTER_PROMPT_SET_FLAG}`]: assignment.value + }) + } + + return { + assignment, + attributeExperiment, + expose + } +} diff --git a/src/workbench/extensions/agent/utils/starterPrompts.test.ts b/src/workbench/extensions/agent/utils/starterPrompts.test.ts index e996816d39d..6abf9187ccb 100644 --- a/src/workbench/extensions/agent/utils/starterPrompts.test.ts +++ b/src/workbench/extensions/agent/utils/starterPrompts.test.ts @@ -16,6 +16,9 @@ describe('starter prompt identity', () => { expect(STARTER_PROMPT_IDS).toHaveLength( enMain.agent.suggestedPrompts.local.length ) + expect(STARTER_PROMPT_IDS).toHaveLength( + enMain.agent.suggestedPrompts.treatment.cloud.length + ) }) it('names a slot beyond the table rather than borrowing a neighbour', () => { diff --git a/src/workbench/extensions/agent/utils/starterPrompts.ts b/src/workbench/extensions/agent/utils/starterPrompts.ts index 7a1d0d457c7..49b3208797d 100644 --- a/src/workbench/extensions/agent/utils/starterPrompts.ts +++ b/src/workbench/extensions/agent/utils/starterPrompts.ts @@ -1,10 +1,14 @@ -import type { AgentStarterPromptId } from '@/platform/telemetry/types' +import type { + AgentStarterPromptAssignment, + AgentStarterPromptId +} from '@/platform/telemetry/types' import { hashPath } from '@/platform/workflow/persistence/base/hashUtil' /** - * Stable ids for the empty state's starter prompts, index-aligned with both - * `agent.suggestedPrompts.cloud` and `agent.suggestedPrompts.local` in the - * English locale, and with the icon list in `EmptyState.vue`. + * Stable ids for the empty state's starter prompts, index-aligned with + * `agent.suggestedPrompts.{cloud,local}` and the Cloud-only + * `agent.suggestedPrompts.treatment.cloud` list in the English locale, and with + * the icon list in `EmptyState.vue`. A Cloud slot shows different copy per arm. * * The copy is owned by product and is expected to change; these ids are not, * which is the whole point of them — an event keyed on display text cannot @@ -27,12 +31,14 @@ export interface AgentStarterPromptAttribution { promptCount: number promptTextHash: string locale: string + assignment?: AgentStarterPromptAssignment } /** What the composer holds until the draft is submitted, or replaced. */ export interface AgentStarterPromptSource { id: AgentStarterPromptId clickId: string + assignment?: AgentStarterPromptAssignment } export function starterPromptIdAt(index: number): AgentStarterPromptId { @@ -43,13 +49,15 @@ export function starterPromptAttribution( text: string, index: number, count: number, - locale: string + locale: string, + assignment?: AgentStarterPromptAssignment ): AgentStarterPromptAttribution { return { promptId: starterPromptIdAt(index), promptIndex: index, promptCount: count, promptTextHash: hashPath(text), - locale + locale, + ...(assignment ? { assignment } : {}) } }