diff --git a/browser_tests/fixtures/components/LiveCloudOnboarding.ts b/browser_tests/fixtures/components/LiveCloudOnboarding.ts new file mode 100644 index 00000000000..551d9931fa0 --- /dev/null +++ b/browser_tests/fixtures/components/LiveCloudOnboarding.ts @@ -0,0 +1,74 @@ +import { expect } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' + +export class LiveCloudOnboarding { + public readonly survey: Locator + public readonly canvas: Locator + public readonly next: Locator + public readonly submit: Locator + public readonly option: Locator + public readonly text: Locator + + constructor(private readonly page: Page) { + this.survey = page.getByRole('heading', { + name: "Let's get to know you", + exact: true + }) + this.canvas = page.locator('#graph-canvas') + this.next = page.getByRole('button', { name: 'Next', exact: true }) + this.submit = page.getByRole('button', { name: 'Submit', exact: true }) + this.option = page + .getByRole('button', { pressed: false }) + .and(page.locator('[aria-pressed="false"]:not([id$="-other"])')) + .first() + this.text = page.getByRole('textbox') + } + + async completeSurveyIfNeeded(frontend: string) { + const { survey, canvas, next, submit } = this + await expect(survey.or(canvas)).toBeVisible() + if (await survey.isVisible()) { + for (let step = 0; step < 20; step++) { + if (await this.answerSurveyQuestion(next, submit)) break + } + await expect( + submit, + 'Complete Cloud onboarding within 20 questions' + ).toBeEnabled() + const [response] = await Promise.all([ + this.page.waitForResponse( + (response) => + response.url() === `${frontend}/api/settings` && + response.request().method() === 'POST' + ), + submit.click() + ]) + expect(response.status(), 'Save Cloud onboarding survey').toBe(200) + await expect(survey).toBeHidden() + } + await expect(canvas).toBeVisible() + } + + private async answerSurveyQuestion(next: Locator, submit: Locator) { + const { option, text } = this + await expect(option.or(text)).toBeVisible() + const control = (await option.isVisible()) ? option : text + const id = await control.getAttribute('id') + if (!id) throw new Error('Cloud survey control has no ID') + const answered = this.page.locator(`[id=${JSON.stringify(id)}]`) + if (await option.isVisible()) await control.click() + else await control.fill('Automated billing E2E') + await expect + .poll( + async () => + !(await answered.isVisible()) || + ((await next.isVisible()) && (await next.isEnabled())) || + ((await submit.isVisible()) && (await submit.isEnabled())) + ) + .toBe(true) + if ((await submit.isVisible()) && (await submit.isEnabled())) return true + if ((await next.isVisible()) && (await next.isEnabled())) await next.click() + await expect(answered).toBeHidden() + return false + } +} diff --git a/browser_tests/fixtures/components/Tour.ts b/browser_tests/fixtures/components/Tour.ts index 9c1569cc8e2..2f92b1094be 100644 --- a/browser_tests/fixtures/components/Tour.ts +++ b/browser_tests/fixtures/components/Tour.ts @@ -17,6 +17,7 @@ export class OnboardingCoachmarks { /** The current spotlight step card (the dialog carrying a "Step N of M" label). */ public readonly spotlight: Locator public readonly card: Locator + public readonly cardSkipButton: Locator public readonly cardNextButton: Locator public readonly cardDoneButton: Locator @@ -31,10 +32,23 @@ export class OnboardingCoachmarks { }) this.spotlight = page.getByTestId('coach-spotlight') this.card = page.getByRole('dialog').filter({ hasText: /Step \d+ of \d+/ }) + this.cardSkipButton = this.card.getByRole('button', { + name: 'Skip', + exact: true + }) this.cardNextButton = this.card.getByRole('button', { name: 'Next' }) this.cardDoneButton = this.card.getByRole('button', { name: 'Done' }) } + async dismissWhenVisible() { + await this.page.addLocatorHandler(this.landing, async () => { + await this.landingSkipButton.click() + }) + await this.page.addLocatorHandler(this.card, async () => { + await this.cardSkipButton.click() + }) + } + /** The tour's in-app help button, which replays it past the seen-flag. */ replayButton(tour: CoachTour): Locator { return this.page.getByRole('button', { name: TOUR_REPLAY_BUTTONS[tour] }) diff --git a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts new file mode 100644 index 00000000000..cfc78a0660b --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts @@ -0,0 +1,99 @@ +import { getLiveCloudEnvironment } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + zBillingStatusResponse, + zCurrentWorkspaceResponse, + zListSavedPaymentMethodsResponse, + zPaymentPortalResponse +} from '@comfyorg/ingest-types/zod' +import type { APIRequestContext, TestInfo } from '@playwright/test' +import { expect } from '@playwright/test' +import type { z } from 'zod' + +export class LiveCloudBillingSession { + private readonly environment: ReturnType + constructor( + private readonly request: APIRequestContext, + private readonly backend: string, + private readonly headers: Record + ) { + this.environment = getLiveCloudEnvironment(backend) + } + + async read(path: string, schema: z.ZodType): Promise { + const response = await this.request + .get(new URL(path, this.backend).href, { + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } + + async post( + path: string, + data: Record, + schema: z.ZodType, + acceptedStatuses = [200] + ): Promise { + const response = await this.request + .post(new URL(path, this.backend).href, { + data, + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(acceptedStatuses, `POST ${path}`).toContain(response.status()) + return schema.parse(await response.json()) + } + + async assertNoCardAccount(testInfo: TestInfo) { + const workspace = await this.read( + '/api/workspaces/current', + zCurrentWorkspaceResponse + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const status = await this.read( + '/api/billing/status', + zBillingStatusResponse + ) + if (this.environment.stripeMode === 'live') { + const paymentMethods = await this.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect( + paymentMethods, + 'Checkout recovery requires an account without saved payment methods' + ).toHaveLength(0) + } + await testInfo.attach('billing-preflight.json', { + body: JSON.stringify({ + workspaceId: workspace.id, + isActive: status.is_active, + billingStatus: status.billing_status, + subscriptionTier: status.subscription_tier, + backend: this.backend + }), + contentType: 'application/json' + }) + if (status.is_active) expect(status.subscription_tier).toBe('FREE') + else + expect(['inactive', 'awaiting_payment_method']).toContain( + status.billing_status + ) + } + + async ensureProvisioned(returnUrl: string) { + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) + } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts new file mode 100644 index 00000000000..85d2f986ea2 --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts @@ -0,0 +1,251 @@ +import { getLiveCloudEnvironment } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + zBillingBalanceResponse, + zBillingOpStatusResponse, + zBillingStatusResponse, + zPaymentPortalResponse, + zPreviewSubscribeResponse, + zSubscribeResponse +} from '@comfyorg/ingest-types/zod' +import { expect } from '@playwright/test' +import type { Page, Response, TestInfo } from '@playwright/test' +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' + +function matchesBillingResponse( + response: Response, + frontend: string, + path: string +) { + return response.url() === new URL(path, frontend).href +} + +export class LiveCloudCheckout { + private readonly environment: ReturnType + readonly page: Page + readonly confirmation + readonly resumePayment + readonly paymentMethodsHeading + readonly savedCards + private readonly subscribe + + constructor( + private readonly comfyPage: ComfyPage, + private readonly frontend: string, + backend: string + ) { + this.environment = getLiveCloudEnvironment(backend) + this.page = comfyPage.page + this.paymentMethodsHeading = this.page.getByText(/^Payment methods?$/) + this.savedCards = this.page.getByText(/•••• \d{4}$/) + this.confirmation = this.page.getByRole('dialog').filter({ + has: this.page.getByRole('heading', { + name: 'Confirm your payment', + exact: true + }) + }) + this.resumePayment = this.confirmation.getByRole('button', { + name: 'Complete your payment', + exact: true + }) + this.subscribe = this.confirmation.getByRole('button', { + name: 'Subscribe to Creator', + exact: true + }) + } + + async open(retryTransientFailures = false) { + let preview: ReturnType | undefined + const loadPreview = async () => { + const [response] = await Promise.all([ + this.page.waitForResponse((response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/preview-subscribe' + ) + ), + this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) + ]) + expect(response.status()).toBe(200) + preview = zPreviewSubscribeResponse.parse(await response.json()) + } + if (retryTransientFailures) { + await expect(loadPreview).toPass({ timeout: 90_000 }) + } else { + await loadPreview() + } + if (!preview) throw new Error('Cloud billing preview was not loaded') + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await expect( + this.confirmation.getByText('Total due today', { exact: true }) + ).toBeVisible() + await expect(this.subscribe.or(this.resumePayment)).toBeEnabled() + return preview + } + + async completeCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo, + expectedPaymentMethodCount = 1 + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4242424242424242') + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + 'checkout-completed.png', + expectedPaymentMethodCount + ) + } + + async submitCard(checkout: Page, cardNumber: string) { + await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) + await checkout.getByLabel('Expiration', { exact: true }).fill('1230') + await checkout + .getByRole('textbox', { + name: 'Credit or debit card CVC/CVV', + exact: true + }) + .fill('123') + await checkout + .getByPlaceholder('Full name on card', { exact: true }) + .fill('Comfy Billing Test') + await checkout + .getByRole('button', { name: 'Enter address manually', exact: true }) + .click() + await checkout.locator('#billingAddressLine1').fill('123 Test Street') + await checkout.getByLabel('City', { exact: true }).fill('San Francisco') + await checkout.getByLabel('State', { exact: true }).selectOption('CA') + await checkout.getByLabel('ZIP', { exact: true }).fill('94107') + await checkout.locator('#enableStripePass').uncheck() + await checkout.getByRole('button', { name: /^Save/ }).click() + } + + async verifyCheckoutCompletion( + session: LiveCloudBillingSession, + testInfo: TestInfo, + balanceBeforeCents: number, + preview: ReturnType, + operationId: string, + screenshotName: string, + expectedPaymentMethodCount = 1 + ) { + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + phase: operation.phase + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ status: 'succeeded' }) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + is_active: true, + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const expectedBalance = + balanceBeforeCents + Number(preview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedBalance) + const paymentMethodCount = await this.verifySavedPaymentMethods( + session, + expectedPaymentMethodCount + ) + await testInfo.attach('checkout-completion.json', { + body: JSON.stringify({ + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + balanceBeforeCents, + balanceAfterCents: expectedBalance, + paymentMethodCount + }), + contentType: 'application/json' + }) + await this.page.goto(this.frontend) + await this.attachScreenshot(screenshotName) + return { + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + paymentMethodCount + } + } + + async verifySavedPaymentMethods( + session: LiveCloudBillingSession, + expectedCount: number + ) { + const portal = await session.post( + '/api/billing/payment-portal', + { return_url: this.frontend }, + zPaymentPortalResponse + ) + expect(new URL(portal.url).origin).toBe('https://checkout.comfy.org') + await this.page.goto(portal.url) + await expect(this.paymentMethodsHeading).toBeVisible() + await expect(this.savedCards).toHaveCount(expectedCount) + return expectedCount + } + + async startCheckout(action = this.subscribe.or(this.resumePayment)) { + await expect(action).toBeEnabled() + const [response, checkout] = await Promise.all([ + this.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/subscribe' + ) && response.request().method() === 'POST' + ), + this.page.waitForEvent('popup'), + action.click() + ]) + expect(response.status()).toBe(200) + const subscription = zSubscribeResponse.parse(await response.json()) + expect(subscription.status).toBe('needs_payment_method') + expect(subscription.billing_op_id).not.toBe('') + const mode = this.environment.stripeMode + const testCheckoutUrl = new RegExp( + `^https://checkout\\.(?:stripe\\.com|comfy\\.org)/c/pay/cs_${mode}_` + ) + expect(subscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(checkout).toHaveURL(testCheckoutUrl) + await expect( + checkout.getByRole('textbox', { name: 'Card number', exact: true }) + ).toBeVisible() + return { checkout, subscription } + } + + async attachScreenshot(name: string) { + await this.comfyPage.attachScreenshot(name, { runInCI: true }) + } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts new file mode 100644 index 00000000000..af892f0373d --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts @@ -0,0 +1,33 @@ +import { randomUUID } from 'node:crypto' + +import type { APIRequestContext } from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +const firebaseApiKey = 'AIzaSyDa_YMeyzV0SkVe92vBZ1tVikWBmOU5KVE' + +const signUpResponseSchema = z.object({ localId: z.string().min(1) }) + +export async function createDisposableCloudAccount(request: APIRequestContext) { + const runId = randomUUID() + const credentials = { + email: `billing-e2e-checkout-completion-${runId}@comfy.org`, + password: `BillingE2E-${randomUUID()}!` + } + const response = await request + .post( + `https://identitytoolkit.googleapis.com/v1/accounts:signUp?key=${firebaseApiKey}`, + { + data: { + ...credentials, + returnSecureToken: true + } + } + ) + .catch(() => { + throw new Error('Disposable Cloud account creation failed') + }) + expect(response.status(), 'Firebase account creation').toBe(200) + const account = signUpResponseSchema.parse(await response.json()) + return { ...credentials, userId: account.localId } +} diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 86249b68f91..0047a9af587 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -1,3 +1,4 @@ +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import { ComfyPage } from '@e2e/fixtures/ComfyPage' import { networkIsolationFixture as base } from '@e2e/fixtures/networkIsolationFixture' import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' @@ -5,13 +6,19 @@ import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBilling export const liveCloudBillingFixture = base.extend<{ comfyPage: ComfyPage + billingSession: LiveCloudBillingSession }>({ baseURL: process.env.PLAYWRIGHT_TEST_URL, liveCloudBillingConfig: async ({ liveCloudBillingConfig }, use) => { await use(liveCloudBillingConfig ?? loadLiveCloudBillingConfig()) }, - comfyPage: async ({ page, request }, use) => { - await signInToLiveCloud(page) + billingSession: async ({ page, liveCloudBillingConfig }, use) => { + await use(await signInToLiveCloud(page, liveCloudBillingConfig)) + }, + comfyPage: async ( + { page, request, billingSession: _billingSession }, + use + ) => { await use(new ComfyPage(page, request)) } }) diff --git a/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts new file mode 100644 index 00000000000..6479ca066fd --- /dev/null +++ b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts @@ -0,0 +1,71 @@ +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudCheckout' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' +import { createDisposableCloudAccount } from '@e2e/fixtures/helpers/LiveCloudDisposableAccount' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +export const liveCloudDisposableBillingFixture = + liveCloudBillingFixture.extend<{ + disposableAccount: Awaited> + disposableCheckout: { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout + } + }>({ + liveCloudBillingConfig: async ({}, use) => { + await use( + loadLiveCloudBillingConfig({ + allowAccountCreation: true, + allowPayments: true, + allowCheckout: true + }) + ) + }, + disposableAccount: async ({ context }, use) => { + await use(await createDisposableCloudAccount(context.request)) + }, + billingSession: async ( + { page, disposableAccount, liveCloudBillingConfig }, + use + ) => { + if (!liveCloudBillingConfig) + throw new Error('Live billing config required') + await use( + await signInToLiveCloud(page, { + ...liveCloudBillingConfig, + CLOUD_ACCOUNT_EMAIL: disposableAccount.email, + CLOUD_ACCOUNT_PASSWORD: disposableAccount.password + }) + ) + }, + disposableCheckout: async ( + { comfyPage, billingSession, liveCloudBillingConfig, disposableAccount }, + use, + testInfo + ) => { + if (!liveCloudBillingConfig) + throw new Error('Live billing config required') + await billingSession.assertNoCardAccount(testInfo) + await billingSession.ensureProvisioned( + liveCloudBillingConfig.PLAYWRIGHT_TEST_URL + ) + await testInfo.attach('disposable-account.json', { + body: JSON.stringify({ + email: disposableAccount.email, + userId: disposableAccount.userId + }), + contentType: 'application/json' + }) + await use({ + billingSession, + checkout: new LiveCloudCheckout( + comfyPage, + liveCloudBillingConfig.PLAYWRIGHT_TEST_URL, + liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL + ) + }) + } + }) + +export { expect } from '@playwright/test' diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index 965f19650bc..85702a0bf4a 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -1,4 +1,4 @@ -import type { APIRequestContext } from '@playwright/test' +import type { APIRequestContext, BrowserContext } from '@playwright/test' import { expect, test as base } from '@playwright/test' import { config as dotenvConfig } from 'dotenv' @@ -6,8 +6,10 @@ import { HERO_SLIDES } from '@/platform/cloud/onboarding/constants/heroSlides' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { installLiveCloudBillingRouting } from '@e2e/fixtures/utils/liveCloudBillingContext' import { + LIVE_CHECKOUT_ORIGINS, getBlockedRequestViolation, isLiveCloudMutationAllowed, + isLiveCloudAuxiliaryPost, isReportedViolation } from '@e2e/fixtures/utils/liveCloudBillingPolicy' import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' @@ -41,7 +43,8 @@ function guardApiRequests( !isLiveCloudMutationAllowed( url, method.toUpperCase(), - liveCloudBillingConfig + liveCloudBillingConfig, + options?.data ) ) { const message = `Mutation ${method} ${url.origin}${url.pathname}` @@ -55,6 +58,86 @@ function guardApiRequests( } } +async function installContextNetworkIsolation( + context: BrowserContext, + networkPolicy: NetworkPolicy, + baseURL?: string, + liveCloudBillingConfig?: LiveCloudBillingConfig +) { + const { origins, unexpected } = networkPolicy + const restore = guardApiRequests( + context.request, + origins, + unexpected, + baseURL, + liveCloudBillingConfig + ) + await context.route('**/*', async (route) => { + const url = new URL(route.request().url()) + if ( + origins.has(url.origin) || + (liveCloudBillingConfig && + (['GET', 'HEAD', 'OPTIONS'].includes(route.request().method()) || + isLiveCloudAuxiliaryPost(url, route.request().method()))) + ) { + await route.continue() + return + } + unexpected.add(getBlockedRequestViolation(url, route.request().method())) + await route.abort('blockedbyclient') + }) + await context.routeWebSocket( + (url) => !origins.has(url.origin.replace(/^ws/, 'http')), + async (socket) => { + const url = new URL(socket.url()) + unexpected.add(`WebSocket ${url.origin}${url.pathname}`) + await socket.close() + } + ) + if (liveCloudBillingConfig) { + await installLiveCloudBillingRouting( + context, + networkPolicy, + liveCloudBillingConfig + ) + return restore + } + await context.route('https://huggingface.co/**/resolve/**', (route) => + route.request().method() === 'HEAD' + ? route.fulfill({ status: 200, body: '' }) + : route.fallback() + ) + await context.route( + 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', + (route) => + route.fulfill({ contentType: 'application/javascript', body: '' }) + ) + await context.route('https://apis.google.com/js/api.js**', (route) => + route.fulfill({ status: 503, body: '' }) + ) + await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => + route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) + ) + for (const slide of HERO_SLIDES) { + await context.route(slide.poster, (route) => + route.fulfill({ path: assetPath('image32x32.webp') }) + ) + await context.route(slide.src, (route) => + route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) + ) + } + await context.route( + 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', + (route) => route.fulfill({ status: 404, body: '' }) + ) + await context.route( + 'https://{api,stagingapi}.comfy.org/nodes{,?*}', + (route) => route.fulfill({ status: 404, body: '' }) + ) + + return restore +} + export const networkIsolationFixture = base.extend<{ networkPolicy: NetworkPolicy liveCloudBillingConfig: LiveCloudBillingConfig | undefined @@ -72,7 +155,19 @@ export const networkIsolationFixture = base.extend<{ liveCloudBillingConfig.customerOrigin, 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', - 'https://dreamboothy-dev.firebaseapp.com' + liveCloudBillingConfig.environment.firebaseOrigin, + ...(liveCloudBillingConfig.allowPayments + ? [ + 'https://pay.google.com', + liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL.replace( + 'cloud.comfy.org', + 'platform.comfy.org' + ) + ] + : []), + ...(liveCloudBillingConfig.allowCheckout + ? LIVE_CHECKOUT_ORIGINS + : []) ]) : new Set( [ @@ -123,71 +218,12 @@ export const networkIsolationFixture = base.extend<{ { context, baseURL, networkPolicy, liveCloudBillingConfig }, use ) => { - const { origins, unexpected } = networkPolicy - const restore = guardApiRequests( - context.request, - origins, - unexpected, + const restore = await installContextNetworkIsolation( + context, + networkPolicy, baseURL, liveCloudBillingConfig ) - await context.route('**/*', async (route) => { - const url = new URL(route.request().url()) - if (origins.has(url.origin)) { - await route.continue() - return - } - unexpected.add(getBlockedRequestViolation(url, route.request().method())) - await route.abort('blockedbyclient') - }) - await context.routeWebSocket( - (url) => !origins.has(url.origin.replace(/^ws/, 'http')), - async (socket) => { - const url = new URL(socket.url()) - unexpected.add(`WebSocket ${url.origin}${url.pathname}`) - await socket.close() - } - ) - await context.route('https://huggingface.co/**/resolve/**', (route) => - route.request().method() === 'HEAD' - ? route.fulfill({ status: 200, body: '' }) - : route.fallback() - ) - await context.route( - 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', - (route) => - route.fulfill({ contentType: 'application/javascript', body: '' }) - ) - await context.route('https://apis.google.com/js/api.js**', (route) => - route.fulfill({ status: 503, body: '' }) - ) - await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => - route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) - ) - for (const slide of HERO_SLIDES) { - await context.route(slide.poster, (route) => - route.fulfill({ path: assetPath('image32x32.webp') }) - ) - await context.route(slide.src, (route) => - route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) - ) - } - await context.route( - 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', - (route) => route.fulfill({ status: 404, body: '' }) - ) - await context.route( - 'https://{api,stagingapi}.comfy.org/nodes{,?*}', - (route) => route.fulfill({ status: 404, body: '' }) - ) - - if (liveCloudBillingConfig) { - await installLiveCloudBillingRouting( - context, - networkPolicy, - liveCloudBillingConfig - ) - } await use(context) restore() } diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index f4f360dc08c..535923b728b 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -13,8 +13,19 @@ const sandboxConfig = { } describe('Live Cloud billing prerequisites', () => { + it('routes an omitted backend to the Staging customer API', () => { + const selected = liveCloudBillingConfigSchema.parse({ + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + CLOUD_ACCOUNT_EMAIL: sandboxConfig.CLOUD_ACCOUNT_EMAIL, + CLOUD_ACCOUNT_PASSWORD: sandboxConfig.CLOUD_ACCOUNT_PASSWORD + }) + expect(selected.PLAYWRIGHT_SETUP_API_URL).toBe( + 'https://stagingcloud.comfy.org' + ) + expect(selected.customerOrigin).toBe('https://stagingapi.comfy.org') + }) + it.for([ - 'https://cloud.comfy.org', 'https://testcloud.comfy.org.example.com', 'https://testcloud.comfy.org/redirect', 'http://testcloud.comfy.org', @@ -37,10 +48,16 @@ describe('Live Cloud billing prerequisites', () => { ).toBe(false) }) - it('accepts a local frontend backed by test Cloud', () => { - expect(liveCloudBillingConfigSchema.safeParse(sandboxConfig).success).toBe( - true - ) + it.for([ + ['https://stagingcloud.comfy.org', 'https://stagingapi.comfy.org'], + ['https://testcloud.comfy.org', 'https://testapi.comfy.org'], + ['https://cloud.comfy.org', 'https://api.comfy.org'] + ])('selects the matching customer API for %s', ([backend, customer]) => { + const selected = liveCloudBillingConfigSchema.parse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL: backend + }) + expect(selected.customerOrigin).toBe(customer) }) it('accepts an isolated preview deployment', () => { @@ -55,6 +72,7 @@ describe('Live Cloud billing prerequisites', () => { }) it('reports missing prerequisites without leaking credentials', () => { + vi.stubEnv('PLAYWRIGHT_TEST_URL', 'http://localhost:5173') vi.stubEnv('CLOUD_ACCOUNT_PASSWORD', 'private-value') vi.stubEnv('CLOUD_ACCOUNT_EMAIL', undefined) expect(loadLiveCloudBillingConfig).toThrow('CLOUD_ACCOUNT_EMAIL') @@ -87,3 +105,29 @@ describe('Frontend origins', () => { expect(config.PLAYWRIGHT_SETUP_API_URL).toBe('https://testcloud.comfy.org') }) }) + +describe('Disposable billing account boundaries', () => { + it.for(['allowPayments', 'allowAccountCreation'])( + 'rejects %s in production', + (permission) => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + [permission]: true + }).success + ).toBe(false) + } + ) + + it('requires permanent credentials unless creating a sandbox account', () => { + const config = { PLAYWRIGHT_TEST_URL: sandboxConfig.PLAYWRIGHT_TEST_URL } + expect(liveCloudBillingConfigSchema.safeParse(config).success).toBe(false) + expect( + liveCloudBillingConfigSchema.safeParse({ + ...config, + allowAccountCreation: true + }).success + ).toBe(true) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 12944db76d2..bc1c448d4d8 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -10,22 +10,64 @@ const originURL = z }, 'Use an origin without a path, query, credentials, or fragment') .transform((value) => new URL(value).origin) -function getLiveCloudCustomerOrigin(cloudOrigin: string): string | undefined { - if (cloudOrigin === 'https://testcloud.comfy.org') - return 'https://testapi.comfy.org' - if (cloudOrigin === 'https://stagingcloud.comfy.org') - return 'https://stagingapi.comfy.org' - if (/^https:\/\/pr-\d+\.testenvs\.comfy\.org$/.test(cloudOrigin)) { - return cloudOrigin.replace('.testenvs.', '-registry.testenvs.') +export function getLiveCloudEnvironment(cloudOrigin: string) { + const production = cloudOrigin === 'https://cloud.comfy.org' + const customerOrigin = + cloudOrigin === 'https://cloud.comfy.org' + ? 'https://api.comfy.org' + : cloudOrigin === 'https://testcloud.comfy.org' + ? 'https://testapi.comfy.org' + : cloudOrigin === 'https://stagingcloud.comfy.org' + ? 'https://stagingapi.comfy.org' + : /^https:\/\/pr-\d+\.testenvs\.comfy\.org$/.test(cloudOrigin) + ? cloudOrigin.replace('.testenvs.', '-registry.testenvs.') + : undefined + if (!customerOrigin) throw new Error('Unsupported Cloud environment') + const stripeMode: 'live' | 'test' = production ? 'live' : 'test' + return { + customerOrigin, + stripeMode, + firebaseOrigin: production + ? 'https://dreamboothy.firebaseapp.com' + : 'https://dreamboothy-dev.firebaseapp.com' } } export const liveCloudBillingConfigSchema = z .object({ PLAYWRIGHT_TEST_URL: originURL, - PLAYWRIGHT_SETUP_API_URL: originURL, - CLOUD_ACCOUNT_EMAIL: z.string().email(), - CLOUD_ACCOUNT_PASSWORD: z.string().min(1) + PLAYWRIGHT_SETUP_API_URL: originURL.default( + 'https://stagingcloud.comfy.org' + ), + CLOUD_ACCOUNT_EMAIL: z.string().email().optional(), + CLOUD_ACCOUNT_PASSWORD: z.string().min(1).optional(), + allowCheckout: z.boolean().default(false), + allowPayments: z.boolean().default(false), + allowAccountCreation: z.boolean().default(false) + }) + .superRefine((config, ctx) => { + if ( + (config.allowPayments || config.allowAccountCreation) && + config.PLAYWRIGHT_SETUP_API_URL === 'https://cloud.comfy.org' + ) + ctx.addIssue({ + code: 'custom', + path: ['PLAYWRIGHT_SETUP_API_URL'], + message: 'Payment tests require a sandbox Cloud backend' + }) + if (!config.allowAccountCreation) { + for (const field of [ + 'CLOUD_ACCOUNT_EMAIL', + 'CLOUD_ACCOUNT_PASSWORD' + ] as const) { + if (!config[field]) + ctx.addIssue({ + code: 'custom', + path: [field], + message: 'Required for permanent-account tests' + }) + } + } }) .superRefine((config, ctx) => { if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return @@ -40,31 +82,45 @@ export const liveCloudBillingConfigSchema = z ctx.addIssue({ code: 'custom', path: ['PLAYWRIGHT_TEST_URL'], - message: 'Use localhost or the selected sandbox origin' + message: 'Use localhost or the selected Cloud origin' }) } }) .transform((config, ctx) => { - const customerOrigin = getLiveCloudCustomerOrigin( - config.PLAYWRIGHT_SETUP_API_URL - ) - if (!customerOrigin) { + try { + const environment = getLiveCloudEnvironment( + config.PLAYWRIGHT_SETUP_API_URL + ) + return { + ...config, + customerOrigin: environment.customerOrigin, + environment + } + } catch { ctx.addIssue({ code: 'custom', path: ['PLAYWRIGHT_SETUP_API_URL'], - message: 'Use a Cloud test, staging, or PR preview origin' + message: 'Use a Cloud production, staging, test, or PR preview origin' }) return z.NEVER } - return { ...config, customerOrigin } }) export type LiveCloudBillingConfig = z.infer< typeof liveCloudBillingConfigSchema > -export function loadLiveCloudBillingConfig() { - const result = liveCloudBillingConfigSchema.safeParse(process.env) +export function loadLiveCloudBillingConfig( + options: { + allowPayments?: boolean + allowAccountCreation?: boolean + allowCheckout?: boolean + } = {} +) { + const result = liveCloudBillingConfigSchema.safeParse({ + ...process.env, + ...options + }) if (!result.success) { throw new Error( `Cloud billing prerequisites: ${result.error.issues diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index 2f1662865c4..8f67266bbc2 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -1,11 +1,15 @@ import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' -import type { BrowserContext, Page } from '@playwright/test' +import type { BrowserContext, Page, Route } from '@playwright/test' +import { expect } from '@playwright/test' -import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { OnboardingCoachmarks } from '@e2e/fixtures/components/Tour' +import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' +import { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { + LIVE_CHECKOUT_ORIGINS, getLiveCloudDestinationViolation, isLiveCloudMutationAllowed } from '@e2e/fixtures/utils/liveCloudBillingPolicy' @@ -35,8 +39,14 @@ export async function installLiveCloudBillingRouting( ? new URL(url.pathname + url.search, config.PLAYWRIGHT_SETUP_API_URL) : url if ( - networkPolicy.origins.has(url.origin) && - !isLiveCloudMutationAllowed(targetUrl, request.method(), config) + (networkPolicy.origins.has(url.origin) || + LIVE_CHECKOUT_ORIGINS.includes(url.origin)) && + !isLiveCloudMutationAllowed( + targetUrl, + request.method(), + config, + url.pathname === '/api/settings' ? request.postDataJSON() : undefined + ) ) { networkPolicy.unexpected.add( `Mutation ${request.method()} ${url.origin}${url.pathname}` @@ -45,26 +55,33 @@ export async function installLiveCloudBillingRouting( return } if (isCloudProxyRequest) { - try { - const response = await route.fetch({ - url: targetUrl.href, - maxRedirects: 0 - }) - await route.fulfill({ response }) - } catch { - throw new Error(`Cloud proxy request failed: ${url.pathname}`) - } + await forwardCloudRequest(route, targetUrl) return } await route.fallback() }) } -export async function signInToLiveCloud(page: Page) { - const config = loadLiveCloudBillingConfig() - await new FeatureFlagHelper(page).seedFlags({ - onboarding_survey_enabled: false - }) +async function forwardCloudRequest(route: Route, targetUrl: URL) { + try { + const response = await route.fetch({ + url: targetUrl.href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + } catch { + if (route.request().frame().page().isClosed()) return + throw new Error(`Cloud proxy request failed: ${targetUrl.pathname}`) + } +} + +export async function signInToLiveCloud( + page: Page, + config = loadLiveCloudBillingConfig() +) { + if (!config.CLOUD_ACCOUNT_EMAIL || !config.CLOUD_ACCOUNT_PASSWORD) + throw new Error('Live Cloud sign-in requires account credentials') + await new OnboardingCoachmarks(page).dismissWhenVisible() await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) await page .getByRole('button', { name: 'Use email instead', exact: true }) @@ -84,4 +101,21 @@ export async function signInToLiveCloud(page: Page) { page.getByRole('button', { name: 'Sign in', exact: true }).click() ]) zBillingStatusResponse.parse(await response.json()) + const authorization = await response.request().headerValue('authorization') + if (!authorization) throw new Error('Missing billing authorization') + if (config.allowAccountCreation) { + await new LiveCloudOnboarding(page).completeSurveyIfNeeded( + config.PLAYWRIGHT_TEST_URL + ) + } else { + await expect( + page.locator('#graph-canvas'), + 'Permanent accounts must complete onboarding when the survey flag is enabled' + ).toBeVisible() + } + return new LiveCloudBillingSession( + page.request, + config.PLAYWRIGHT_SETUP_API_URL, + { authorization } + ) } diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index 71c5cdbe9d9..4ddd7c7eac3 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -1,18 +1,26 @@ import { describe, expect, it } from 'vitest' -import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + getLiveCloudEnvironment, + liveCloudBillingConfigSchema +} from '@e2e/fixtures/utils/liveCloudBillingConfig' import { getBlockedRequestViolation, getLiveCloudDestinationViolation, isLiveCloudMutationAllowed, + isLiveCloudAuxiliaryPost, isReportedViolation } from '@e2e/fixtures/utils/liveCloudBillingPolicy' const config = { PLAYWRIGHT_TEST_URL: 'http://localhost:5173', PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', - customerOrigin: 'https://testapi.comfy.org' + customerOrigin: 'https://testapi.comfy.org', + environment: getLiveCloudEnvironment('https://testcloud.comfy.org'), + allowCheckout: false, + allowPayments: false, + allowAccountCreation: false } describe('live Cloud mutation policy', () => { @@ -23,6 +31,17 @@ describe('live Cloud mutation policy', () => { { method: 'POST', path: '/customers', allowed: false }, { method: 'POST', path: '/api/auth/token', allowed: true }, { method: 'POST', path: '/api/auth/session', allowed: true }, + { + method: 'POST', + path: '/api/settings/Comfy.InstalledVersion', + allowed: true + }, + { method: 'POST', path: '/api/settings/other', allowed: false }, + { + method: 'POST', + path: '/api/settings/Comfy.OnboardingCoachmarks.Seen', + allowed: true + }, { method: 'POST', path: '/api/billing/subscribe', allowed: false }, { method: 'POST', path: '/api/billing/payment-portal', allowed: false }, { method: 'POST', path: '/customers/credit', allowed: false }, @@ -61,6 +80,142 @@ describe('live Cloud mutation policy', () => { allowed ) }) + + it.for([ + { data: { onboarding_survey: { intent: 'exploring' } }, allowed: true }, + { data: undefined, allowed: false }, + { data: {}, allowed: false }, + { data: { onboarding_survey: null }, allowed: false }, + { data: { unrelated: true }, allowed: false }, + { + data: { onboarding_survey: { intent: 'exploring' }, unrelated: true }, + allowed: false + } + ])('survey settings $data allowed=$allowed', ({ data, allowed }) => { + expect( + isLiveCloudMutationAllowed( + new URL('/api/settings', config.PLAYWRIGHT_SETUP_API_URL), + 'POST', + config, + data + ) + ).toBe(allowed) + expect( + isLiveCloudMutationAllowed( + new URL('/api/settings', 'https://stagingcloud.comfy.org'), + 'POST', + config, + data + ) + ).toBe(false) + }) +}) + +describe('live Cloud checkout mutation policy', () => { + it('blocks checkout initialization when checkout is disabled', () => { + expect( + isLiveCloudMutationAllowed( + new URL('https://api.stripe.com/v1/payment_pages/cs_test_example/init'), + 'POST', + config + ) + ).toBe(false) + }) + + it.for([ + { + url: 'https://testcloud.comfy.org/api/billing/preview-subscribe', + method: 'POST', + allowed: true + }, + { + url: 'https://testcloud.comfy.org/api/billing/subscribe', + method: 'POST', + allowed: true + }, + { + url: 'https://stagingcloud.comfy.org/api/billing/subscribe', + method: 'POST', + allowed: false + }, + { + url: 'https://testcloud.comfy.org/api/billing/payment-portal', + method: 'POST', + allowed: false + }, + { + url: 'https://testcloud.comfy.org/api/billing/subscribe', + method: 'DELETE', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'POST', + allowed: true + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_live_example/init', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_methods', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'PUT', + allowed: false + }, + { + url: 'https://checkout.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'POST', + allowed: false + }, + { url: 'https://r.stripe.com/b', method: 'POST', allowed: true }, + { url: 'https://r.stripe.com/0', method: 'POST', allowed: true }, + { url: 'https://r.stripe.com/other', method: 'POST', allowed: false }, + { url: 'https://m.stripe.com/6', method: 'POST', allowed: true } + ])( + '$method $url allowed=$allowed with checkout enabled', + ({ url, method, allowed }) => { + expect( + isLiveCloudMutationAllowed(new URL(url), method, { + ...config, + allowCheckout: true + }) + ).toBe(allowed) + } + ) +}) + +describe('production checkout', () => { + it.for([ + { mode: 'live', action: 'init', allowed: true }, + { mode: 'test', action: 'init', allowed: false }, + { mode: 'live', action: 'confirm', allowed: false } + ])('$mode checkout $action allowed=$allowed', ({ mode, action, allowed }) => { + expect( + isLiveCloudMutationAllowed( + new URL( + `https://api.stripe.com/v1/payment_pages/cs_${mode}_example/${action}` + ), + 'POST', + { + ...config, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + environment: getLiveCloudEnvironment('https://cloud.comfy.org'), + allowCheckout: true + } + ) + ).toBe(allowed) + }) }) describe('live Cloud mutation origins', () => { @@ -80,6 +235,10 @@ describe('live Cloud mutation origins', () => { }) it.for([ + { + cloud: 'https://cloud.comfy.org', + customer: 'https://api.comfy.org' + }, { cloud: 'https://testcloud.comfy.org', customer: 'https://testapi.comfy.org' @@ -200,3 +359,80 @@ describe('blocked request violations', () => { } ) }) + +describe('Disposable payment permissions', () => { + it.for([ + { + url: 'https://identitytoolkit.googleapis.com/v1/accounts:signUp', + permission: 'allowAccountCreation', + opposite: 'allowPayments' + }, + { + url: 'https://api.stripe.com/v1/payment_methods', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://checkout.comfy.org/ajax/metrics_batch', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://testcloud.comfy.org/api/billing/payment-portal', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + } + ])('requires $permission for $url', ({ url, permission, opposite }) => { + const enabled = { ...config, [permission]: true } + expect(isLiveCloudMutationAllowed(new URL(url), 'POST', config)).toBe(false) + expect(isLiveCloudMutationAllowed(new URL(url), 'POST', enabled)).toBe(true) + expect( + isLiveCloudMutationAllowed(new URL(url), 'POST', { + ...config, + [opposite]: true + }) + ).toBe(false) + expect( + isLiveCloudMutationAllowed(new URL(url), 'POST', { + ...enabled, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + environment: getLiveCloudEnvironment('https://cloud.comfy.org') + }) + ).toBe(false) + expect(isLiveCloudMutationAllowed(new URL(url), 'PUT', enabled)).toBe(false) + const wrongOrigin = new URL(url) + wrongOrigin.hostname = 'untrusted.example.com' + expect(isLiveCloudMutationAllowed(wrongOrigin, 'POST', enabled)).toBe(false) + expect( + isLiveCloudMutationAllowed(new URL(`${url}/unexpected`), 'POST', enabled) + ).toBe(false) + }) +}) + +describe('Live Cloud auxiliary services', () => { + it.for([ + { url: 'https://t.comfy.org/flags/?v=2', method: 'POST', allowed: true }, + { + url: 'https://www.google-analytics.com/g/collect', + method: 'POST', + allowed: true + }, + { url: 'https://mp.comfy.org/track/', method: 'POST', allowed: true }, + { url: 'https://cdp.customer.io/v1/i', method: 'POST', allowed: true }, + { + url: 'https://unknown.example.com/collect', + method: 'POST', + allowed: false + }, + { url: 'https://t.comfy.org/admin', method: 'POST', allowed: false }, + { url: 'https://t.comfy.org/flags/', method: 'DELETE', allowed: false }, + { url: 'http://t.comfy.org/flags/', method: 'POST', allowed: false } + ])('$method $url allowed=$allowed', ({ url, method, allowed }) => { + expect(isLiveCloudAuxiliaryPost(new URL(url), method)).toBe(allowed) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index db2327fa970..02b4950f8d9 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -17,6 +17,52 @@ export function getLiveCloudDestinationViolation( } } +export const LIVE_CHECKOUT_ORIGINS = [ + 'https://checkout.stripe.com', + 'https://checkout.comfy.org', + 'https://api.stripe.com', + 'https://js.stripe.com', + 'https://m.stripe.network', + 'https://m.stripe.com', + 'https://r.stripe.com', + 'https://q.stripe.com', + 'https://b.stripecdn.com', + 'https://newassets.hcaptcha.com' +] + +const AUXILIARY_POST_URLS = new Set([ + 'https://www.google-analytics.com/g/collect', + 'https://www.google.com/ccm/collect', + 'https://ad.doubleclick.net/ccm/s/collect', + 'https://www.google.com/g/collect', + 'https://www.google.com/rmkt/collect/17902754986/', + 'https://analytics.google.com/g/collect', + 'https://stats.g.doubleclick.net/g/collect', + 'https://t.comfy.org/flags/', + 'https://cdp.customer.io/v1/p', + 'https://mpc-prod-27-s6uit34pua-uk.a.run.app/events', + 'https://5z-2b6b7616f94640c2840d1841e1ac24c3.ecs.us-east-1.on.aws/events', + 'https://www.facebook.com/tr/', + 'https://www.google.com/pagead/form-data/17902754986', + 'https://www.google.com/ccm/form-data/17902754986', + 'https://google.com/pagead/form-data/17902754986', + 'https://google.com/ccm/form-data/17902754986', + 'https://px.ads.linkedin.com/wa/', + 'https://consumer.cloud.gist.build/api/v4/users', + 'https://cdp.customer.io/v1/i', + 'https://www.google.com/measurement/conversion', + 'https://e2.sy-d.io/events', + 'https://cdp.customer.io/v1/t', + 'https://mp.comfy.org/track/', + 'https://mp.comfy.org/engage/' +]) + +export function isLiveCloudAuxiliaryPost(url: URL, method: string): boolean { + return ( + method === 'POST' && AUXILIARY_POST_URLS.has(`${url.origin}${url.pathname}`) + ) +} + const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] export function getBlockedRequestViolation(url: URL, method: string): string { @@ -37,11 +83,36 @@ export function isLiveCloudMutationAllowed( method: string, config: Pick< LiveCloudBillingConfig, - 'PLAYWRIGHT_SETUP_API_URL' | 'customerOrigin' - > + | 'PLAYWRIGHT_SETUP_API_URL' + | 'customerOrigin' + | 'environment' + | 'allowCheckout' + | 'allowPayments' + | 'allowAccountCreation' + >, + data?: unknown ): boolean { if (SAFE_METHODS.includes(method)) return true if (method !== 'POST') return false + if ( + url.origin === config.PLAYWRIGHT_SETUP_API_URL && + url.pathname === '/api/settings' + ) { + return isOnboardingSurveyUpdate(data) + } + if ( + config.allowAccountCreation && + config.environment.stripeMode === 'test' && + url.origin === 'https://identitytoolkit.googleapis.com' && + url.pathname === '/v1/accounts:signUp' + ) + return true + if ( + config.allowPayments && + config.environment.stripeMode === 'test' && + isPaymentPostAllowed(url, config.PLAYWRIGHT_SETUP_API_URL) + ) + return true if (url.origin === 'https://identitytoolkit.googleapis.com') { return ['/v1/accounts:signInWithPassword', '/v1/accounts:lookup'].includes( url.pathname @@ -50,11 +121,74 @@ export function isLiveCloudMutationAllowed( if (url.origin === 'https://securetoken.googleapis.com') { return url.pathname === '/v1/token' } + if ( + config.allowCheckout && + isCheckoutPostAllowed( + url, + config.PLAYWRIGHT_SETUP_API_URL, + config.environment.stripeMode + ) + ) + return true if (url.pathname === '/customers') { return url.origin === config.customerOrigin } return ( url.origin === config.PLAYWRIGHT_SETUP_API_URL && - ['/api/auth/token', '/api/auth/session'].includes(url.pathname) + [ + '/api/auth/token', + '/api/auth/session', + '/api/settings/Comfy.InstalledVersion', + '/api/settings/Comfy.OnboardingCoachmarks.Seen' + ].includes(url.pathname) + ) +} + +function isOnboardingSurveyUpdate(data: unknown): boolean { + return ( + typeof data === 'object' && + data !== null && + Object.keys(data).length === 1 && + 'onboarding_survey' in data && + typeof data.onboarding_survey === 'object' && + data.onboarding_survey !== null + ) +} + +function isPaymentPostAllowed(url: URL, backend: string): boolean { + if ( + url.origin === 'https://checkout.comfy.org' && + url.pathname === '/ajax/metrics_batch' + ) + return true + if (url.origin === backend && url.pathname === '/api/billing/payment-portal') + return true + return ( + url.origin === 'https://api.stripe.com' && + (url.pathname === '/v1/payment_methods' || + /^\/v1\/payment_pages\/cs_test_[A-Za-z0-9]+\/confirm$/.test(url.pathname)) + ) +} + +function isCheckoutPostAllowed( + url: URL, + backend: string, + mode: 'live' | 'test' +): boolean { + if (url.origin === backend) { + return [ + '/api/billing/preview-subscribe', + '/api/billing/subscribe' + ].includes(url.pathname) + } + if (url.origin === 'https://api.stripe.com') { + return new RegExp(`^/v1/payment_pages/cs_${mode}_[A-Za-z0-9]+/init$`).test( + url.pathname + ) + } + return ( + (url.origin === 'https://r.stripe.com' && + ['/b', '/0'].includes(url.pathname)) || + (url.origin === 'https://m.stripe.com' && url.pathname === '/6') ) } diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 2a8a6ef88fd..99d02154f50 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -1,84 +1,109 @@ +import { globSync } from 'node:fs' import { spawnSync } from 'node:child_process' import { describe, expect, it } from 'vitest' import { z } from 'zod' +function collectLiveBilling(enabled: '0' | '1', args: string[]) { + const result = spawnSync( + 'pnpm', + ['exec', 'playwright', 'test', ...args, '--list', '--reporter=json'], + { + cwd: process.cwd(), + env: { + ...process.env, + PLAYWRIGHT_CLOUD_LIVE: enabled, + DISTRIBUTION: 'cloud' + }, + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 10_000_000 + } + ) + expect(result.error).toBeUndefined() + const report = z + .object({ + suites: z.array( + z.object({ + suites: z.array( + z.object({ + specs: z.array( + z.object({ + file: z.string(), + tests: z.array(z.object({ projectName: z.string() })) + }) + ) + }) + ) + }) + ), + config: z.object({ + globalSetup: z.unknown(), + globalTeardown: z.unknown(), + projects: z.array(z.object({ name: z.string() })) + }) + }) + .parse(JSON.parse(result.stdout)) + const specs = report.suites.flatMap((suite) => + suite.suites.flatMap((child) => child.specs) + ) + const files = specs.map((spec) => spec.file.replaceAll('\\', '/')) + return { status: result.status, config: report.config, files, specs } +} + describe('Live billing opt-in', () => { + it('excludes live billing when disabled', { timeout: 90_000 }, () => { + const { status, config, files } = collectLiveBilling('0', [ + 'tests/liveCloud' + ]) + expect(status).toBe(1) + expect(config.projects.map((project) => project.name)).not.toContain( + 'cloud-live' + ) + expect(files).toEqual([]) + expect(config.globalSetup).toBeTruthy() + expect(config.globalTeardown).toBeTruthy() + }) + it.for([ + { name: 'without a project filter', args: [] }, { - name: 'disabled', - enabled: '0', - args: ['tests/liveCloud'], - status: 1, - count: 0, - hooks: true - }, - { - name: 'enabled without a project filter', - enabled: '1', - args: [], - status: 0, - count: 1, - hooks: false - }, - { - name: 'enabled with a project filter', - enabled: '1', - args: ['--project=cloud-live'], - status: 0, - count: 1, - hooks: false + name: 'with a project filter', + args: [ + '--project=cloud-live', + '--project=cloud-live-disposable', + '--project=cloud-live-paid' + ] } ])( - 'collects live billing only when explicitly enabled: $name', + 'collects only live billing when enabled $name', { timeout: 90_000 }, - ({ enabled, args, status, count, hooks }) => { - const result = spawnSync( - 'pnpm', - ['exec', 'playwright', 'test', ...args, '--list', '--reporter=json'], - { - cwd: process.cwd(), - env: { - ...process.env, - PLAYWRIGHT_CLOUD_LIVE: enabled, - DISTRIBUTION: 'cloud' - }, - encoding: 'utf8', - timeout: 60_000, - maxBuffer: 10_000_000 - } - ) - expect(result.error).toBeUndefined() - expect(result.status).toBe(status) - const report = z - .object({ - suites: z.array(z.unknown()), - config: z.object({ - globalSetup: z.unknown(), - globalTeardown: z.unknown(), - projects: z.array(z.object({ name: z.string() })) - }) - }) - .parse(JSON.parse(result.stdout)) - expect( - report.config.projects.some((project) => project.name === 'cloud-live') - ).toBe(enabled === '1') - expect( - report.config.projects.every((project) => project.name === 'cloud-live') - ).toBe(!hooks) - const suites = z - .array( - z.object({ - suites: z.array(z.object({ specs: z.array(z.unknown()) })) - }) - ) - .parse(report.suites) - const specs = suites.flatMap((suite) => - suite.suites.flatMap((child) => child.specs) + ({ args }) => { + const { status, config, files, specs } = collectLiveBilling('1', args) + expect(status).toBe(0) + expect(config.projects.map((project) => project.name)).toEqual([ + 'cloud-live', + 'cloud-live-disposable', + 'cloud-live-paid' + ]) + expect(files.every((file) => file.startsWith('tests/liveCloud/'))).toBe( + true ) - expect(specs).toHaveLength(count) - expect(Boolean(report.config.globalSetup)).toBe(hooks) - expect(Boolean(report.config.globalTeardown)).toBe(hooks) + for (const spec of specs) { + const file = spec.file.replaceAll('\\', '/') + const project = file.startsWith('tests/liveCloud/disposable/') + ? 'cloud-live-disposable' + : file.startsWith('tests/liveCloud/paid/') + ? 'cloud-live-paid' + : 'cloud-live' + expect(spec.tests.map((test) => test.projectName)).toEqual([project]) + } + const expectedFiles = globSync( + 'browser_tests/tests/liveCloud/**/*.spec.ts' + ).map((file) => file.replace('browser_tests/', '').replaceAll('\\', '/')) + expect([...new Set(files)].sort()).toEqual(expectedFiles.sort()) + expect(config.globalSetup).toBeFalsy() + expect(config.globalTeardown).toBeFalsy() } ) }) diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index 39bf61729b0..697356524ec 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -8,12 +8,15 @@ import { join } from 'node:path' import { promisify } from 'node:util' import type { Duplex } from 'node:stream' -import { expect } from '@playwright/test' - -import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { + expect, + liveCloudDisposableBillingFixture +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' +import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' +import { OnboardingCoachmarks } from '@e2e/fixtures/components/Tour' import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' -const test = liveCloudBillingFixture.extend<{ +const test = liveCloudDisposableBillingFixture.extend<{ sandboxProxy: { server: string; requests: string[] } }>({ sandboxProxy: async ({ liveCloudBillingConfig }, use) => { @@ -94,6 +97,97 @@ const test = liveCloudBillingFixture.extend<{ }) test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { + for (const testId of ['coach-landing', 'coach-card']) { + test(`dismisses late ${testId} across navigation without closing billing`, async ({ + page + }) => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ + contentType: 'text/html', + body: ` + +
+ +
+ + ` + }) + ) + await new OnboardingCoachmarks(page).dismissWhenVisible() + for (let visit = 0; visit < 2; visit++) { + await page.goto('http://localhost:5173/') + await page.getByRole('button', { name: 'Show tutorial' }).click() + await page.getByRole('button', { name: 'Continue billing' }).click() + await expect(page.getByTestId(testId)).toBeHidden() + await expect( + page.getByRole('dialog', { name: 'Confirm your payment' }) + ).toBeVisible() + } + }) + } + + test('completes disposable survey questions before waiting for the canvas', async ({ + page + }) => { + await test.step('Open a survey with option and text questions', async () => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ + contentType: 'text/html', + body: ` +

Let's get to know you

+ + + + + + ` + }) + ) + await page.goto('http://localhost:5173/') + }) + + await test.step('Advance with Next, submit all answers and reach the canvas', async () => { + const [request] = await Promise.all([ + page.waitForRequest( + (request) => + request.url() === 'http://localhost:5173/api/settings' && + request.method() === 'POST' + ), + new LiveCloudOnboarding(page).completeSurveyIfNeeded( + 'http://localhost:5173' + ) + ]) + expect(request.postDataJSON()).toEqual({ + onboarding_survey: { + intent: 'exploring', + usage: 'Automated billing E2E', + details: 'Automated billing E2E' + } + }) + await expect(page.locator('#graph-canvas')).toBeVisible() + }) + }) + test('rewrites browser auth to Cloud and provisions customers at the Comfy API', async ({ page, sandboxProxy @@ -116,13 +210,66 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ) ) ).toBe('POST https://testapi.comfy.org/customers') + expect( + await page.evaluate(() => + fetch('/api/settings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ onboarding_survey: { intent: 'exploring' } }) + }).then((response) => response.text()) + ) + ).toBe('POST https://testcloud.comfy.org/api/settings') expect(sandboxProxy.requests).toEqual([ 'POST https://testcloud.comfy.org/api/auth/token', - 'POST https://testapi.comfy.org/customers' + 'POST https://testapi.comfy.org/customers', + 'POST https://testcloud.comfy.org/api/settings' ]) }) - for (const path of ['/api/billing/subscribe', '/customers']) { + test('uses upstream responses for assets, location, Google auth, and flags', async ({ + context, + sandboxProxy + }) => { + const page = await test.step('Open the local frontend', async () => { + const page = await context.newPage() + await page.route('http://localhost:5173/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://localhost:5173/') + return page + }) + + await test.step('Verify auxiliary services use upstream responses', async () => { + for (const url of [ + 'https://cloud.comfy.org/cdn-cgi/trace', + 'https://apis.google.com/js/api.js', + 'https://huggingface.co/example/resolve/main/model.safetensors', + 'https://t.comfy.org/flags/' + ]) { + expect( + await page.evaluate(async (url) => { + const response = await fetch(url) + return await response.text() + }, url) + ).toBe(`GET ${url}`) + } + expect( + await page.evaluate(() => + fetch('https://t.comfy.org/flags/', { method: 'POST' }).then( + (response) => response.text() + ) + ) + ).toBe('POST https://t.comfy.org/flags/') + expect(sandboxProxy.requests).toContain('GET https://t.comfy.org/flags/') + }) + }) + + for (const path of [ + '/api/billing/subscribe', + '/customers', + 'http://127.0.0.1:39281/collect', + 'https://api.stripe.com/v1/payment_pages/cs_test_example/init' + ]) { test(`blocks POST ${path} and fails even when the app catches it`, async ({ page, networkPolicy @@ -142,7 +289,7 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ) ).toBe('blocked') expect([...networkPolicy.unexpected]).toEqual([ - `Mutation POST http://localhost:5173${path}` + `Mutation POST ${new URL(path, 'http://localhost:5173').href}` ]) test.fail(true, 'The recorded mutation must fail fixture teardown') }) @@ -166,6 +313,15 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { expect(await auth.text()).toBe( 'POST https://testcloud.comfy.org/api/auth/session' ) + const survey = await api.post( + 'https://testcloud.comfy.org/api/settings', + { + data: { onboarding_survey: { intent: 'exploring' } } + } + ) + expect(await survey.text()).toBe( + 'POST https://testcloud.comfy.org/api/settings' + ) await expect( api.post('https://testcloud.comfy.org/customers') ).rejects.toThrow('Forbidden live Cloud request') @@ -183,7 +339,8 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ).rejects.toThrow('Forbidden live Cloud request') expect(sandboxProxy.requests).toEqual([ 'POST https://testapi.comfy.org/customers', - 'POST https://testcloud.comfy.org/api/auth/session' + 'POST https://testcloud.comfy.org/api/auth/session', + 'POST https://testcloud.comfy.org/api/settings' ]) expect([...networkPolicy.unexpected]).toEqual([ 'Mutation POST https://testcloud.comfy.org/customers', @@ -213,3 +370,62 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { test.fail(true, 'The recorded WebSocket must fail fixture teardown') }) }) + +test.describe('Live Cloud checkout network boundary', () => { + test.use({ + liveCloudBillingConfig: liveCloudBillingConfigSchema.parse({ + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', + CLOUD_ACCOUNT_EMAIL: 'routing@example.com', + CLOUD_ACCOUNT_PASSWORD: 'unused', + allowCheckout: true + }) + }) + + test('allows checkout handoff in browser and API clients', async ({ + page, + request, + context, + sandboxProxy + }) => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://localhost:5173/') + expect( + await page.evaluate(() => + fetch('/api/billing/subscribe', { method: 'POST' }).then((response) => + response.text() + ) + ) + ).toBe('POST https://testcloud.comfy.org/api/billing/subscribe') + for (const api of [request, context.request]) { + const response = await api.post( + 'https://testcloud.comfy.org/api/billing/subscribe' + ) + expect(await response.text()).toBe( + 'POST https://testcloud.comfy.org/api/billing/subscribe' + ) + } + expect(sandboxProxy.requests).toEqual([ + 'POST https://testcloud.comfy.org/api/billing/subscribe', + 'POST https://testcloud.comfy.org/api/billing/subscribe', + 'POST https://testcloud.comfy.org/api/billing/subscribe' + ]) + }) + + test('still rejects payment confirmation with checkout enabled', async ({ + context, + networkPolicy + }) => { + await expect( + context.request.post( + 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' + ) + ).rejects.toThrow('Forbidden live Cloud request') + expect([...networkPolicy.unexpected]).toEqual([ + 'Mutation POST https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' + ]) + test.fail(true, 'The recorded mutation must fail fixture teardown') + }) +}) diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md index 9b6809bbcaf..656bc995cc4 100644 --- a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -21,7 +21,9 @@ sandbox credentials. Use a dedicated sandbox account and the public billing API. Establish the connection with a sign-in and billing-read smoke test. Checkout recovery -scenarios build on this harness in a separate change. +scenarios build on this harness in a separate change. Smoke remains read-only for +billing state; authentication, customer provisioning, installation-version, and +onboarding settings writes are permitted for startup. A separate JSON configuration and an external reset executable add unnecessary setup. Embedding database writes and Stripe/Temporal administration in the fixture diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index c28f4c81eac..3d247b86229 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -14,29 +14,73 @@ command above, or inject them and run the test in one command: op run --env-file -- pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 --headed ``` -| Variable | Value | -| -------------------------- | ---------------------------------------------------- | -| `PLAYWRIGHT_TEST_URL` | Local frontend URL or deployed sandbox origin | -| `PLAYWRIGHT_SETUP_API_URL` | Matching test, staging, or PR sandbox backend origin | -| `CLOUD_ACCOUNT_EMAIL` | Sandbox account email | -| `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | +| Variable | Value | +| -------------------------- | ------------------------------------------------------------------ | +| `PLAYWRIGHT_TEST_URL` | URL serving the PR frontend, usually `http://localhost:5173` | +| `PLAYWRIGHT_SETUP_API_URL` | Cloud backend origin; defaults to `https://stagingcloud.comfy.org` | +| `CLOUD_ACCOUNT_EMAIL` | Dedicated account email for the selected environment | +| `CLOUD_ACCOUNT_PASSWORD` | Account password for the selected environment | -For a local frontend, start `pnpm dev:cloud` and use its Vite URL with -`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The test routes browser -API requests directly to the selected sandbox, independently of the local Vite proxy. Production targets are rejected. +Start the PR checkout's frontend with the matching existing dev script. Set the +backend variable in the terminal running Playwright: -The fixture disables `onboarding_survey_enabled` through the existing dev-only -feature-flag helper before navigation. Deployed builds ignore this override, so -accounts used against a deployed frontend must have completed the survey. +| Environment | Frontend command | `PLAYWRIGHT_SETUP_API_URL` | +| ----------------- | ------------------------------------------ | -------------------------------- | +| Staging (default) | `pnpm dev:cloud:staging` | `https://stagingcloud.comfy.org` | +| Test Cloud | `pnpm dev:cloud:test` | `https://testcloud.comfy.org` | +| Production | `USE_PROD_CONFIG=true pnpm dev:cloud:prod` | `https://cloud.comfy.org` | + +For example, after starting `pnpm dev:cloud:staging` and injecting credentials: + +```sh +DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test --project=cloud-live --workers=1 +``` + +Restart the frontend when switching environments and use that environment's +account credentials. The fixture forwards frontend API requests to the selected +backend and derives its matching customer API and Stripe checkout mode. A deployed +frontend must match the selected backend origin; local URLs exercise the PR code. + +Live tests do not install shared response mocks or override feature flags. +Permanent accounts must have completed onboarding when the survey flag is +enabled. Disposable setup completes the real survey and waits for the canvas. Known tutorial dialogs are dismissed through their Skip button whenever +they appear during an interaction, including after navigation. Billing dialogs +are left to the scenario. Assets, +location detection, Google auth scripts, analytics, and feature flags use their real +services. Accounts must have the hosted checkout flow enabled in the selected +environment. The smoke test signs in, validates the browser's real billing-status response, waits for app readiness, and checks that the graph canvas is visible. -It attaches a post-login screenshot. The shared network boundary rejects mutations except the specific +It attaches a post-login screenshot. The shared network boundary rejects Cloud and Stripe mutations except the specific POST endpoints used for Firebase sign-in/token refresh, customer provisioning, -and Cloud auth tokens/session cookies. A rejected mutation fails the test even -if the app catches the request error. Checkout, payment, and reset mutations -are blocked in both browser routing and Playwright API clients. +Cloud auth tokens/session cookies, the startup write to +`/api/settings/Comfy.InstalledVersion`, and onboarding survey/tutorial state. +Bulk settings writes may contain only `onboarding_survey`. Observed analytics, +customer-messaging, and feature-flag POST endpoints have exact URL allowances; +unknown off-origin mutations are blocked and fail teardown. +A rejected mutation fails the test even +if the app catches the request error. The smoke fixture blocks checkout, payment, and reset mutations in both browser +routing and Playwright API clients. + +## Shared setup + +Use dedicated accounts for the selected environment. PRs #17870, #17538, #17543, +#17548, and #17545 each depend directly on this setup PR, #17481. + +Shared support includes authenticated API reads/writes, disposable sandbox +accounts, hosted checkout interaction, and paid-subscription provisioning used +by checkout and lifecycle tests. Shared helpers do not add scenario specs here. +Unpaid recovery, top-up assertions, decline recovery, lifecycle transitions, and +3DS outcomes stay in their own PRs. + +The smoke fixture denies billing mutations. Scenario fixtures explicitly opt in +to the permissions they need. Payment and disposable-account tests reject +production; Staging remains the default backend. Select `cloud-live-paid` for +saved-card tests and `cloud-live-disposable` for new-account tests. These projects +collect tests only when their respective scenario PRs are present. -Use a dedicated sandbox account. Checkout recovery tests in #17481 additionally -require a personal workspace with no paid plan or saved card. Those tests own -the account preflight and direct API assertions. +Automatic tracing stays disabled because authenticated traffic contains reusable +credentials. Use screenshots for evidence. +Completed-payment assertions verify saved cards through the real hosted billing +portal, because sandbox environments may disable the embedded-checkout saved-card API. diff --git a/playwright.config.ts b/playwright.config.ts index 926bd95a22b..e07f999022d 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -72,6 +72,38 @@ export default defineConfig({ { name: 'cloud-live', testMatch: '**/tests/liveCloud/**/*.spec.ts', + testIgnore: ['**/liveCloud/disposable/**', '**/liveCloud/paid/**'], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + } satisfies NonNullable[number], + { + name: 'cloud-live-disposable', + testMatch: '**/tests/liveCloud/disposable/**/*.spec.ts', + testIgnore: [], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + } satisfies NonNullable[number], + { + name: 'cloud-live-paid', + testMatch: '**/tests/liveCloud/paid/**/*.spec.ts', testIgnore: [], fullyParallel: false, retries: 0,