From 4768e541ac926511e5c33dad3de1a6678fe3ae1a Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:04:16 -0700 Subject: [PATCH 1/9] test: configure real Cloud billing setup across environments --- .../fixtures/liveCloudBillingFixture.ts | 4 +- .../fixtures/networkIsolationFixture.ts | 147 ++++++++++-------- .../utils/liveCloudBillingConfig.test.ts | 15 +- .../fixtures/utils/liveCloudBillingConfig.ts | 9 +- .../fixtures/utils/liveCloudBillingContext.ts | 11 +- .../utils/liveCloudBillingPolicy.test.ts | 10 ++ .../fixtures/utils/liveCloudBillingPolicy.ts | 18 ++- browser_tests/playwrightSelection.test.ts | 139 ++++++++--------- .../liveCloudBillingRouting.spec.ts | 33 +++- docs/testing/cloud-billing-e2e.md | 61 +++++--- 10 files changed, 277 insertions(+), 170 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 86249b68f91..f1bb2dc5585 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -10,8 +10,8 @@ export const liveCloudBillingFixture = base.extend<{ liveCloudBillingConfig: async ({ liveCloudBillingConfig }, use) => { await use(liveCloudBillingConfig ?? loadLiveCloudBillingConfig()) }, - comfyPage: async ({ page, request }, use) => { - await signInToLiveCloud(page) + comfyPage: async ({ page, request, liveCloudBillingConfig }, use) => { + await signInToLiveCloud(page, liveCloudBillingConfig) await use(new ComfyPage(page, request)) } }) diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index 965f19650bc..bc5cc6303cb 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -1,4 +1,4 @@ -import type { APIRequestContext } from '@playwright/test' +import type { APIRequestContext, BrowserContext } from '@playwright/test' import { expect, test as base } from '@playwright/test' import { config as dotenvConfig } from 'dotenv' @@ -55,6 +55,81 @@ function guardApiRequests( } } +export async function installContextNetworkIsolation( + context: BrowserContext, + networkPolicy: NetworkPolicy, + baseURL?: string, + liveCloudBillingConfig?: LiveCloudBillingConfig +) { + const { origins, unexpected } = networkPolicy + const restore = guardApiRequests( + context.request, + origins, + unexpected, + baseURL, + liveCloudBillingConfig + ) + await context.route('**/*', async (route) => { + const url = new URL(route.request().url()) + if (origins.has(url.origin) || liveCloudBillingConfig) { + await route.continue() + return + } + unexpected.add(getBlockedRequestViolation(url, route.request().method())) + await route.abort('blockedbyclient') + }) + await context.routeWebSocket( + (url) => !origins.has(url.origin.replace(/^ws/, 'http')), + async (socket) => { + const url = new URL(socket.url()) + unexpected.add(`WebSocket ${url.origin}${url.pathname}`) + await socket.close() + } + ) + if (liveCloudBillingConfig) { + await installLiveCloudBillingRouting( + context, + networkPolicy, + liveCloudBillingConfig + ) + return restore + } + await context.route('https://huggingface.co/**/resolve/**', (route) => + route.request().method() === 'HEAD' + ? route.fulfill({ status: 200, body: '' }) + : route.fallback() + ) + await context.route( + 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', + (route) => + route.fulfill({ contentType: 'application/javascript', body: '' }) + ) + await context.route('https://apis.google.com/js/api.js**', (route) => + route.fulfill({ status: 503, body: '' }) + ) + await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => + route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) + ) + for (const slide of HERO_SLIDES) { + await context.route(slide.poster, (route) => + route.fulfill({ path: assetPath('image32x32.webp') }) + ) + await context.route(slide.src, (route) => + route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) + ) + } + await context.route( + 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', + (route) => route.fulfill({ status: 404, body: '' }) + ) + await context.route( + 'https://{api,stagingapi}.comfy.org/nodes{,?*}', + (route) => route.fulfill({ status: 404, body: '' }) + ) + + return restore +} + export const networkIsolationFixture = base.extend<{ networkPolicy: NetworkPolicy liveCloudBillingConfig: LiveCloudBillingConfig | undefined @@ -72,7 +147,10 @@ export const networkIsolationFixture = base.extend<{ liveCloudBillingConfig.customerOrigin, 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', - 'https://dreamboothy-dev.firebaseapp.com' + liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL === + 'https://cloud.comfy.org' + ? 'https://dreamboothy.firebaseapp.com' + : 'https://dreamboothy-dev.firebaseapp.com' ]) : new Set( [ @@ -123,71 +201,12 @@ export const networkIsolationFixture = base.extend<{ { context, baseURL, networkPolicy, liveCloudBillingConfig }, use ) => { - const { origins, unexpected } = networkPolicy - const restore = guardApiRequests( - context.request, - origins, - unexpected, + const restore = await installContextNetworkIsolation( + context, + networkPolicy, baseURL, liveCloudBillingConfig ) - await context.route('**/*', async (route) => { - const url = new URL(route.request().url()) - if (origins.has(url.origin)) { - await route.continue() - return - } - unexpected.add(getBlockedRequestViolation(url, route.request().method())) - await route.abort('blockedbyclient') - }) - await context.routeWebSocket( - (url) => !origins.has(url.origin.replace(/^ws/, 'http')), - async (socket) => { - const url = new URL(socket.url()) - unexpected.add(`WebSocket ${url.origin}${url.pathname}`) - await socket.close() - } - ) - await context.route('https://huggingface.co/**/resolve/**', (route) => - route.request().method() === 'HEAD' - ? route.fulfill({ status: 200, body: '' }) - : route.fallback() - ) - await context.route( - 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', - (route) => - route.fulfill({ contentType: 'application/javascript', body: '' }) - ) - await context.route('https://apis.google.com/js/api.js**', (route) => - route.fulfill({ status: 503, body: '' }) - ) - await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => - route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) - ) - for (const slide of HERO_SLIDES) { - await context.route(slide.poster, (route) => - route.fulfill({ path: assetPath('image32x32.webp') }) - ) - await context.route(slide.src, (route) => - route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) - ) - } - await context.route( - 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', - (route) => route.fulfill({ status: 404, body: '' }) - ) - await context.route( - 'https://{api,stagingapi}.comfy.org/nodes{,?*}', - (route) => route.fulfill({ status: 404, body: '' }) - ) - - if (liveCloudBillingConfig) { - await installLiveCloudBillingRouting( - context, - networkPolicy, - liveCloudBillingConfig - ) - } await use(context) restore() } diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index f4f360dc08c..d0cfbfb89ab 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -14,7 +14,6 @@ const sandboxConfig = { describe('Live Cloud billing prerequisites', () => { it.for([ - 'https://cloud.comfy.org', 'https://testcloud.comfy.org.example.com', 'https://testcloud.comfy.org/redirect', 'http://testcloud.comfy.org', @@ -37,10 +36,16 @@ describe('Live Cloud billing prerequisites', () => { ).toBe(false) }) - it('accepts a local frontend backed by test Cloud', () => { - expect(liveCloudBillingConfigSchema.safeParse(sandboxConfig).success).toBe( - true - ) + it.for([ + ['https://stagingcloud.comfy.org', 'https://stagingapi.comfy.org'], + ['https://testcloud.comfy.org', 'https://testapi.comfy.org'], + ['https://cloud.comfy.org', 'https://api.comfy.org'] + ])('selects the matching customer API for %s', ([backend, customer]) => { + const selected = liveCloudBillingConfigSchema.parse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL: backend + }) + expect(selected.customerOrigin).toBe(customer) }) it('accepts an isolated preview deployment', () => { diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 12944db76d2..ff2f4f5b15c 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -11,6 +11,7 @@ const originURL = z .transform((value) => new URL(value).origin) function getLiveCloudCustomerOrigin(cloudOrigin: string): string | undefined { + if (cloudOrigin === 'https://cloud.comfy.org') return 'https://api.comfy.org' if (cloudOrigin === 'https://testcloud.comfy.org') return 'https://testapi.comfy.org' if (cloudOrigin === 'https://stagingcloud.comfy.org') @@ -23,7 +24,9 @@ function getLiveCloudCustomerOrigin(cloudOrigin: string): string | undefined { export const liveCloudBillingConfigSchema = z .object({ PLAYWRIGHT_TEST_URL: originURL, - PLAYWRIGHT_SETUP_API_URL: originURL, + PLAYWRIGHT_SETUP_API_URL: originURL.default( + 'https://stagingcloud.comfy.org' + ), CLOUD_ACCOUNT_EMAIL: z.string().email(), CLOUD_ACCOUNT_PASSWORD: z.string().min(1) }) @@ -40,7 +43,7 @@ export const liveCloudBillingConfigSchema = z ctx.addIssue({ code: 'custom', path: ['PLAYWRIGHT_TEST_URL'], - message: 'Use localhost or the selected sandbox origin' + message: 'Use localhost or the selected Cloud origin' }) } }) @@ -52,7 +55,7 @@ export const liveCloudBillingConfigSchema = z ctx.addIssue({ code: 'custom', path: ['PLAYWRIGHT_SETUP_API_URL'], - message: 'Use a Cloud test, staging, or PR preview origin' + message: 'Use a Cloud production, staging, test, or PR preview origin' }) return z.NEVER } diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index 2f1662865c4..f245e4338e8 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -6,6 +6,7 @@ import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillin import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { + LIVE_CHECKOUT_ORIGINS, getLiveCloudDestinationViolation, isLiveCloudMutationAllowed } from '@e2e/fixtures/utils/liveCloudBillingPolicy' @@ -35,7 +36,8 @@ export async function installLiveCloudBillingRouting( ? new URL(url.pathname + url.search, config.PLAYWRIGHT_SETUP_API_URL) : url if ( - networkPolicy.origins.has(url.origin) && + (networkPolicy.origins.has(url.origin) || + LIVE_CHECKOUT_ORIGINS.includes(url.origin)) && !isLiveCloudMutationAllowed(targetUrl, request.method(), config) ) { networkPolicy.unexpected.add( @@ -52,6 +54,7 @@ export async function installLiveCloudBillingRouting( }) await route.fulfill({ response }) } catch { + if (request.frame().page().isClosed()) return throw new Error(`Cloud proxy request failed: ${url.pathname}`) } return @@ -60,8 +63,10 @@ export async function installLiveCloudBillingRouting( }) } -export async function signInToLiveCloud(page: Page) { - const config = loadLiveCloudBillingConfig() +export async function signInToLiveCloud( + page: Page, + config = loadLiveCloudBillingConfig() +) { await new FeatureFlagHelper(page).seedFlags({ onboarding_survey_enabled: false }) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index 71c5cdbe9d9..1cb4e3e486e 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -23,6 +23,12 @@ describe('live Cloud mutation policy', () => { { method: 'POST', path: '/customers', allowed: false }, { method: 'POST', path: '/api/auth/token', allowed: true }, { method: 'POST', path: '/api/auth/session', allowed: true }, + { + method: 'POST', + path: '/api/settings/Comfy.InstalledVersion', + allowed: true + }, + { method: 'POST', path: '/api/settings/other', allowed: false }, { method: 'POST', path: '/api/billing/subscribe', allowed: false }, { method: 'POST', path: '/api/billing/payment-portal', allowed: false }, { method: 'POST', path: '/customers/credit', allowed: false }, @@ -80,6 +86,10 @@ describe('live Cloud mutation origins', () => { }) it.for([ + { + cloud: 'https://cloud.comfy.org', + customer: 'https://api.comfy.org' + }, { cloud: 'https://testcloud.comfy.org', customer: 'https://testapi.comfy.org' diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index db2327fa970..4b83799f2b3 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -17,6 +17,18 @@ export function getLiveCloudDestinationViolation( } } +export const LIVE_CHECKOUT_ORIGINS = [ + 'https://checkout.stripe.com', + 'https://checkout.comfy.org', + 'https://api.stripe.com', + 'https://js.stripe.com', + 'https://m.stripe.network', + 'https://m.stripe.com', + 'https://r.stripe.com', + 'https://q.stripe.com', + 'https://b.stripecdn.com' +] + const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] export function getBlockedRequestViolation(url: URL, method: string): string { @@ -55,6 +67,10 @@ export function isLiveCloudMutationAllowed( } return ( url.origin === config.PLAYWRIGHT_SETUP_API_URL && - ['/api/auth/token', '/api/auth/session'].includes(url.pathname) + [ + '/api/auth/token', + '/api/auth/session', + '/api/settings/Comfy.InstalledVersion' + ].includes(url.pathname) ) } diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 2a8a6ef88fd..28b5eed787e 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -3,82 +3,81 @@ import { spawnSync } from 'node:child_process' import { describe, expect, it } from 'vitest' import { z } from 'zod' -describe('Live billing opt-in', () => { - it.for([ - { - name: 'disabled', - enabled: '0', - args: ['tests/liveCloud'], - status: 1, - count: 0, - hooks: true - }, +function collectLiveBilling(enabled: '0' | '1', args: string[]) { + const result = spawnSync( + 'pnpm', + ['exec', 'playwright', 'test', ...args, '--list', '--reporter=json'], { - name: 'enabled without a project filter', - enabled: '1', - args: [], - status: 0, - count: 1, - hooks: false - }, - { - name: 'enabled with a project filter', - enabled: '1', - args: ['--project=cloud-live'], - status: 0, - count: 1, - hooks: false + cwd: process.cwd(), + env: { + ...process.env, + PLAYWRIGHT_CLOUD_LIVE: enabled, + DISTRIBUTION: 'cloud' + }, + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 10_000_000 } + ) + expect(result.error).toBeUndefined() + const report = z + .object({ + suites: z.array( + z.object({ + suites: z.array( + z.object({ specs: z.array(z.object({ file: z.string() })) }) + ) + }) + ), + config: z.object({ + globalSetup: z.unknown(), + globalTeardown: z.unknown(), + projects: z.array(z.object({ name: z.string() })) + }) + }) + .parse(JSON.parse(result.stdout)) + const files = report.suites.flatMap((suite) => + suite.suites.flatMap((child) => + child.specs.map((spec) => spec.file.replaceAll('\\', '/')) + ) + ) + return { status: result.status, config: report.config, files } +} + +describe('Live billing opt-in', () => { + it('excludes live billing when disabled', { timeout: 90_000 }, () => { + const { status, config, files } = collectLiveBilling('0', [ + 'tests/liveCloud' + ]) + expect(status).toBe(1) + expect(config.projects.map((project) => project.name)).not.toContain( + 'cloud-live' + ) + expect(files).toEqual([]) + expect(config.globalSetup).toBeTruthy() + expect(config.globalTeardown).toBeTruthy() + }) + + it.for([ + { name: 'without a project filter', args: [] }, + { name: 'with a project filter', args: ['--project=cloud-live'] } ])( - 'collects live billing only when explicitly enabled: $name', + 'collects only live billing when enabled $name', { timeout: 90_000 }, - ({ enabled, args, status, count, hooks }) => { - const result = spawnSync( - 'pnpm', - ['exec', 'playwright', 'test', ...args, '--list', '--reporter=json'], - { - cwd: process.cwd(), - env: { - ...process.env, - PLAYWRIGHT_CLOUD_LIVE: enabled, - DISTRIBUTION: 'cloud' - }, - encoding: 'utf8', - timeout: 60_000, - maxBuffer: 10_000_000 - } + ({ args }) => { + const { status, config, files } = collectLiveBilling('1', args) + expect(status).toBe(0) + expect(config.projects.map((project) => project.name)).toEqual([ + 'cloud-live' + ]) + expect(files.every((file) => file.startsWith('tests/liveCloud/'))).toBe( + true ) - expect(result.error).toBeUndefined() - expect(result.status).toBe(status) - const report = z - .object({ - suites: z.array(z.unknown()), - config: z.object({ - globalSetup: z.unknown(), - globalTeardown: z.unknown(), - projects: z.array(z.object({ name: z.string() })) - }) - }) - .parse(JSON.parse(result.stdout)) - expect( - report.config.projects.some((project) => project.name === 'cloud-live') - ).toBe(enabled === '1') - expect( - report.config.projects.every((project) => project.name === 'cloud-live') - ).toBe(!hooks) - const suites = z - .array( - z.object({ - suites: z.array(z.object({ specs: z.array(z.unknown()) })) - }) - ) - .parse(report.suites) - const specs = suites.flatMap((suite) => - suite.suites.flatMap((child) => child.specs) + expect(files).toEqual( + expect.arrayContaining(['tests/liveCloud/billingSmoke.spec.ts']) ) - expect(specs).toHaveLength(count) - expect(Boolean(report.config.globalSetup)).toBe(hooks) - expect(Boolean(report.config.globalTeardown)).toBe(hooks) + expect(config.globalSetup).toBeFalsy() + expect(config.globalTeardown).toBeFalsy() } ) }) diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index 39bf61729b0..fb60695b4a2 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -122,7 +122,36 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ]) }) - for (const path of ['/api/billing/subscribe', '/customers']) { + test('uses upstream responses for assets, location, Google auth, and flags', async ({ + context, + sandboxProxy + }) => { + const page = await context.newPage() + await page.route('http://localhost:5173/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://localhost:5173/') + for (const url of [ + 'https://cloud.comfy.org/cdn-cgi/trace', + 'https://apis.google.com/js/api.js', + 'https://huggingface.co/example/resolve/main/model.safetensors', + 'https://t.comfy.org/flags/' + ]) { + expect( + await page.evaluate(async (url) => { + const response = await fetch(url) + return await response.text() + }, url) + ).toBe(`GET ${url}`) + } + expect(sandboxProxy.requests).toContain('GET https://t.comfy.org/flags/') + }) + + for (const path of [ + '/api/billing/subscribe', + '/customers', + 'https://api.stripe.com/v1/payment_pages/cs_test_example/init' + ]) { test(`blocks POST ${path} and fails even when the app catches it`, async ({ page, networkPolicy @@ -142,7 +171,7 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ) ).toBe('blocked') expect([...networkPolicy.unexpected]).toEqual([ - `Mutation POST http://localhost:5173${path}` + `Mutation POST ${new URL(path, 'http://localhost:5173').href}` ]) test.fail(true, 'The recorded mutation must fail fixture teardown') }) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index c28f4c81eac..cab05a50c07 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -14,29 +14,50 @@ command above, or inject them and run the test in one command: op run --env-file -- pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 --headed ``` -| Variable | Value | -| -------------------------- | ---------------------------------------------------- | -| `PLAYWRIGHT_TEST_URL` | Local frontend URL or deployed sandbox origin | -| `PLAYWRIGHT_SETUP_API_URL` | Matching test, staging, or PR sandbox backend origin | -| `CLOUD_ACCOUNT_EMAIL` | Sandbox account email | -| `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | +| Variable | Value | +| -------------------------- | ------------------------------------------------------------------ | +| `PLAYWRIGHT_TEST_URL` | URL serving the PR frontend, usually `http://localhost:5173` | +| `PLAYWRIGHT_SETUP_API_URL` | Cloud backend origin; defaults to `https://stagingcloud.comfy.org` | +| `CLOUD_ACCOUNT_EMAIL` | Dedicated account email for the selected environment | +| `CLOUD_ACCOUNT_PASSWORD` | Account password for the selected environment | -For a local frontend, start `pnpm dev:cloud` and use its Vite URL with -`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The test routes browser -API requests directly to the selected sandbox, independently of the local Vite proxy. Production targets are rejected. +Start the PR checkout's frontend with the matching existing dev script. Set the +backend variable in the terminal running Playwright: -The fixture disables `onboarding_survey_enabled` through the existing dev-only -feature-flag helper before navigation. Deployed builds ignore this override, so -accounts used against a deployed frontend must have completed the survey. +| Environment | Frontend command | `PLAYWRIGHT_SETUP_API_URL` | +| ----------------- | ------------------------------------------ | -------------------------------- | +| Staging (default) | `pnpm dev:cloud:staging` | `https://stagingcloud.comfy.org` | +| Test Cloud | `pnpm dev:cloud:test` | `https://testcloud.comfy.org` | +| Production | `USE_PROD_CONFIG=true pnpm dev:cloud:prod` | `https://cloud.comfy.org` | + +For example, after starting `pnpm dev:cloud:staging` and injecting credentials: + +```sh +DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test --project=cloud-live --workers=1 +``` + +Restart the frontend when switching environments and use that environment's +account credentials. The fixture forwards frontend API requests to the selected +backend and derives its matching customer API and Stripe checkout mode. A deployed +frontend must match the selected backend origin; local URLs exercise the PR code. + +Live tests do not install shared response mocks. They override only +`onboarding_survey_enabled=false` in the local PR frontend to skip the unrelated +survey. Deployed builds ignore this dev-only override and require completed +onboarding. Assets, +location detection, Google auth scripts, analytics, and feature flags use their real +services. Accounts must have the hosted checkout flow enabled in the selected +environment. The smoke test signs in, validates the browser's real billing-status response, waits for app readiness, and checks that the graph canvas is visible. -It attaches a post-login screenshot. The shared network boundary rejects mutations except the specific +It attaches a post-login screenshot. The shared network boundary rejects Cloud and Stripe mutations except the specific POST endpoints used for Firebase sign-in/token refresh, customer provisioning, -and Cloud auth tokens/session cookies. A rejected mutation fails the test even -if the app catches the request error. Checkout, payment, and reset mutations -are blocked in both browser routing and Playwright API clients. - -Use a dedicated sandbox account. Checkout recovery tests in #17481 additionally -require a personal workspace with no paid plan or saved card. Those tests own -the account preflight and direct API assertions. +Cloud auth tokens/session cookies, and the startup write to +`/api/settings/Comfy.InstalledVersion`. A rejected mutation fails the test even +if the app catches the request error. The smoke fixture blocks checkout, payment, and reset mutations in both browser +routing and Playwright API clients. + +Use a dedicated account. This setup checks authentication, billing reads, and +canvas readiness. It does not submit checkout, payment, or reset requests. +Checkout scenarios and account requirements belong to follow-up PRs. From 2ebbbbc6926f4d147d20449d7cb15b816c2d9d31 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:44:30 -0700 Subject: [PATCH 2/9] test: share live Cloud setup across independent billing scenarios --- .../helpers/LiveCloudBillingSession.ts | 95 ++++++++ .../fixtures/helpers/LiveCloudCheckout.ts | 229 ++++++++++++++++++ .../helpers/LiveCloudDisposableAccount.ts | 33 +++ .../fixtures/liveCloudBillingFixture.ts | 19 +- .../liveCloudDisposableBillingFixture.ts | 71 ++++++ .../fixtures/networkIsolationFixture.ts | 17 +- .../utils/liveCloudBillingConfig.test.ts | 27 +++ .../fixtures/utils/liveCloudBillingConfig.ts | 42 +++- .../fixtures/utils/liveCloudBillingContext.ts | 10 + .../utils/liveCloudBillingPolicy.test.ts | 140 ++++++++++- .../fixtures/utils/liveCloudBillingPolicy.ts | 59 ++++- browser_tests/playwrightSelection.test.ts | 21 +- .../liveCloudBillingRouting.spec.ts | 68 +++++- docs/testing/cloud-billing-e2e.md | 23 +- playwright.config.ts | 32 +++ 15 files changed, 863 insertions(+), 23 deletions(-) create mode 100644 browser_tests/fixtures/helpers/LiveCloudBillingSession.ts create mode 100644 browser_tests/fixtures/helpers/LiveCloudCheckout.ts create mode 100644 browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts create mode 100644 browser_tests/fixtures/liveCloudDisposableBillingFixture.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts new file mode 100644 index 00000000000..77fc470aa59 --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts @@ -0,0 +1,95 @@ +import { + zBillingStatusResponse, + zCurrentWorkspaceResponse, + zListSavedPaymentMethodsResponse, + zPaymentPortalResponse +} from '@comfyorg/ingest-types/zod' +import type { APIRequestContext, TestInfo } from '@playwright/test' +import { expect } from '@playwright/test' +import type { z } from 'zod' + +export class LiveCloudBillingSession { + constructor( + private readonly request: APIRequestContext, + private readonly backend: string, + private readonly headers: Record + ) {} + + async read(path: string, schema: z.ZodType): Promise { + const response = await this.request + .get(new URL(path, this.backend).href, { + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } + + async post( + path: string, + data: Record, + schema: z.ZodType, + acceptedStatuses = [200] + ): Promise { + const response = await this.request + .post(new URL(path, this.backend).href, { + data, + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(acceptedStatuses, `POST ${path}`).toContain(response.status()) + return schema.parse(await response.json()) + } + + async assertNoCardAccount(testInfo: TestInfo) { + const workspace = await this.read( + '/api/workspaces/current', + zCurrentWorkspaceResponse + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const status = await this.read( + '/api/billing/status', + zBillingStatusResponse + ) + if (this.backend === 'https://cloud.comfy.org') { + const paymentMethods = await this.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect( + paymentMethods, + 'Checkout recovery requires an account without saved payment methods' + ).toHaveLength(0) + } + await testInfo.attach('billing-preflight.json', { + body: JSON.stringify({ + workspaceId: workspace.id, + isActive: status.is_active, + billingStatus: status.billing_status, + subscriptionTier: status.subscription_tier, + backend: this.backend + }), + contentType: 'application/json' + }) + if (status.is_active) expect(status.subscription_tier).toBe('FREE') + else + expect(['inactive', 'awaiting_payment_method']).toContain( + status.billing_status + ) + } + + async ensureProvisioned(returnUrl: string) { + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) + } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts new file mode 100644 index 00000000000..a937f26fb0f --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts @@ -0,0 +1,229 @@ +import { + zBillingBalanceResponse, + zBillingOpStatusResponse, + zBillingStatusResponse, + zListSavedPaymentMethodsResponse, + zPreviewSubscribeResponse, + zSubscribeResponse +} from '@comfyorg/ingest-types/zod' +import { expect } from '@playwright/test' +import type { Page, Response, TestInfo } from '@playwright/test' +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' + +function matchesBillingResponse( + response: Response, + frontend: string, + path: string +) { + return response.url() === new URL(path, frontend).href +} + +export class LiveCloudCheckout { + readonly page: Page + readonly confirmation + readonly resumePayment + private readonly subscribe + + constructor( + private readonly comfyPage: ComfyPage, + private readonly frontend: string, + private readonly backend: string + ) { + this.page = comfyPage.page + this.confirmation = this.page.getByRole('dialog').filter({ + has: this.page.getByRole('heading', { + name: 'Confirm your payment', + exact: true + }) + }) + this.resumePayment = this.confirmation.getByRole('button', { + name: 'Complete your payment', + exact: true + }) + this.subscribe = this.confirmation.getByRole('button', { + name: 'Subscribe to Creator', + exact: true + }) + } + + async open(retryTransientFailures = false) { + let preview: ReturnType | undefined + const loadPreview = async () => { + const [response] = await Promise.all([ + this.page.waitForResponse((response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/preview-subscribe' + ) + ), + this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) + ]) + expect(response.status()).toBe(200) + preview = zPreviewSubscribeResponse.parse(await response.json()) + } + if (retryTransientFailures) { + await expect(loadPreview).toPass({ timeout: 90_000 }) + } else { + await loadPreview() + } + if (!preview) throw new Error('Cloud billing preview was not loaded') + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await expect( + this.confirmation.getByText('Total due today', { exact: true }) + ).toBeVisible() + await expect(this.subscribe.or(this.resumePayment)).toBeEnabled() + return preview + } + + async completeCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo, + expectedPaymentMethodCount = 1 + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4242424242424242') + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + 'checkout-completed.png', + expectedPaymentMethodCount + ) + } + + async submitCard(checkout: Page, cardNumber: string) { + await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) + await checkout.getByLabel('Expiration', { exact: true }).fill('1230') + await checkout + .getByRole('textbox', { + name: 'Credit or debit card CVC/CVV', + exact: true + }) + .fill('123') + await checkout + .getByPlaceholder('Full name on card', { exact: true }) + .fill('Comfy Billing Test') + await checkout + .getByRole('button', { name: 'Enter address manually', exact: true }) + .click() + await checkout.locator('#billingAddressLine1').fill('123 Test Street') + await checkout.getByLabel('City', { exact: true }).fill('San Francisco') + await checkout.getByLabel('State', { exact: true }).selectOption('CA') + await checkout.getByLabel('ZIP', { exact: true }).fill('94107') + await checkout.locator('#enableStripePass').uncheck() + await checkout.getByRole('button', { name: /^Save/ }).click() + } + + async verifyCheckoutCompletion( + session: LiveCloudBillingSession, + testInfo: TestInfo, + balanceBeforeCents: number, + preview: ReturnType, + operationId: string, + screenshotName: string, + expectedPaymentMethodCount = 1 + ) { + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + phase: operation.phase + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ status: 'succeeded' }) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + is_active: true, + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const expectedBalance = + balanceBeforeCents + Number(preview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedBalance) + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(expectedPaymentMethodCount) + await testInfo.attach('checkout-completion.json', { + body: JSON.stringify({ + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + balanceBeforeCents, + balanceAfterCents: expectedBalance, + paymentMethodCount: methods.length + }), + contentType: 'application/json' + }) + await this.page.goto(this.frontend) + await this.attachScreenshot(screenshotName) + return { + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + paymentMethodCount: methods.length + } + } + + async startCheckout(action = this.subscribe.or(this.resumePayment)) { + await expect(action).toBeEnabled() + const [response, checkout] = await Promise.all([ + this.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/subscribe' + ) && response.request().method() === 'POST' + ), + this.page.waitForEvent('popup'), + action.click() + ]) + expect(response.status()).toBe(200) + const subscription = zSubscribeResponse.parse(await response.json()) + expect(subscription.status).toBe('needs_payment_method') + expect(subscription.billing_op_id).not.toBe('') + const mode = this.backend === 'https://cloud.comfy.org' ? 'live' : 'test' + const testCheckoutUrl = new RegExp( + `^https://checkout\\.(?:stripe\\.com|comfy\\.org)/c/pay/cs_${mode}_` + ) + expect(subscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(checkout).toHaveURL(testCheckoutUrl) + await expect( + checkout.getByRole('textbox', { name: 'Card number', exact: true }) + ).toBeVisible() + return { checkout, subscription } + } + + async attachScreenshot(name: string) { + await this.comfyPage.attachScreenshot(name, { runInCI: true }) + } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts new file mode 100644 index 00000000000..af892f0373d --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts @@ -0,0 +1,33 @@ +import { randomUUID } from 'node:crypto' + +import type { APIRequestContext } from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +const firebaseApiKey = 'AIzaSyDa_YMeyzV0SkVe92vBZ1tVikWBmOU5KVE' + +const signUpResponseSchema = z.object({ localId: z.string().min(1) }) + +export async function createDisposableCloudAccount(request: APIRequestContext) { + const runId = randomUUID() + const credentials = { + email: `billing-e2e-checkout-completion-${runId}@comfy.org`, + password: `BillingE2E-${randomUUID()}!` + } + const response = await request + .post( + `https://identitytoolkit.googleapis.com/v1/accounts:signUp?key=${firebaseApiKey}`, + { + data: { + ...credentials, + returnSecureToken: true + } + } + ) + .catch(() => { + throw new Error('Disposable Cloud account creation failed') + }) + expect(response.status(), 'Firebase account creation').toBe(200) + const account = signUpResponseSchema.parse(await response.json()) + return { ...credentials, userId: account.localId } +} diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index f1bb2dc5585..d9b81caeb04 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -1,3 +1,4 @@ +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import { ComfyPage } from '@e2e/fixtures/ComfyPage' import { networkIsolationFixture as base } from '@e2e/fixtures/networkIsolationFixture' import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' @@ -5,13 +6,27 @@ import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBilling export const liveCloudBillingFixture = base.extend<{ comfyPage: ComfyPage + billingSession: LiveCloudBillingSession }>({ baseURL: process.env.PLAYWRIGHT_TEST_URL, liveCloudBillingConfig: async ({ liveCloudBillingConfig }, use) => { await use(liveCloudBillingConfig ?? loadLiveCloudBillingConfig()) }, - comfyPage: async ({ page, request, liveCloudBillingConfig }, use) => { - await signInToLiveCloud(page, liveCloudBillingConfig) + billingSession: async ({ page, liveCloudBillingConfig }, use, testInfo) => { + const session = await signInToLiveCloud(page, liveCloudBillingConfig) + await page.context().tracing.start({ screenshots: true, snapshots: true }) + try { + await use(session) + } finally { + const path = testInfo.outputPath('billing-trace.zip') + await page.context().tracing.stop({ path }) + await testInfo.attach('trace', { path, contentType: 'application/zip' }) + } + }, + comfyPage: async ( + { page, request, billingSession: _billingSession }, + use + ) => { await use(new ComfyPage(page, request)) } }) diff --git a/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts new file mode 100644 index 00000000000..6479ca066fd --- /dev/null +++ b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts @@ -0,0 +1,71 @@ +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudCheckout' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' +import { createDisposableCloudAccount } from '@e2e/fixtures/helpers/LiveCloudDisposableAccount' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +export const liveCloudDisposableBillingFixture = + liveCloudBillingFixture.extend<{ + disposableAccount: Awaited> + disposableCheckout: { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout + } + }>({ + liveCloudBillingConfig: async ({}, use) => { + await use( + loadLiveCloudBillingConfig({ + allowAccountCreation: true, + allowPayments: true, + allowCheckout: true + }) + ) + }, + disposableAccount: async ({ context }, use) => { + await use(await createDisposableCloudAccount(context.request)) + }, + billingSession: async ( + { page, disposableAccount, liveCloudBillingConfig }, + use + ) => { + if (!liveCloudBillingConfig) + throw new Error('Live billing config required') + await use( + await signInToLiveCloud(page, { + ...liveCloudBillingConfig, + CLOUD_ACCOUNT_EMAIL: disposableAccount.email, + CLOUD_ACCOUNT_PASSWORD: disposableAccount.password + }) + ) + }, + disposableCheckout: async ( + { comfyPage, billingSession, liveCloudBillingConfig, disposableAccount }, + use, + testInfo + ) => { + if (!liveCloudBillingConfig) + throw new Error('Live billing config required') + await billingSession.assertNoCardAccount(testInfo) + await billingSession.ensureProvisioned( + liveCloudBillingConfig.PLAYWRIGHT_TEST_URL + ) + await testInfo.attach('disposable-account.json', { + body: JSON.stringify({ + email: disposableAccount.email, + userId: disposableAccount.userId + }), + contentType: 'application/json' + }) + await use({ + billingSession, + checkout: new LiveCloudCheckout( + comfyPage, + liveCloudBillingConfig.PLAYWRIGHT_TEST_URL, + liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL + ) + }) + } + }) + +export { expect } from '@playwright/test' diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index bc5cc6303cb..71ce840d662 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -6,6 +6,7 @@ import { HERO_SLIDES } from '@/platform/cloud/onboarding/constants/heroSlides' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { installLiveCloudBillingRouting } from '@e2e/fixtures/utils/liveCloudBillingContext' import { + LIVE_CHECKOUT_ORIGINS, getBlockedRequestViolation, isLiveCloudMutationAllowed, isReportedViolation @@ -55,7 +56,7 @@ function guardApiRequests( } } -export async function installContextNetworkIsolation( +async function installContextNetworkIsolation( context: BrowserContext, networkPolicy: NetworkPolicy, baseURL?: string, @@ -150,7 +151,19 @@ export const networkIsolationFixture = base.extend<{ liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL === 'https://cloud.comfy.org' ? 'https://dreamboothy.firebaseapp.com' - : 'https://dreamboothy-dev.firebaseapp.com' + : 'https://dreamboothy-dev.firebaseapp.com', + ...(liveCloudBillingConfig.allowPayments + ? [ + 'https://pay.google.com', + liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL.replace( + 'cloud.comfy.org', + 'platform.comfy.org' + ) + ] + : []), + ...(liveCloudBillingConfig.allowCheckout + ? LIVE_CHECKOUT_ORIGINS + : []) ]) : new Set( [ diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index d0cfbfb89ab..60bbdf99688 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -60,6 +60,7 @@ describe('Live Cloud billing prerequisites', () => { }) it('reports missing prerequisites without leaking credentials', () => { + vi.stubEnv('PLAYWRIGHT_TEST_URL', 'http://localhost:5173') vi.stubEnv('CLOUD_ACCOUNT_PASSWORD', 'private-value') vi.stubEnv('CLOUD_ACCOUNT_EMAIL', undefined) expect(loadLiveCloudBillingConfig).toThrow('CLOUD_ACCOUNT_EMAIL') @@ -92,3 +93,29 @@ describe('Frontend origins', () => { expect(config.PLAYWRIGHT_SETUP_API_URL).toBe('https://testcloud.comfy.org') }) }) + +describe('Disposable billing account boundaries', () => { + it.for(['allowPayments', 'allowAccountCreation'])( + 'rejects %s in production', + (permission) => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + [permission]: true + }).success + ).toBe(false) + } + ) + + it('requires permanent credentials unless creating a sandbox account', () => { + const config = { PLAYWRIGHT_TEST_URL: sandboxConfig.PLAYWRIGHT_TEST_URL } + expect(liveCloudBillingConfigSchema.safeParse(config).success).toBe(false) + expect( + liveCloudBillingConfigSchema.safeParse({ + ...config, + allowAccountCreation: true + }).success + ).toBe(true) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index ff2f4f5b15c..65f51e2eb62 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -27,10 +27,35 @@ export const liveCloudBillingConfigSchema = z PLAYWRIGHT_SETUP_API_URL: originURL.default( 'https://stagingcloud.comfy.org' ), - CLOUD_ACCOUNT_EMAIL: z.string().email(), - CLOUD_ACCOUNT_PASSWORD: z.string().min(1) + CLOUD_ACCOUNT_EMAIL: z.string().email().optional(), + CLOUD_ACCOUNT_PASSWORD: z.string().min(1).optional(), + allowCheckout: z.boolean().default(false), + allowPayments: z.boolean().default(false), + allowAccountCreation: z.boolean().default(false) }) .superRefine((config, ctx) => { + if ( + (config.allowPayments || config.allowAccountCreation) && + config.PLAYWRIGHT_SETUP_API_URL === 'https://cloud.comfy.org' + ) + ctx.addIssue({ + code: 'custom', + path: ['PLAYWRIGHT_SETUP_API_URL'], + message: 'Payment tests require a sandbox Cloud backend' + }) + if (!config.allowAccountCreation) { + for (const field of [ + 'CLOUD_ACCOUNT_EMAIL', + 'CLOUD_ACCOUNT_PASSWORD' + ] as const) { + if (!config[field]) + ctx.addIssue({ + code: 'custom', + path: [field], + message: 'Required for permanent-account tests' + }) + } + } if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return const frontend = new URL(config.PLAYWRIGHT_TEST_URL) const local = @@ -66,8 +91,17 @@ export type LiveCloudBillingConfig = z.infer< typeof liveCloudBillingConfigSchema > -export function loadLiveCloudBillingConfig() { - const result = liveCloudBillingConfigSchema.safeParse(process.env) +export function loadLiveCloudBillingConfig( + options: { + allowPayments?: boolean + allowAccountCreation?: boolean + allowCheckout?: boolean + } = {} +) { + const result = liveCloudBillingConfigSchema.safeParse({ + ...process.env, + ...options + }) if (!result.success) { throw new Error( `Cloud billing prerequisites: ${result.error.issues diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index f245e4338e8..5fb1911b7dc 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -2,6 +2,7 @@ import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' import type { BrowserContext, Page } from '@playwright/test' import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' @@ -67,6 +68,8 @@ export async function signInToLiveCloud( page: Page, config = loadLiveCloudBillingConfig() ) { + if (!config.CLOUD_ACCOUNT_EMAIL || !config.CLOUD_ACCOUNT_PASSWORD) + throw new Error('Live Cloud sign-in requires account credentials') await new FeatureFlagHelper(page).seedFlags({ onboarding_survey_enabled: false }) @@ -89,4 +92,11 @@ export async function signInToLiveCloud( page.getByRole('button', { name: 'Sign in', exact: true }).click() ]) zBillingStatusResponse.parse(await response.json()) + const authorization = await response.request().headerValue('authorization') + if (!authorization) throw new Error('Missing billing authorization') + return new LiveCloudBillingSession( + page.request, + config.PLAYWRIGHT_SETUP_API_URL, + { authorization } + ) } diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index 1cb4e3e486e..76303958355 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -12,7 +12,10 @@ import { const config = { PLAYWRIGHT_TEST_URL: 'http://localhost:5173', PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', - customerOrigin: 'https://testapi.comfy.org' + customerOrigin: 'https://testapi.comfy.org', + allowCheckout: false, + allowPayments: false, + allowAccountCreation: false } describe('live Cloud mutation policy', () => { @@ -69,6 +72,112 @@ describe('live Cloud mutation policy', () => { }) }) +describe('live Cloud checkout mutation policy', () => { + it('blocks checkout initialization when checkout is disabled', () => { + expect( + isLiveCloudMutationAllowed( + new URL('https://api.stripe.com/v1/payment_pages/cs_test_example/init'), + 'POST', + config + ) + ).toBe(false) + }) + + it.for([ + { + url: 'https://testcloud.comfy.org/api/billing/preview-subscribe', + method: 'POST', + allowed: true + }, + { + url: 'https://testcloud.comfy.org/api/billing/subscribe', + method: 'POST', + allowed: true + }, + { + url: 'https://stagingcloud.comfy.org/api/billing/subscribe', + method: 'POST', + allowed: false + }, + { + url: 'https://testcloud.comfy.org/api/billing/payment-portal', + method: 'POST', + allowed: false + }, + { + url: 'https://testcloud.comfy.org/api/billing/subscribe', + method: 'DELETE', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'POST', + allowed: true + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_live_example/init', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_methods', + method: 'POST', + allowed: false + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'PUT', + allowed: false + }, + { + url: 'https://checkout.stripe.com/v1/payment_pages/cs_test_example/init', + method: 'POST', + allowed: false + }, + { url: 'https://r.stripe.com/b', method: 'POST', allowed: true }, + { url: 'https://r.stripe.com/0', method: 'POST', allowed: true }, + { url: 'https://r.stripe.com/other', method: 'POST', allowed: false }, + { url: 'https://m.stripe.com/6', method: 'POST', allowed: true } + ])( + '$method $url allowed=$allowed with checkout enabled', + ({ url, method, allowed }) => { + expect( + isLiveCloudMutationAllowed(new URL(url), method, { + ...config, + allowCheckout: true + }) + ).toBe(allowed) + } + ) +}) + +describe('production checkout', () => { + it.for([ + { mode: 'live', action: 'init', allowed: true }, + { mode: 'test', action: 'init', allowed: false }, + { mode: 'live', action: 'confirm', allowed: false } + ])('$mode checkout $action allowed=$allowed', ({ mode, action, allowed }) => { + expect( + isLiveCloudMutationAllowed( + new URL( + `https://api.stripe.com/v1/payment_pages/cs_${mode}_example/${action}` + ), + 'POST', + { + ...config, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + allowCheckout: true + } + ) + ).toBe(allowed) + }) +}) + describe('live Cloud mutation origins', () => { it.for([ 'https://dreamboothy-dev.firebaseapp.com/customers', @@ -210,3 +319,32 @@ describe('blocked request violations', () => { } ) }) + +describe('Disposable payment permissions', () => { + it.for([ + 'https://identitytoolkit.googleapis.com/v1/accounts:signUp', + 'https://api.stripe.com/v1/payment_methods', + 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' + ])('allows %s only for opted-in sandbox tests', (url) => { + expect(isLiveCloudMutationAllowed(new URL(url), 'POST', config)).toBe(false) + const enabled = { + ...config, + allowAccountCreation: true, + allowPayments: true + } + expect(isLiveCloudMutationAllowed(new URL(url), 'POST', enabled)).toBe(true) + expect( + isLiveCloudMutationAllowed(new URL(url), 'POST', { + ...enabled, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org' + }) + ).toBe(false) + expect( + isLiveCloudMutationAllowed( + new URL(url.replace('cs_test_', 'cs_live_')), + 'POST', + { ...enabled, allowPayments: false, allowAccountCreation: false } + ) + ).toBe(false) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index 4b83799f2b3..772c26a8edb 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -26,7 +26,8 @@ export const LIVE_CHECKOUT_ORIGINS = [ 'https://m.stripe.com', 'https://r.stripe.com', 'https://q.stripe.com', - 'https://b.stripecdn.com' + 'https://b.stripecdn.com', + 'https://newassets.hcaptcha.com' ] const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] @@ -49,11 +50,40 @@ export function isLiveCloudMutationAllowed( method: string, config: Pick< LiveCloudBillingConfig, - 'PLAYWRIGHT_SETUP_API_URL' | 'customerOrigin' + | 'PLAYWRIGHT_SETUP_API_URL' + | 'customerOrigin' + | 'allowCheckout' + | 'allowPayments' + | 'allowAccountCreation' > ): boolean { if (SAFE_METHODS.includes(method)) return true if (method !== 'POST') return false + if ( + config.allowAccountCreation && + config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' && + url.origin === 'https://identitytoolkit.googleapis.com' && + url.pathname === '/v1/accounts:signUp' + ) + return true + if ( + config.allowPayments && + config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' + ) { + if ( + url.origin === config.PLAYWRIGHT_SETUP_API_URL && + url.pathname === '/api/billing/payment-portal' + ) + return true + if ( + url.origin === 'https://api.stripe.com' && + (url.pathname === '/v1/payment_methods' || + /^\/v1\/payment_pages\/cs_test_[A-Za-z0-9]+\/confirm$/.test( + url.pathname + )) + ) + return true + } if (url.origin === 'https://identitytoolkit.googleapis.com') { return ['/v1/accounts:signInWithPassword', '/v1/accounts:lookup'].includes( url.pathname @@ -62,6 +92,11 @@ export function isLiveCloudMutationAllowed( if (url.origin === 'https://securetoken.googleapis.com') { return url.pathname === '/v1/token' } + if ( + config.allowCheckout && + isCheckoutPostAllowed(url, config.PLAYWRIGHT_SETUP_API_URL) + ) + return true if (url.pathname === '/customers') { return url.origin === config.customerOrigin } @@ -74,3 +109,23 @@ export function isLiveCloudMutationAllowed( ].includes(url.pathname) ) } + +function isCheckoutPostAllowed(url: URL, backend: string): boolean { + if (url.origin === backend) { + return [ + '/api/billing/preview-subscribe', + '/api/billing/subscribe' + ].includes(url.pathname) + } + if (url.origin === 'https://api.stripe.com') { + const mode = backend === 'https://cloud.comfy.org' ? 'live' : 'test' + return new RegExp(`^/v1/payment_pages/cs_${mode}_[A-Za-z0-9]+/init$`).test( + url.pathname + ) + } + return ( + (url.origin === 'https://r.stripe.com' && + ['/b', '/0'].includes(url.pathname)) || + (url.origin === 'https://m.stripe.com' && url.pathname === '/6') + ) +} diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 28b5eed787e..d22cb620ffb 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -1,3 +1,4 @@ +import { globSync } from 'node:fs' import { spawnSync } from 'node:child_process' import { describe, expect, it } from 'vitest' @@ -60,7 +61,14 @@ describe('Live billing opt-in', () => { it.for([ { name: 'without a project filter', args: [] }, - { name: 'with a project filter', args: ['--project=cloud-live'] } + { + name: 'with a project filter', + args: [ + '--project=cloud-live', + '--project=cloud-live-disposable', + '--project=cloud-live-paid' + ] + } ])( 'collects only live billing when enabled $name', { timeout: 90_000 }, @@ -68,14 +76,17 @@ describe('Live billing opt-in', () => { const { status, config, files } = collectLiveBilling('1', args) expect(status).toBe(0) expect(config.projects.map((project) => project.name)).toEqual([ - 'cloud-live' + 'cloud-live', + 'cloud-live-disposable', + 'cloud-live-paid' ]) expect(files.every((file) => file.startsWith('tests/liveCloud/'))).toBe( true ) - expect(files).toEqual( - expect.arrayContaining(['tests/liveCloud/billingSmoke.spec.ts']) - ) + const expectedFiles = globSync( + 'browser_tests/tests/liveCloud/**/*.spec.ts' + ).map((file) => file.replace('browser_tests/', '').replaceAll('\\', '/')) + expect([...new Set(files)].sort()).toEqual(expectedFiles.sort()) expect(config.globalSetup).toBeFalsy() expect(config.globalTeardown).toBeFalsy() } diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index fb60695b4a2..7aec6212208 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -8,12 +8,13 @@ import { join } from 'node:path' import { promisify } from 'node:util' import type { Duplex } from 'node:stream' -import { expect } from '@playwright/test' - -import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { + expect, + liveCloudDisposableBillingFixture +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' -const test = liveCloudBillingFixture.extend<{ +const test = liveCloudDisposableBillingFixture.extend<{ sandboxProxy: { server: string; requests: string[] } }>({ sandboxProxy: async ({ liveCloudBillingConfig }, use) => { @@ -242,3 +243,62 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { test.fail(true, 'The recorded WebSocket must fail fixture teardown') }) }) + +test.describe('Live Cloud checkout network boundary', () => { + test.use({ + liveCloudBillingConfig: liveCloudBillingConfigSchema.parse({ + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', + CLOUD_ACCOUNT_EMAIL: 'routing@example.com', + CLOUD_ACCOUNT_PASSWORD: 'unused', + allowCheckout: true + }) + }) + + test('allows checkout handoff in browser and API clients', async ({ + page, + request, + context, + sandboxProxy + }) => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://localhost:5173/') + expect( + await page.evaluate(() => + fetch('/api/billing/subscribe', { method: 'POST' }).then((response) => + response.text() + ) + ) + ).toBe('POST https://testcloud.comfy.org/api/billing/subscribe') + for (const api of [request, context.request]) { + const response = await api.post( + 'https://testcloud.comfy.org/api/billing/subscribe' + ) + expect(await response.text()).toBe( + 'POST https://testcloud.comfy.org/api/billing/subscribe' + ) + } + expect(sandboxProxy.requests).toEqual([ + 'POST https://testcloud.comfy.org/api/billing/subscribe', + 'POST https://testcloud.comfy.org/api/billing/subscribe', + 'POST https://testcloud.comfy.org/api/billing/subscribe' + ]) + }) + + test('still rejects payment confirmation with checkout enabled', async ({ + context, + networkPolicy + }) => { + await expect( + context.request.post( + 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' + ) + ).rejects.toThrow('Forbidden live Cloud request') + expect([...networkPolicy.unexpected]).toEqual([ + 'Mutation POST https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' + ]) + test.fail(true, 'The recorded mutation must fail fixture teardown') + }) +}) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index cab05a50c07..ec359ab067d 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -58,6 +58,23 @@ Cloud auth tokens/session cookies, and the startup write to if the app catches the request error. The smoke fixture blocks checkout, payment, and reset mutations in both browser routing and Playwright API clients. -Use a dedicated account. This setup checks authentication, billing reads, and -canvas readiness. It does not submit checkout, payment, or reset requests. -Checkout scenarios and account requirements belong to follow-up PRs. +## Shared setup + +Use dedicated accounts for the selected environment. PRs #17870, #17538, #17543, +#17548, and #17545 each depend directly on this setup PR, #17481. + +Shared support includes authenticated API reads/writes, disposable sandbox +accounts, hosted checkout interaction, and paid-subscription provisioning used +by checkout and lifecycle tests. Shared helpers do not add scenario specs here. +Unpaid recovery, top-up assertions, decline recovery, lifecycle transitions, and +3DS outcomes stay in their own PRs. + +The smoke fixture denies billing mutations. Scenario fixtures explicitly opt in +to the permissions they need. Payment and disposable-account tests reject +production; Staging remains the default backend. Select `cloud-live-paid` for +saved-card tests and `cloud-live-disposable` for new-account tests. These projects +collect tests only when their respective scenario PRs are present. + +Traces start after permanent-account sign-in and contain authenticated network +traffic. Keep reports private. Disposable accounts require no reusable credentials. +The saved-card endpoint must be available for completed-payment assertions. diff --git a/playwright.config.ts b/playwright.config.ts index 1cdb814120e..99328e11495 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -72,6 +72,38 @@ export default defineConfig({ { name: 'cloud-live', testMatch: '**/tests/liveCloud/**/*.spec.ts', + testIgnore: ['**/liveCloud/disposable/**', '**/liveCloud/paid/**'], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + } satisfies NonNullable[number], + { + name: 'cloud-live-disposable', + testMatch: '**/tests/liveCloud/disposable/**/*.spec.ts', + testIgnore: [], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + } satisfies NonNullable[number], + { + name: 'cloud-live-paid', + testMatch: '**/tests/liveCloud/paid/**/*.spec.ts', testIgnore: [], fullyParallel: false, retries: 0, From 9fb511bbc763a75faedb98bb745a21e283318d56 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:49:03 -0700 Subject: [PATCH 3/9] test: handle real Cloud onboarding without flag overrides --- .../components/LiveCloudOnboarding.ts | 76 +++++++++++++++++++ .../fixtures/networkIsolationFixture.ts | 3 +- .../fixtures/utils/liveCloudBillingContext.ts | 15 ++-- .../utils/liveCloudBillingPolicy.test.ts | 34 +++++++++ .../fixtures/utils/liveCloudBillingPolicy.ts | 19 ++++- .../liveCloudBillingRouting.spec.ts | 56 +++++++++++++- docs/testing/cloud-billing-e2e.md | 15 ++-- 7 files changed, 202 insertions(+), 16 deletions(-) create mode 100644 browser_tests/fixtures/components/LiveCloudOnboarding.ts diff --git a/browser_tests/fixtures/components/LiveCloudOnboarding.ts b/browser_tests/fixtures/components/LiveCloudOnboarding.ts new file mode 100644 index 00000000000..ac4999da400 --- /dev/null +++ b/browser_tests/fixtures/components/LiveCloudOnboarding.ts @@ -0,0 +1,76 @@ +import { expect } from '@playwright/test' +import type { Page } from '@playwright/test' + +export class LiveCloudOnboarding { + constructor(private readonly page: Page) {} + + async dismissTutorialsWhenVisible() { + for (const testId of ['coach-landing', 'coach-card']) { + await this.page.addLocatorHandler( + this.page.getByTestId(testId), + async (dialog) => { + await dialog + .getByRole('button', { name: 'Skip', exact: true }) + .click() + } + ) + } + } + + async completeSurveyIfNeeded(frontend: string) { + const survey = this.page.getByRole('heading', { + name: "Let's get to know you", + exact: true + }) + const canvas = this.page.locator('#graph-canvas') + await expect(survey.or(canvas)).toBeVisible() + if (await survey.isVisible()) { + const next = this.page.getByRole('button', { name: 'Next', exact: true }) + const submit = this.page.getByRole('button', { + name: 'Submit', + exact: true + }) + for (let step = 0; step < 20; step++) { + const option = this.page + .getByRole('button', { pressed: false }) + .and(this.page.locator(':not([id$="-other"])')) + .first() + const text = this.page.getByRole('textbox') + await expect(option.or(text)).toBeVisible() + const control = (await option.isVisible()) ? option : text + const id = await control.getAttribute('id') + if (!id) throw new Error('Cloud survey control has no ID') + const answered = this.page.locator(`[id=${JSON.stringify(id)}]`) + if (await option.isVisible()) await control.click() + else await control.fill('Automated billing E2E') + await expect + .poll( + async () => + !(await answered.isVisible()) || + ((await next.isVisible()) && (await next.isEnabled())) || + ((await submit.isVisible()) && (await submit.isEnabled())) + ) + .toBe(true) + if ((await submit.isVisible()) && (await submit.isEnabled())) break + if ((await next.isVisible()) && (await next.isEnabled())) + await next.click() + await expect(answered).toBeHidden() + } + await expect( + submit, + 'Complete Cloud onboarding within 20 questions' + ).toBeEnabled() + const [response] = await Promise.all([ + this.page.waitForResponse( + (response) => + response.url() === `${frontend}/api/settings` && + response.request().method() === 'POST' + ), + submit.click() + ]) + expect(response.status(), 'Save Cloud onboarding survey').toBe(200) + await expect(survey).toBeHidden() + } + await expect(canvas).toBeVisible() + } +} diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index 71ce840d662..c694b1378e7 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -42,7 +42,8 @@ function guardApiRequests( !isLiveCloudMutationAllowed( url, method.toUpperCase(), - liveCloudBillingConfig + liveCloudBillingConfig, + options?.data ) ) { const message = `Mutation ${method} ${url.origin}${url.pathname}` diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index 5fb1911b7dc..b61995c7d07 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -1,7 +1,7 @@ import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' import type { BrowserContext, Page } from '@playwright/test' -import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' import { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' @@ -39,7 +39,12 @@ export async function installLiveCloudBillingRouting( if ( (networkPolicy.origins.has(url.origin) || LIVE_CHECKOUT_ORIGINS.includes(url.origin)) && - !isLiveCloudMutationAllowed(targetUrl, request.method(), config) + !isLiveCloudMutationAllowed( + targetUrl, + request.method(), + config, + url.pathname === '/api/settings' ? request.postDataJSON() : undefined + ) ) { networkPolicy.unexpected.add( `Mutation ${request.method()} ${url.origin}${url.pathname}` @@ -70,9 +75,8 @@ export async function signInToLiveCloud( ) { if (!config.CLOUD_ACCOUNT_EMAIL || !config.CLOUD_ACCOUNT_PASSWORD) throw new Error('Live Cloud sign-in requires account credentials') - await new FeatureFlagHelper(page).seedFlags({ - onboarding_survey_enabled: false - }) + const onboarding = new LiveCloudOnboarding(page) + await onboarding.dismissTutorialsWhenVisible() await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) await page .getByRole('button', { name: 'Use email instead', exact: true }) @@ -94,6 +98,7 @@ export async function signInToLiveCloud( zBillingStatusResponse.parse(await response.json()) const authorization = await response.request().headerValue('authorization') if (!authorization) throw new Error('Missing billing authorization') + await onboarding.completeSurveyIfNeeded(config.PLAYWRIGHT_TEST_URL) return new LiveCloudBillingSession( page.request, config.PLAYWRIGHT_SETUP_API_URL, diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index 76303958355..bf3b427ab8c 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -32,6 +32,11 @@ describe('live Cloud mutation policy', () => { allowed: true }, { method: 'POST', path: '/api/settings/other', allowed: false }, + { + method: 'POST', + path: '/api/settings/Comfy.OnboardingCoachmarks.Seen', + allowed: true + }, { method: 'POST', path: '/api/billing/subscribe', allowed: false }, { method: 'POST', path: '/api/billing/payment-portal', allowed: false }, { method: 'POST', path: '/customers/credit', allowed: false }, @@ -70,6 +75,35 @@ describe('live Cloud mutation policy', () => { allowed ) }) + + it.for([ + { data: { onboarding_survey: { intent: 'exploring' } }, allowed: true }, + { data: undefined, allowed: false }, + { data: {}, allowed: false }, + { data: { onboarding_survey: null }, allowed: false }, + { data: { unrelated: true }, allowed: false }, + { + data: { onboarding_survey: { intent: 'exploring' }, unrelated: true }, + allowed: false + } + ])('survey settings $data allowed=$allowed', ({ data, allowed }) => { + expect( + isLiveCloudMutationAllowed( + new URL('/api/settings', config.PLAYWRIGHT_SETUP_API_URL), + 'POST', + config, + data + ) + ).toBe(allowed) + expect( + isLiveCloudMutationAllowed( + new URL('/api/settings', 'https://stagingcloud.comfy.org'), + 'POST', + config, + data + ) + ).toBe(false) + }) }) describe('live Cloud checkout mutation policy', () => { diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index 772c26a8edb..ae419cbfdd8 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -55,10 +55,24 @@ export function isLiveCloudMutationAllowed( | 'allowCheckout' | 'allowPayments' | 'allowAccountCreation' - > + >, + data?: unknown ): boolean { if (SAFE_METHODS.includes(method)) return true if (method !== 'POST') return false + if ( + url.origin === config.PLAYWRIGHT_SETUP_API_URL && + url.pathname === '/api/settings' + ) { + return ( + typeof data === 'object' && + data !== null && + Object.keys(data).length === 1 && + 'onboarding_survey' in data && + typeof data.onboarding_survey === 'object' && + data.onboarding_survey !== null + ) + } if ( config.allowAccountCreation && config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' && @@ -105,7 +119,8 @@ export function isLiveCloudMutationAllowed( [ '/api/auth/token', '/api/auth/session', - '/api/settings/Comfy.InstalledVersion' + '/api/settings/Comfy.InstalledVersion', + '/api/settings/Comfy.OnboardingCoachmarks.Seen' ].includes(url.pathname) ) } diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index 7aec6212208..0a2016ce003 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -12,6 +12,7 @@ import { expect, liveCloudDisposableBillingFixture } from '@e2e/fixtures/liveCloudDisposableBillingFixture' +import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' const test = liveCloudDisposableBillingFixture.extend<{ @@ -95,6 +96,37 @@ const test = liveCloudDisposableBillingFixture.extend<{ }) test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { + for (const testId of ['coach-landing', 'coach-card']) { + test(`dismisses late ${testId} across navigation without closing billing`, async ({ + page + }) => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ + contentType: 'text/html', + body: ` + +
+ +
+ + ` + }) + ) + await new LiveCloudOnboarding(page).dismissTutorialsWhenVisible() + for (let visit = 0; visit < 2; visit++) { + await page.goto('http://localhost:5173/') + await page.getByRole('button', { name: 'Show tutorial' }).click() + await page.getByRole('button', { name: 'Continue billing' }).click() + await expect(page.getByTestId(testId)).toBeHidden() + await expect( + page.getByRole('dialog', { name: 'Confirm your payment' }) + ).toBeVisible() + } + }) + } + test('rewrites browser auth to Cloud and provisions customers at the Comfy API', async ({ page, sandboxProxy @@ -117,9 +149,19 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ) ) ).toBe('POST https://testapi.comfy.org/customers') + expect( + await page.evaluate(() => + fetch('/api/settings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ onboarding_survey: { intent: 'exploring' } }) + }).then((response) => response.text()) + ) + ).toBe('POST https://testcloud.comfy.org/api/settings') expect(sandboxProxy.requests).toEqual([ 'POST https://testcloud.comfy.org/api/auth/token', - 'POST https://testapi.comfy.org/customers' + 'POST https://testapi.comfy.org/customers', + 'POST https://testcloud.comfy.org/api/settings' ]) }) @@ -196,6 +238,15 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { expect(await auth.text()).toBe( 'POST https://testcloud.comfy.org/api/auth/session' ) + const survey = await api.post( + 'https://testcloud.comfy.org/api/settings', + { + data: { onboarding_survey: { intent: 'exploring' } } + } + ) + expect(await survey.text()).toBe( + 'POST https://testcloud.comfy.org/api/settings' + ) await expect( api.post('https://testcloud.comfy.org/customers') ).rejects.toThrow('Forbidden live Cloud request') @@ -213,7 +264,8 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ).rejects.toThrow('Forbidden live Cloud request') expect(sandboxProxy.requests).toEqual([ 'POST https://testapi.comfy.org/customers', - 'POST https://testcloud.comfy.org/api/auth/session' + 'POST https://testcloud.comfy.org/api/auth/session', + 'POST https://testcloud.comfy.org/api/settings' ]) expect([...networkPolicy.unexpected]).toEqual([ 'Mutation POST https://testcloud.comfy.org/customers', diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index ec359ab067d..d6d74bcaa36 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -41,10 +41,11 @@ account credentials. The fixture forwards frontend API requests to the selected backend and derives its matching customer API and Stripe checkout mode. A deployed frontend must match the selected backend origin; local URLs exercise the PR code. -Live tests do not install shared response mocks. They override only -`onboarding_survey_enabled=false` in the local PR frontend to skip the unrelated -survey. Deployed builds ignore this dev-only override and require completed -onboarding. Assets, +Live tests do not install shared response mocks or override feature flags. +Sign-in completes the real onboarding survey when required and waits for the +canvas. Known tutorial dialogs are dismissed through their Skip button whenever +they appear during an interaction, including after navigation. Billing dialogs +are left to the scenario. Assets, location detection, Google auth scripts, analytics, and feature flags use their real services. Accounts must have the hosted checkout flow enabled in the selected environment. @@ -53,8 +54,10 @@ The smoke test signs in, validates the browser's real billing-status response, waits for app readiness, and checks that the graph canvas is visible. It attaches a post-login screenshot. The shared network boundary rejects Cloud and Stripe mutations except the specific POST endpoints used for Firebase sign-in/token refresh, customer provisioning, -Cloud auth tokens/session cookies, and the startup write to -`/api/settings/Comfy.InstalledVersion`. A rejected mutation fails the test even +Cloud auth tokens/session cookies, the startup write to +`/api/settings/Comfy.InstalledVersion`, and onboarding survey/tutorial state. +Bulk settings writes may contain only `onboarding_survey`. +A rejected mutation fails the test even if the app catches the request error. The smoke fixture blocks checkout, payment, and reset mutations in both browser routing and Playwright API clients. From bd177a2092dedea965b24601fddd15027a53237e Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:25:59 -0700 Subject: [PATCH 4/9] test: verify saved cards through hosted billing portal --- .../fixtures/helpers/LiveCloudCheckout.ts | 30 ++++++++++++++----- .../utils/liveCloudBillingPolicy.test.ts | 1 + .../fixtures/utils/liveCloudBillingPolicy.ts | 5 ++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts index a937f26fb0f..1c7a5aa00dc 100644 --- a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts +++ b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts @@ -2,7 +2,7 @@ import { zBillingBalanceResponse, zBillingOpStatusResponse, zBillingStatusResponse, - zListSavedPaymentMethodsResponse, + zPaymentPortalResponse, zPreviewSubscribeResponse, zSubscribeResponse } from '@comfyorg/ingest-types/zod' @@ -167,11 +167,10 @@ export class LiveCloudCheckout { .amount_micros ) .toBe(expectedBalance) - const methods = await session.read( - '/api/billing/payment-methods', - zListSavedPaymentMethodsResponse + const paymentMethodCount = await this.verifySavedPaymentMethods( + session, + expectedPaymentMethodCount ) - expect(methods).toHaveLength(expectedPaymentMethodCount) await testInfo.attach('checkout-completion.json', { body: JSON.stringify({ operationId, @@ -179,7 +178,7 @@ export class LiveCloudCheckout { grantCents: Number(preview.credits_today_cents), balanceBeforeCents, balanceAfterCents: expectedBalance, - paymentMethodCount: methods.length + paymentMethodCount }), contentType: 'application/json' }) @@ -189,10 +188,27 @@ export class LiveCloudCheckout { operationId, planSlug: preview.new_plan.slug, grantCents: Number(preview.credits_today_cents), - paymentMethodCount: methods.length + paymentMethodCount } } + async verifySavedPaymentMethods( + session: LiveCloudBillingSession, + expectedCount: number + ) { + const portal = await session.post( + '/api/billing/payment-portal', + { return_url: this.frontend }, + zPaymentPortalResponse + ) + expect(new URL(portal.url).origin).toBe('https://checkout.comfy.org') + await this.page.goto(portal.url) + await expect(this.page.getByText(/^Payment methods?$/)).toBeVisible() + const cards = this.page.getByText(/•••• \d{4}$/) + await expect(cards).toHaveCount(expectedCount) + return await cards.count() + } + async startCheckout(action = this.subscribe.or(this.resumePayment)) { await expect(action).toBeEnabled() const [response, checkout] = await Promise.all([ diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index bf3b427ab8c..fbdea395459 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -358,6 +358,7 @@ describe('Disposable payment permissions', () => { it.for([ 'https://identitytoolkit.googleapis.com/v1/accounts:signUp', 'https://api.stripe.com/v1/payment_methods', + 'https://checkout.comfy.org/ajax/metrics_batch', 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' ])('allows %s only for opted-in sandbox tests', (url) => { expect(isLiveCloudMutationAllowed(new URL(url), 'POST', config)).toBe(false) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index ae419cbfdd8..b0425370749 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -89,6 +89,11 @@ export function isLiveCloudMutationAllowed( url.pathname === '/api/billing/payment-portal' ) return true + if ( + url.origin === 'https://checkout.comfy.org' && + url.pathname === '/ajax/metrics_batch' + ) + return true if ( url.origin === 'https://api.stripe.com' && (url.pathname === '/v1/payment_methods' || From 547243b561c70fc7acd7e9238acff2a109291b4a Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:37:36 -0700 Subject: [PATCH 5/9] docs: explain hosted saved-card verification --- docs/testing/cloud-billing-e2e.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index d6d74bcaa36..96bdcab10e3 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -80,4 +80,5 @@ collect tests only when their respective scenario PRs are present. Traces start after permanent-account sign-in and contain authenticated network traffic. Keep reports private. Disposable accounts require no reusable credentials. -The saved-card endpoint must be available for completed-payment assertions. +Completed-payment assertions verify saved cards through the real hosted billing +portal, because Staging disables the embedded-checkout saved-card API. From 2570532fdc865f09ac06a0831c07a79e84d1e994 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:45:38 -0700 Subject: [PATCH 6/9] test: wait for disposable billing customer provisioning --- .../fixtures/helpers/LiveCloudBillingSession.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts index 77fc470aa59..8e43baeff49 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts @@ -86,10 +86,12 @@ export class LiveCloudBillingSession { } async ensureProvisioned(returnUrl: string) { - await this.post( - '/api/billing/payment-portal', - { return_url: returnUrl }, - zPaymentPortalResponse - ) + await expect(async () => { + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) + }).toPass({ timeout: 30_000 }) } } From 21390997265df1de8959dc2fef90901276a6f718 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:57:10 -0700 Subject: [PATCH 7/9] test: simplify live Cloud billing fixture complexity --- .../components/LiveCloudOnboarding.ts | 54 +++++++++-------- .../fixtures/utils/liveCloudBillingConfig.ts | 2 + .../fixtures/utils/liveCloudBillingContext.ts | 26 ++++---- .../fixtures/utils/liveCloudBillingPolicy.ts | 60 ++++++++++--------- 4 files changed, 77 insertions(+), 65 deletions(-) diff --git a/browser_tests/fixtures/components/LiveCloudOnboarding.ts b/browser_tests/fixtures/components/LiveCloudOnboarding.ts index ac4999da400..550a8669226 100644 --- a/browser_tests/fixtures/components/LiveCloudOnboarding.ts +++ b/browser_tests/fixtures/components/LiveCloudOnboarding.ts @@ -1,5 +1,5 @@ import { expect } from '@playwright/test' -import type { Page } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' export class LiveCloudOnboarding { constructor(private readonly page: Page) {} @@ -31,30 +31,7 @@ export class LiveCloudOnboarding { exact: true }) for (let step = 0; step < 20; step++) { - const option = this.page - .getByRole('button', { pressed: false }) - .and(this.page.locator(':not([id$="-other"])')) - .first() - const text = this.page.getByRole('textbox') - await expect(option.or(text)).toBeVisible() - const control = (await option.isVisible()) ? option : text - const id = await control.getAttribute('id') - if (!id) throw new Error('Cloud survey control has no ID') - const answered = this.page.locator(`[id=${JSON.stringify(id)}]`) - if (await option.isVisible()) await control.click() - else await control.fill('Automated billing E2E') - await expect - .poll( - async () => - !(await answered.isVisible()) || - ((await next.isVisible()) && (await next.isEnabled())) || - ((await submit.isVisible()) && (await submit.isEnabled())) - ) - .toBe(true) - if ((await submit.isVisible()) && (await submit.isEnabled())) break - if ((await next.isVisible()) && (await next.isEnabled())) - await next.click() - await expect(answered).toBeHidden() + if (await this.answerSurveyQuestion(next, submit)) break } await expect( submit, @@ -73,4 +50,31 @@ export class LiveCloudOnboarding { } await expect(canvas).toBeVisible() } + + private async answerSurveyQuestion(next: Locator, submit: Locator) { + const option = this.page + .getByRole('button', { pressed: false }) + .and(this.page.locator(':not([id$="-other"])')) + .first() + const text = this.page.getByRole('textbox') + await expect(option.or(text)).toBeVisible() + const control = (await option.isVisible()) ? option : text + const id = await control.getAttribute('id') + if (!id) throw new Error('Cloud survey control has no ID') + const answered = this.page.locator(`[id=${JSON.stringify(id)}]`) + if (await option.isVisible()) await control.click() + else await control.fill('Automated billing E2E') + await expect + .poll( + async () => + !(await answered.isVisible()) || + ((await next.isVisible()) && (await next.isEnabled())) || + ((await submit.isVisible()) && (await submit.isEnabled())) + ) + .toBe(true) + if ((await submit.isVisible()) && (await submit.isEnabled())) return true + if ((await next.isVisible()) && (await next.isEnabled())) await next.click() + await expect(answered).toBeHidden() + return false + } } diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 65f51e2eb62..d19272edd65 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -56,6 +56,8 @@ export const liveCloudBillingConfigSchema = z }) } } + }) + .superRefine((config, ctx) => { if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return const frontend = new URL(config.PLAYWRIGHT_TEST_URL) const local = diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index b61995c7d07..1a5bd45fea6 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -1,5 +1,5 @@ import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' -import type { BrowserContext, Page } from '@playwright/test' +import type { BrowserContext, Page, Route } from '@playwright/test' import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' import { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' @@ -53,22 +53,26 @@ export async function installLiveCloudBillingRouting( return } if (isCloudProxyRequest) { - try { - const response = await route.fetch({ - url: targetUrl.href, - maxRedirects: 0 - }) - await route.fulfill({ response }) - } catch { - if (request.frame().page().isClosed()) return - throw new Error(`Cloud proxy request failed: ${url.pathname}`) - } + await forwardCloudRequest(route, targetUrl) return } await route.fallback() }) } +async function forwardCloudRequest(route: Route, targetUrl: URL) { + try { + const response = await route.fetch({ + url: targetUrl.href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + } catch { + if (route.request().frame().page().isClosed()) return + throw new Error(`Cloud proxy request failed: ${targetUrl.pathname}`) + } +} + export async function signInToLiveCloud( page: Page, config = loadLiveCloudBillingConfig() diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index b0425370749..1ddcee00a41 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -64,14 +64,7 @@ export function isLiveCloudMutationAllowed( url.origin === config.PLAYWRIGHT_SETUP_API_URL && url.pathname === '/api/settings' ) { - return ( - typeof data === 'object' && - data !== null && - Object.keys(data).length === 1 && - 'onboarding_survey' in data && - typeof data.onboarding_survey === 'object' && - data.onboarding_survey !== null - ) + return isOnboardingSurveyUpdate(data) } if ( config.allowAccountCreation && @@ -82,27 +75,10 @@ export function isLiveCloudMutationAllowed( return true if ( config.allowPayments && - config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' - ) { - if ( - url.origin === config.PLAYWRIGHT_SETUP_API_URL && - url.pathname === '/api/billing/payment-portal' - ) - return true - if ( - url.origin === 'https://checkout.comfy.org' && - url.pathname === '/ajax/metrics_batch' - ) - return true - if ( - url.origin === 'https://api.stripe.com' && - (url.pathname === '/v1/payment_methods' || - /^\/v1\/payment_pages\/cs_test_[A-Za-z0-9]+\/confirm$/.test( - url.pathname - )) - ) - return true - } + config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' && + isPaymentPostAllowed(url, config.PLAYWRIGHT_SETUP_API_URL) + ) + return true if (url.origin === 'https://identitytoolkit.googleapis.com') { return ['/v1/accounts:signInWithPassword', '/v1/accounts:lookup'].includes( url.pathname @@ -130,6 +106,32 @@ export function isLiveCloudMutationAllowed( ) } +function isOnboardingSurveyUpdate(data: unknown): boolean { + return ( + typeof data === 'object' && + data !== null && + Object.keys(data).length === 1 && + 'onboarding_survey' in data && + typeof data.onboarding_survey === 'object' && + data.onboarding_survey !== null + ) +} + +function isPaymentPostAllowed(url: URL, backend: string): boolean { + if ( + url.origin === 'https://checkout.comfy.org' && + url.pathname === '/ajax/metrics_batch' + ) + return true + if (url.origin === backend && url.pathname === '/api/billing/payment-portal') + return true + return ( + url.origin === 'https://api.stripe.com' && + (url.pathname === '/v1/payment_methods' || + /^\/v1\/payment_pages\/cs_test_[A-Za-z0-9]+\/confirm$/.test(url.pathname)) + ) +} + function isCheckoutPostAllowed(url: URL, backend: string): boolean { if (url.origin === backend) { return [ From a6a7df43ac061c2d61b0f4f4e29f7e7853b2bc13 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:22:38 -0700 Subject: [PATCH 8/9] fix: tighten live Cloud test setup and validation --- .../components/LiveCloudOnboarding.ts | 48 ++++---- browser_tests/fixtures/components/Tour.ts | 14 +++ .../helpers/LiveCloudBillingSession.ts | 20 ++-- .../fixtures/helpers/LiveCloudCheckout.ts | 18 ++- .../fixtures/liveCloudBillingFixture.ts | 12 +- .../fixtures/networkIsolationFixture.ts | 13 ++- .../utils/liveCloudBillingConfig.test.ts | 12 ++ .../fixtures/utils/liveCloudBillingConfig.ts | 43 ++++--- .../fixtures/utils/liveCloudBillingContext.ts | 16 ++- .../utils/liveCloudBillingPolicy.test.ts | 87 +++++++++++--- .../fixtures/utils/liveCloudBillingPolicy.ts | 51 ++++++++- browser_tests/playwrightSelection.test.ts | 29 +++-- .../liveCloudBillingRouting.spec.ts | 108 +++++++++++++++--- ...-BILLING-0031-real-cloud-checkout-tests.md | 4 +- docs/testing/cloud-billing-e2e.md | 14 ++- 15 files changed, 361 insertions(+), 128 deletions(-) diff --git a/browser_tests/fixtures/components/LiveCloudOnboarding.ts b/browser_tests/fixtures/components/LiveCloudOnboarding.ts index 550a8669226..551d9931fa0 100644 --- a/browser_tests/fixtures/components/LiveCloudOnboarding.ts +++ b/browser_tests/fixtures/components/LiveCloudOnboarding.ts @@ -2,34 +2,32 @@ import { expect } from '@playwright/test' import type { Locator, Page } from '@playwright/test' export class LiveCloudOnboarding { - constructor(private readonly page: Page) {} + public readonly survey: Locator + public readonly canvas: Locator + public readonly next: Locator + public readonly submit: Locator + public readonly option: Locator + public readonly text: Locator - async dismissTutorialsWhenVisible() { - for (const testId of ['coach-landing', 'coach-card']) { - await this.page.addLocatorHandler( - this.page.getByTestId(testId), - async (dialog) => { - await dialog - .getByRole('button', { name: 'Skip', exact: true }) - .click() - } - ) - } - } - - async completeSurveyIfNeeded(frontend: string) { - const survey = this.page.getByRole('heading', { + constructor(private readonly page: Page) { + this.survey = page.getByRole('heading', { name: "Let's get to know you", exact: true }) - const canvas = this.page.locator('#graph-canvas') + this.canvas = page.locator('#graph-canvas') + this.next = page.getByRole('button', { name: 'Next', exact: true }) + this.submit = page.getByRole('button', { name: 'Submit', exact: true }) + this.option = page + .getByRole('button', { pressed: false }) + .and(page.locator('[aria-pressed="false"]:not([id$="-other"])')) + .first() + this.text = page.getByRole('textbox') + } + + async completeSurveyIfNeeded(frontend: string) { + const { survey, canvas, next, submit } = this await expect(survey.or(canvas)).toBeVisible() if (await survey.isVisible()) { - const next = this.page.getByRole('button', { name: 'Next', exact: true }) - const submit = this.page.getByRole('button', { - name: 'Submit', - exact: true - }) for (let step = 0; step < 20; step++) { if (await this.answerSurveyQuestion(next, submit)) break } @@ -52,11 +50,7 @@ export class LiveCloudOnboarding { } private async answerSurveyQuestion(next: Locator, submit: Locator) { - const option = this.page - .getByRole('button', { pressed: false }) - .and(this.page.locator(':not([id$="-other"])')) - .first() - const text = this.page.getByRole('textbox') + const { option, text } = this await expect(option.or(text)).toBeVisible() const control = (await option.isVisible()) ? option : text const id = await control.getAttribute('id') diff --git a/browser_tests/fixtures/components/Tour.ts b/browser_tests/fixtures/components/Tour.ts index 9c1569cc8e2..2f92b1094be 100644 --- a/browser_tests/fixtures/components/Tour.ts +++ b/browser_tests/fixtures/components/Tour.ts @@ -17,6 +17,7 @@ export class OnboardingCoachmarks { /** The current spotlight step card (the dialog carrying a "Step N of M" label). */ public readonly spotlight: Locator public readonly card: Locator + public readonly cardSkipButton: Locator public readonly cardNextButton: Locator public readonly cardDoneButton: Locator @@ -31,10 +32,23 @@ export class OnboardingCoachmarks { }) this.spotlight = page.getByTestId('coach-spotlight') this.card = page.getByRole('dialog').filter({ hasText: /Step \d+ of \d+/ }) + this.cardSkipButton = this.card.getByRole('button', { + name: 'Skip', + exact: true + }) this.cardNextButton = this.card.getByRole('button', { name: 'Next' }) this.cardDoneButton = this.card.getByRole('button', { name: 'Done' }) } + async dismissWhenVisible() { + await this.page.addLocatorHandler(this.landing, async () => { + await this.landingSkipButton.click() + }) + await this.page.addLocatorHandler(this.card, async () => { + await this.cardSkipButton.click() + }) + } + /** The tour's in-app help button, which replays it past the seen-flag. */ replayButton(tour: CoachTour): Locator { return this.page.getByRole('button', { name: TOUR_REPLAY_BUTTONS[tour] }) diff --git a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts index 8e43baeff49..cfc78a0660b 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBillingSession.ts @@ -1,3 +1,4 @@ +import { getLiveCloudEnvironment } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { zBillingStatusResponse, zCurrentWorkspaceResponse, @@ -9,11 +10,14 @@ import { expect } from '@playwright/test' import type { z } from 'zod' export class LiveCloudBillingSession { + private readonly environment: ReturnType constructor( private readonly request: APIRequestContext, private readonly backend: string, private readonly headers: Record - ) {} + ) { + this.environment = getLiveCloudEnvironment(backend) + } async read(path: string, schema: z.ZodType): Promise { const response = await this.request @@ -58,7 +62,7 @@ export class LiveCloudBillingSession { '/api/billing/status', zBillingStatusResponse ) - if (this.backend === 'https://cloud.comfy.org') { + if (this.environment.stripeMode === 'live') { const paymentMethods = await this.read( '/api/billing/payment-methods', zListSavedPaymentMethodsResponse @@ -86,12 +90,10 @@ export class LiveCloudBillingSession { } async ensureProvisioned(returnUrl: string) { - await expect(async () => { - await this.post( - '/api/billing/payment-portal', - { return_url: returnUrl }, - zPaymentPortalResponse - ) - }).toPass({ timeout: 30_000 }) + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) } } diff --git a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts index 1c7a5aa00dc..85d2f986ea2 100644 --- a/browser_tests/fixtures/helpers/LiveCloudCheckout.ts +++ b/browser_tests/fixtures/helpers/LiveCloudCheckout.ts @@ -1,3 +1,4 @@ +import { getLiveCloudEnvironment } from '@e2e/fixtures/utils/liveCloudBillingConfig' import { zBillingBalanceResponse, zBillingOpStatusResponse, @@ -20,17 +21,23 @@ function matchesBillingResponse( } export class LiveCloudCheckout { + private readonly environment: ReturnType readonly page: Page readonly confirmation readonly resumePayment + readonly paymentMethodsHeading + readonly savedCards private readonly subscribe constructor( private readonly comfyPage: ComfyPage, private readonly frontend: string, - private readonly backend: string + backend: string ) { + this.environment = getLiveCloudEnvironment(backend) this.page = comfyPage.page + this.paymentMethodsHeading = this.page.getByText(/^Payment methods?$/) + this.savedCards = this.page.getByText(/•••• \d{4}$/) this.confirmation = this.page.getByRole('dialog').filter({ has: this.page.getByRole('heading', { name: 'Confirm your payment', @@ -203,10 +210,9 @@ export class LiveCloudCheckout { ) expect(new URL(portal.url).origin).toBe('https://checkout.comfy.org') await this.page.goto(portal.url) - await expect(this.page.getByText(/^Payment methods?$/)).toBeVisible() - const cards = this.page.getByText(/•••• \d{4}$/) - await expect(cards).toHaveCount(expectedCount) - return await cards.count() + await expect(this.paymentMethodsHeading).toBeVisible() + await expect(this.savedCards).toHaveCount(expectedCount) + return expectedCount } async startCheckout(action = this.subscribe.or(this.resumePayment)) { @@ -227,7 +233,7 @@ export class LiveCloudCheckout { const subscription = zSubscribeResponse.parse(await response.json()) expect(subscription.status).toBe('needs_payment_method') expect(subscription.billing_op_id).not.toBe('') - const mode = this.backend === 'https://cloud.comfy.org' ? 'live' : 'test' + const mode = this.environment.stripeMode const testCheckoutUrl = new RegExp( `^https://checkout\\.(?:stripe\\.com|comfy\\.org)/c/pay/cs_${mode}_` ) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index d9b81caeb04..0047a9af587 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -12,16 +12,8 @@ export const liveCloudBillingFixture = base.extend<{ liveCloudBillingConfig: async ({ liveCloudBillingConfig }, use) => { await use(liveCloudBillingConfig ?? loadLiveCloudBillingConfig()) }, - billingSession: async ({ page, liveCloudBillingConfig }, use, testInfo) => { - const session = await signInToLiveCloud(page, liveCloudBillingConfig) - await page.context().tracing.start({ screenshots: true, snapshots: true }) - try { - await use(session) - } finally { - const path = testInfo.outputPath('billing-trace.zip') - await page.context().tracing.stop({ path }) - await testInfo.attach('trace', { path, contentType: 'application/zip' }) - } + billingSession: async ({ page, liveCloudBillingConfig }, use) => { + await use(await signInToLiveCloud(page, liveCloudBillingConfig)) }, comfyPage: async ( { page, request, billingSession: _billingSession }, diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index c694b1378e7..85702a0bf4a 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -9,6 +9,7 @@ import { LIVE_CHECKOUT_ORIGINS, getBlockedRequestViolation, isLiveCloudMutationAllowed, + isLiveCloudAuxiliaryPost, isReportedViolation } from '@e2e/fixtures/utils/liveCloudBillingPolicy' import type { NetworkPolicy } from '@e2e/fixtures/utils/networkPolicy' @@ -73,7 +74,12 @@ async function installContextNetworkIsolation( ) await context.route('**/*', async (route) => { const url = new URL(route.request().url()) - if (origins.has(url.origin) || liveCloudBillingConfig) { + if ( + origins.has(url.origin) || + (liveCloudBillingConfig && + (['GET', 'HEAD', 'OPTIONS'].includes(route.request().method()) || + isLiveCloudAuxiliaryPost(url, route.request().method()))) + ) { await route.continue() return } @@ -149,10 +155,7 @@ export const networkIsolationFixture = base.extend<{ liveCloudBillingConfig.customerOrigin, 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', - liveCloudBillingConfig.PLAYWRIGHT_SETUP_API_URL === - 'https://cloud.comfy.org' - ? 'https://dreamboothy.firebaseapp.com' - : 'https://dreamboothy-dev.firebaseapp.com', + liveCloudBillingConfig.environment.firebaseOrigin, ...(liveCloudBillingConfig.allowPayments ? [ 'https://pay.google.com', diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index 60bbdf99688..535923b728b 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -13,6 +13,18 @@ const sandboxConfig = { } describe('Live Cloud billing prerequisites', () => { + it('routes an omitted backend to the Staging customer API', () => { + const selected = liveCloudBillingConfigSchema.parse({ + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + CLOUD_ACCOUNT_EMAIL: sandboxConfig.CLOUD_ACCOUNT_EMAIL, + CLOUD_ACCOUNT_PASSWORD: sandboxConfig.CLOUD_ACCOUNT_PASSWORD + }) + expect(selected.PLAYWRIGHT_SETUP_API_URL).toBe( + 'https://stagingcloud.comfy.org' + ) + expect(selected.customerOrigin).toBe('https://stagingapi.comfy.org') + }) + it.for([ 'https://testcloud.comfy.org.example.com', 'https://testcloud.comfy.org/redirect', diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index d19272edd65..bc1c448d4d8 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -10,14 +10,26 @@ const originURL = z }, 'Use an origin without a path, query, credentials, or fragment') .transform((value) => new URL(value).origin) -function getLiveCloudCustomerOrigin(cloudOrigin: string): string | undefined { - if (cloudOrigin === 'https://cloud.comfy.org') return 'https://api.comfy.org' - if (cloudOrigin === 'https://testcloud.comfy.org') - return 'https://testapi.comfy.org' - if (cloudOrigin === 'https://stagingcloud.comfy.org') - return 'https://stagingapi.comfy.org' - if (/^https:\/\/pr-\d+\.testenvs\.comfy\.org$/.test(cloudOrigin)) { - return cloudOrigin.replace('.testenvs.', '-registry.testenvs.') +export function getLiveCloudEnvironment(cloudOrigin: string) { + const production = cloudOrigin === 'https://cloud.comfy.org' + const customerOrigin = + cloudOrigin === 'https://cloud.comfy.org' + ? 'https://api.comfy.org' + : cloudOrigin === 'https://testcloud.comfy.org' + ? 'https://testapi.comfy.org' + : cloudOrigin === 'https://stagingcloud.comfy.org' + ? 'https://stagingapi.comfy.org' + : /^https:\/\/pr-\d+\.testenvs\.comfy\.org$/.test(cloudOrigin) + ? cloudOrigin.replace('.testenvs.', '-registry.testenvs.') + : undefined + if (!customerOrigin) throw new Error('Unsupported Cloud environment') + const stripeMode: 'live' | 'test' = production ? 'live' : 'test' + return { + customerOrigin, + stripeMode, + firebaseOrigin: production + ? 'https://dreamboothy.firebaseapp.com' + : 'https://dreamboothy-dev.firebaseapp.com' } } @@ -75,10 +87,16 @@ export const liveCloudBillingConfigSchema = z } }) .transform((config, ctx) => { - const customerOrigin = getLiveCloudCustomerOrigin( - config.PLAYWRIGHT_SETUP_API_URL - ) - if (!customerOrigin) { + try { + const environment = getLiveCloudEnvironment( + config.PLAYWRIGHT_SETUP_API_URL + ) + return { + ...config, + customerOrigin: environment.customerOrigin, + environment + } + } catch { ctx.addIssue({ code: 'custom', path: ['PLAYWRIGHT_SETUP_API_URL'], @@ -86,7 +104,6 @@ export const liveCloudBillingConfigSchema = z }) return z.NEVER } - return { ...config, customerOrigin } }) export type LiveCloudBillingConfig = z.infer< diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index 1a5bd45fea6..8f67266bbc2 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -1,6 +1,8 @@ import { zBillingStatusResponse } from '@comfyorg/ingest-types/zod' import type { BrowserContext, Page, Route } from '@playwright/test' +import { expect } from '@playwright/test' +import { OnboardingCoachmarks } from '@e2e/fixtures/components/Tour' import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' import { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import type { LiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' @@ -79,8 +81,7 @@ export async function signInToLiveCloud( ) { if (!config.CLOUD_ACCOUNT_EMAIL || !config.CLOUD_ACCOUNT_PASSWORD) throw new Error('Live Cloud sign-in requires account credentials') - const onboarding = new LiveCloudOnboarding(page) - await onboarding.dismissTutorialsWhenVisible() + await new OnboardingCoachmarks(page).dismissWhenVisible() await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) await page .getByRole('button', { name: 'Use email instead', exact: true }) @@ -102,7 +103,16 @@ export async function signInToLiveCloud( zBillingStatusResponse.parse(await response.json()) const authorization = await response.request().headerValue('authorization') if (!authorization) throw new Error('Missing billing authorization') - await onboarding.completeSurveyIfNeeded(config.PLAYWRIGHT_TEST_URL) + if (config.allowAccountCreation) { + await new LiveCloudOnboarding(page).completeSurveyIfNeeded( + config.PLAYWRIGHT_TEST_URL + ) + } else { + await expect( + page.locator('#graph-canvas'), + 'Permanent accounts must complete onboarding when the survey flag is enabled' + ).toBeVisible() + } return new LiveCloudBillingSession( page.request, config.PLAYWRIGHT_SETUP_API_URL, diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index fbdea395459..4ddd7c7eac3 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -1,11 +1,15 @@ import { describe, expect, it } from 'vitest' -import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + getLiveCloudEnvironment, + liveCloudBillingConfigSchema +} from '@e2e/fixtures/utils/liveCloudBillingConfig' import { getBlockedRequestViolation, getLiveCloudDestinationViolation, isLiveCloudMutationAllowed, + isLiveCloudAuxiliaryPost, isReportedViolation } from '@e2e/fixtures/utils/liveCloudBillingPolicy' @@ -13,6 +17,7 @@ const config = { PLAYWRIGHT_TEST_URL: 'http://localhost:5173', PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', customerOrigin: 'https://testapi.comfy.org', + environment: getLiveCloudEnvironment('https://testcloud.comfy.org'), allowCheckout: false, allowPayments: false, allowAccountCreation: false @@ -205,6 +210,7 @@ describe('production checkout', () => { { ...config, PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + environment: getLiveCloudEnvironment('https://cloud.comfy.org'), allowCheckout: true } ) @@ -356,30 +362,77 @@ describe('blocked request violations', () => { describe('Disposable payment permissions', () => { it.for([ - 'https://identitytoolkit.googleapis.com/v1/accounts:signUp', - 'https://api.stripe.com/v1/payment_methods', - 'https://checkout.comfy.org/ajax/metrics_batch', - 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm' - ])('allows %s only for opted-in sandbox tests', (url) => { - expect(isLiveCloudMutationAllowed(new URL(url), 'POST', config)).toBe(false) - const enabled = { - ...config, - allowAccountCreation: true, - allowPayments: true + { + url: 'https://identitytoolkit.googleapis.com/v1/accounts:signUp', + permission: 'allowAccountCreation', + opposite: 'allowPayments' + }, + { + url: 'https://api.stripe.com/v1/payment_methods', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://checkout.comfy.org/ajax/metrics_batch', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://testcloud.comfy.org/api/billing/payment-portal', + permission: 'allowPayments', + opposite: 'allowAccountCreation' + }, + { + url: 'https://api.stripe.com/v1/payment_pages/cs_test_example/confirm', + permission: 'allowPayments', + opposite: 'allowAccountCreation' } + ])('requires $permission for $url', ({ url, permission, opposite }) => { + const enabled = { ...config, [permission]: true } + expect(isLiveCloudMutationAllowed(new URL(url), 'POST', config)).toBe(false) expect(isLiveCloudMutationAllowed(new URL(url), 'POST', enabled)).toBe(true) + expect( + isLiveCloudMutationAllowed(new URL(url), 'POST', { + ...config, + [opposite]: true + }) + ).toBe(false) expect( isLiveCloudMutationAllowed(new URL(url), 'POST', { ...enabled, - PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org' + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org', + environment: getLiveCloudEnvironment('https://cloud.comfy.org') }) ).toBe(false) + expect(isLiveCloudMutationAllowed(new URL(url), 'PUT', enabled)).toBe(false) + const wrongOrigin = new URL(url) + wrongOrigin.hostname = 'untrusted.example.com' + expect(isLiveCloudMutationAllowed(wrongOrigin, 'POST', enabled)).toBe(false) expect( - isLiveCloudMutationAllowed( - new URL(url.replace('cs_test_', 'cs_live_')), - 'POST', - { ...enabled, allowPayments: false, allowAccountCreation: false } - ) + isLiveCloudMutationAllowed(new URL(`${url}/unexpected`), 'POST', enabled) ).toBe(false) }) }) + +describe('Live Cloud auxiliary services', () => { + it.for([ + { url: 'https://t.comfy.org/flags/?v=2', method: 'POST', allowed: true }, + { + url: 'https://www.google-analytics.com/g/collect', + method: 'POST', + allowed: true + }, + { url: 'https://mp.comfy.org/track/', method: 'POST', allowed: true }, + { url: 'https://cdp.customer.io/v1/i', method: 'POST', allowed: true }, + { + url: 'https://unknown.example.com/collect', + method: 'POST', + allowed: false + }, + { url: 'https://t.comfy.org/admin', method: 'POST', allowed: false }, + { url: 'https://t.comfy.org/flags/', method: 'DELETE', allowed: false }, + { url: 'http://t.comfy.org/flags/', method: 'POST', allowed: false } + ])('$method $url allowed=$allowed', ({ url, method, allowed }) => { + expect(isLiveCloudAuxiliaryPost(new URL(url), method)).toBe(allowed) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index 1ddcee00a41..02b4950f8d9 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -30,6 +30,39 @@ export const LIVE_CHECKOUT_ORIGINS = [ 'https://newassets.hcaptcha.com' ] +const AUXILIARY_POST_URLS = new Set([ + 'https://www.google-analytics.com/g/collect', + 'https://www.google.com/ccm/collect', + 'https://ad.doubleclick.net/ccm/s/collect', + 'https://www.google.com/g/collect', + 'https://www.google.com/rmkt/collect/17902754986/', + 'https://analytics.google.com/g/collect', + 'https://stats.g.doubleclick.net/g/collect', + 'https://t.comfy.org/flags/', + 'https://cdp.customer.io/v1/p', + 'https://mpc-prod-27-s6uit34pua-uk.a.run.app/events', + 'https://5z-2b6b7616f94640c2840d1841e1ac24c3.ecs.us-east-1.on.aws/events', + 'https://www.facebook.com/tr/', + 'https://www.google.com/pagead/form-data/17902754986', + 'https://www.google.com/ccm/form-data/17902754986', + 'https://google.com/pagead/form-data/17902754986', + 'https://google.com/ccm/form-data/17902754986', + 'https://px.ads.linkedin.com/wa/', + 'https://consumer.cloud.gist.build/api/v4/users', + 'https://cdp.customer.io/v1/i', + 'https://www.google.com/measurement/conversion', + 'https://e2.sy-d.io/events', + 'https://cdp.customer.io/v1/t', + 'https://mp.comfy.org/track/', + 'https://mp.comfy.org/engage/' +]) + +export function isLiveCloudAuxiliaryPost(url: URL, method: string): boolean { + return ( + method === 'POST' && AUXILIARY_POST_URLS.has(`${url.origin}${url.pathname}`) + ) +} + const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] export function getBlockedRequestViolation(url: URL, method: string): string { @@ -52,6 +85,7 @@ export function isLiveCloudMutationAllowed( LiveCloudBillingConfig, | 'PLAYWRIGHT_SETUP_API_URL' | 'customerOrigin' + | 'environment' | 'allowCheckout' | 'allowPayments' | 'allowAccountCreation' @@ -68,14 +102,14 @@ export function isLiveCloudMutationAllowed( } if ( config.allowAccountCreation && - config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' && + config.environment.stripeMode === 'test' && url.origin === 'https://identitytoolkit.googleapis.com' && url.pathname === '/v1/accounts:signUp' ) return true if ( config.allowPayments && - config.PLAYWRIGHT_SETUP_API_URL !== 'https://cloud.comfy.org' && + config.environment.stripeMode === 'test' && isPaymentPostAllowed(url, config.PLAYWRIGHT_SETUP_API_URL) ) return true @@ -89,7 +123,11 @@ export function isLiveCloudMutationAllowed( } if ( config.allowCheckout && - isCheckoutPostAllowed(url, config.PLAYWRIGHT_SETUP_API_URL) + isCheckoutPostAllowed( + url, + config.PLAYWRIGHT_SETUP_API_URL, + config.environment.stripeMode + ) ) return true if (url.pathname === '/customers') { @@ -132,7 +170,11 @@ function isPaymentPostAllowed(url: URL, backend: string): boolean { ) } -function isCheckoutPostAllowed(url: URL, backend: string): boolean { +function isCheckoutPostAllowed( + url: URL, + backend: string, + mode: 'live' | 'test' +): boolean { if (url.origin === backend) { return [ '/api/billing/preview-subscribe', @@ -140,7 +182,6 @@ function isCheckoutPostAllowed(url: URL, backend: string): boolean { ].includes(url.pathname) } if (url.origin === 'https://api.stripe.com') { - const mode = backend === 'https://cloud.comfy.org' ? 'live' : 'test' return new RegExp(`^/v1/payment_pages/cs_${mode}_[A-Za-z0-9]+/init$`).test( url.pathname ) diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index d22cb620ffb..99d02154f50 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -26,7 +26,14 @@ function collectLiveBilling(enabled: '0' | '1', args: string[]) { suites: z.array( z.object({ suites: z.array( - z.object({ specs: z.array(z.object({ file: z.string() })) }) + z.object({ + specs: z.array( + z.object({ + file: z.string(), + tests: z.array(z.object({ projectName: z.string() })) + }) + ) + }) ) }) ), @@ -37,12 +44,11 @@ function collectLiveBilling(enabled: '0' | '1', args: string[]) { }) }) .parse(JSON.parse(result.stdout)) - const files = report.suites.flatMap((suite) => - suite.suites.flatMap((child) => - child.specs.map((spec) => spec.file.replaceAll('\\', '/')) - ) + const specs = report.suites.flatMap((suite) => + suite.suites.flatMap((child) => child.specs) ) - return { status: result.status, config: report.config, files } + const files = specs.map((spec) => spec.file.replaceAll('\\', '/')) + return { status: result.status, config: report.config, files, specs } } describe('Live billing opt-in', () => { @@ -73,7 +79,7 @@ describe('Live billing opt-in', () => { 'collects only live billing when enabled $name', { timeout: 90_000 }, ({ args }) => { - const { status, config, files } = collectLiveBilling('1', args) + const { status, config, files, specs } = collectLiveBilling('1', args) expect(status).toBe(0) expect(config.projects.map((project) => project.name)).toEqual([ 'cloud-live', @@ -83,6 +89,15 @@ describe('Live billing opt-in', () => { expect(files.every((file) => file.startsWith('tests/liveCloud/'))).toBe( true ) + for (const spec of specs) { + const file = spec.file.replaceAll('\\', '/') + const project = file.startsWith('tests/liveCloud/disposable/') + ? 'cloud-live-disposable' + : file.startsWith('tests/liveCloud/paid/') + ? 'cloud-live-paid' + : 'cloud-live' + expect(spec.tests.map((test) => test.projectName)).toEqual([project]) + } const expectedFiles = globSync( 'browser_tests/tests/liveCloud/**/*.spec.ts' ).map((file) => file.replace('browser_tests/', '').replaceAll('\\', '/')) diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index 0a2016ce003..9c80a4724f1 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -13,6 +13,7 @@ import { liveCloudDisposableBillingFixture } from '@e2e/fixtures/liveCloudDisposableBillingFixture' import { LiveCloudOnboarding } from '@e2e/fixtures/components/LiveCloudOnboarding' +import { OnboardingCoachmarks } from '@e2e/fixtures/components/Tour' import { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' const test = liveCloudDisposableBillingFixture.extend<{ @@ -109,12 +110,13 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { ` }) ) - await new LiveCloudOnboarding(page).dismissTutorialsWhenVisible() + await new OnboardingCoachmarks(page).dismissWhenVisible() for (let visit = 0; visit < 2; visit++) { await page.goto('http://localhost:5173/') await page.getByRole('button', { name: 'Show tutorial' }).click() @@ -127,6 +129,60 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { }) } + test('completes disposable survey questions before waiting for the canvas', async ({ + page + }) => { + await test.step('Open a survey with option and text questions', async () => { + await page.route('http://localhost:5173/', (route) => + route.fulfill({ + contentType: 'text/html', + body: ` +

Let's get to know you

+ + + + + ` + }) + ) + await page.goto('http://localhost:5173/') + }) + + await test.step('Submit both answers and reach the canvas', async () => { + const [request] = await Promise.all([ + page.waitForRequest( + (request) => + request.url() === 'http://localhost:5173/api/settings' && + request.method() === 'POST' + ), + new LiveCloudOnboarding(page).completeSurveyIfNeeded( + 'http://localhost:5173' + ) + ]) + expect(request.postDataJSON()).toEqual({ + onboarding_survey: { + intent: 'exploring', + details: 'Automated billing E2E' + } + }) + await expect(page.locator('#graph-canvas')).toBeVisible() + }) + }) + test('rewrites browser auth to Cloud and provisions customers at the Comfy API', async ({ page, sandboxProxy @@ -169,30 +225,44 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { context, sandboxProxy }) => { - const page = await context.newPage() - await page.route('http://localhost:5173/', (route) => - route.fulfill({ contentType: 'text/html', body: '' }) - ) - await page.goto('http://localhost:5173/') - for (const url of [ - 'https://cloud.comfy.org/cdn-cgi/trace', - 'https://apis.google.com/js/api.js', - 'https://huggingface.co/example/resolve/main/model.safetensors', - 'https://t.comfy.org/flags/' - ]) { + const page = await test.step('Open the local frontend', async () => { + const page = await context.newPage() + await page.route('http://localhost:5173/', (route) => + route.fulfill({ contentType: 'text/html', body: '' }) + ) + await page.goto('http://localhost:5173/') + return page + }) + + await test.step('Verify auxiliary services use upstream responses', async () => { + for (const url of [ + 'https://cloud.comfy.org/cdn-cgi/trace', + 'https://apis.google.com/js/api.js', + 'https://huggingface.co/example/resolve/main/model.safetensors', + 'https://t.comfy.org/flags/' + ]) { + expect( + await page.evaluate(async (url) => { + const response = await fetch(url) + return await response.text() + }, url) + ).toBe(`GET ${url}`) + } expect( - await page.evaluate(async (url) => { - const response = await fetch(url) - return await response.text() - }, url) - ).toBe(`GET ${url}`) - } - expect(sandboxProxy.requests).toContain('GET https://t.comfy.org/flags/') + await page.evaluate(() => + fetch('https://t.comfy.org/flags/', { method: 'POST' }).then( + (response) => response.text() + ) + ) + ).toBe('POST https://t.comfy.org/flags/') + expect(sandboxProxy.requests).toContain('GET https://t.comfy.org/flags/') + }) }) for (const path of [ '/api/billing/subscribe', '/customers', + 'http://127.0.0.1:39281/collect', 'https://api.stripe.com/v1/payment_pages/cs_test_example/init' ]) { test(`blocks POST ${path} and fails even when the app catches it`, async ({ diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md index 9b6809bbcaf..656bc995cc4 100644 --- a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -21,7 +21,9 @@ sandbox credentials. Use a dedicated sandbox account and the public billing API. Establish the connection with a sign-in and billing-read smoke test. Checkout recovery -scenarios build on this harness in a separate change. +scenarios build on this harness in a separate change. Smoke remains read-only for +billing state; authentication, customer provisioning, installation-version, and +onboarding settings writes are permitted for startup. A separate JSON configuration and an external reset executable add unnecessary setup. Embedding database writes and Stripe/Temporal administration in the fixture diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 96bdcab10e3..3d247b86229 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -42,8 +42,8 @@ backend and derives its matching customer API and Stripe checkout mode. A deploy frontend must match the selected backend origin; local URLs exercise the PR code. Live tests do not install shared response mocks or override feature flags. -Sign-in completes the real onboarding survey when required and waits for the -canvas. Known tutorial dialogs are dismissed through their Skip button whenever +Permanent accounts must have completed onboarding when the survey flag is +enabled. Disposable setup completes the real survey and waits for the canvas. Known tutorial dialogs are dismissed through their Skip button whenever they appear during an interaction, including after navigation. Billing dialogs are left to the scenario. Assets, location detection, Google auth scripts, analytics, and feature flags use their real @@ -56,7 +56,9 @@ It attaches a post-login screenshot. The shared network boundary rejects Cloud a POST endpoints used for Firebase sign-in/token refresh, customer provisioning, Cloud auth tokens/session cookies, the startup write to `/api/settings/Comfy.InstalledVersion`, and onboarding survey/tutorial state. -Bulk settings writes may contain only `onboarding_survey`. +Bulk settings writes may contain only `onboarding_survey`. Observed analytics, +customer-messaging, and feature-flag POST endpoints have exact URL allowances; +unknown off-origin mutations are blocked and fail teardown. A rejected mutation fails the test even if the app catches the request error. The smoke fixture blocks checkout, payment, and reset mutations in both browser routing and Playwright API clients. @@ -78,7 +80,7 @@ production; Staging remains the default backend. Select `cloud-live-paid` for saved-card tests and `cloud-live-disposable` for new-account tests. These projects collect tests only when their respective scenario PRs are present. -Traces start after permanent-account sign-in and contain authenticated network -traffic. Keep reports private. Disposable accounts require no reusable credentials. +Automatic tracing stays disabled because authenticated traffic contains reusable +credentials. Use screenshots for evidence. Completed-payment assertions verify saved cards through the real hosted billing -portal, because Staging disables the embedded-checkout saved-card API. +portal, because sandbox environments may disable the embedded-checkout saved-card API. From bc83bc138514f6ff3462ce308fed77ed6259edfa Mon Sep 17 00:00:00 2001 From: ShihChi Huang Date: Wed, 16 Sep 2026 19:19:25 -0700 Subject: [PATCH 9/9] test: cover intermediate survey Next transition --- .../infrastructure/liveCloudBillingRouting.spec.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts index 9c80a4724f1..697356524ec 100644 --- a/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts +++ b/browser_tests/tests/infrastructure/liveCloudBillingRouting.spec.ts @@ -138,7 +138,11 @@ test.describe('Live Cloud network boundary', { tag: '@smoke' }, () => { contentType: 'text/html', body: `

Let's get to know you

- + +