diff --git a/browser_tests/README.md b/browser_tests/README.md index e3cbbcc57be..27f67636f98 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -22,6 +22,11 @@ to a section below. ## Prerequisites +For real Cloud billing checkout recovery, sandbox setup, and remaining release +criteria, see [Cloud billing E2E coverage](../docs/testing/cloud-billing-e2e.md). +Its dedicated runner requires real authentication and a dedicated no-card account; +the live suite is excluded from the ordinary browser test configurations. + **CRITICAL**: Start the ComfyUI backend with `--multi-user`: ```bash diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts new file mode 100644 index 00000000000..88c8879f5b8 --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -0,0 +1,710 @@ +import { + zBillingBalanceResponse, + zBillingOpStatusResponse, + zBillingStatusResponse, + zCancelSubscriptionAcceptedResponse, + zCancelSubscriptionResponse, + zCurrentWorkspaceResponse, + zListSavedPaymentMethodsResponse, + zPaymentPortalResponse, + zPreviewSubscribeResponse, + zResubscribeResponse, + zSubscribeResponse +} from '@comfyorg/ingest-types/zod' +import type { + APIRequestContext, + Page, + Response, + TestInfo +} from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' + +export class LiveCloudBillingSession { + constructor( + private readonly request: APIRequestContext, + private readonly backend: string, + private readonly headers: Record + ) {} + + async read(path: string, schema: z.ZodType): Promise { + const response = await this.request + .get(new URL(path, this.backend).href, { + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } + + async post( + path: string, + data: Record, + schema: z.ZodType, + acceptedStatuses = [200] + ): Promise { + const response = await this.request + .post(new URL(path, this.backend).href, { + data, + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(acceptedStatuses, `POST ${path}`).toContain(response.status()) + return schema.parse(await response.json()) + } + + async assertNoCardAccount(testInfo: TestInfo) { + const workspace = await this.read( + '/api/workspaces/current', + zCurrentWorkspaceResponse + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const status = await this.read( + '/api/billing/status', + zBillingStatusResponse + ) + const methods = await this.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + await testInfo.attach('billing-preflight.json', { + body: JSON.stringify({ + workspaceId: workspace.id, + isActive: status.is_active, + billingStatus: status.billing_status, + subscriptionTier: status.subscription_tier, + paymentMethodCount: methods.length, + backend: this.backend + }), + contentType: 'application/json' + }) + if (status.is_active) expect(status.subscription_tier).toBe('FREE') + else + expect([undefined, 'inactive', 'awaiting_payment_method']).toContain( + status.billing_status + ) + expect(methods).toHaveLength(0) + } + + async expectPending(operationId: string) { + await expect + .poll(() => + this.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + ) + .toMatchObject({ + id: operationId, + status: 'pending', + phase: 'awaiting_payment_method' + }) + } + + async expectSucceeded(operationId: string) { + await expect + .poll( + async () => { + const operation = await this.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return operation.status + }, + { timeout: 60_000 } + ) + .toBe('succeeded') + } + + async ensureProvisioned(returnUrl: string) { + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) + } +} + +export function matchesBillingResponse( + response: Response, + frontend: string, + path: string +) { + return response.url() === new URL(path, frontend).href +} + +export class LiveCloudCheckout { + private readonly page: Page + readonly confirmation + readonly resumePayment + private readonly subscribe + + constructor( + private readonly comfyPage: ComfyPage, + private readonly frontend: string + ) { + this.page = comfyPage.page + this.confirmation = this.page.getByRole('dialog').filter({ + has: this.page.getByRole('heading', { + name: 'Confirm your payment', + exact: true + }) + }) + this.resumePayment = this.confirmation.getByRole('button', { + name: 'Complete your payment', + exact: true + }) + this.subscribe = this.confirmation.getByRole('button', { + name: 'Subscribe to Creator', + exact: true + }) + } + + async open(retryTransientFailures = false) { + let preview: ReturnType | undefined + const loadPreview = async () => { + const [response] = await Promise.all([ + this.page.waitForResponse((response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/preview-subscribe' + ) + ), + this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) + ]) + expect(response.status()).toBe(200) + preview = zPreviewSubscribeResponse.parse(await response.json()) + } + if (retryTransientFailures) { + await expect(loadPreview).toPass({ timeout: 90_000 }) + } else { + await loadPreview() + } + if (!preview) throw new Error('Cloud billing preview was not loaded') + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await expect( + this.confirmation.getByText('Total due today', { exact: true }) + ).toBeVisible() + await expect(this.subscribe.or(this.resumePayment)).toBeEnabled() + return preview + } + + async abandonCheckout(testInfo: TestInfo, resume = false) { + const action = resume + ? this.resumePayment + : this.subscribe.or(this.resumePayment) + const { checkout, subscription } = await this.startCheckout(action) + await testInfo.attach('billing-operation.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + status: subscription.status + }), + contentType: 'application/json' + }) + await testInfo.attach('stripe-checkout.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await checkout.close() + return subscription.billing_op_id + } + + async completeCheckout(session: LiveCloudBillingSession, testInfo: TestInfo) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4242424242424242') + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + 'checkout-completed.png' + ) + } + + async completeAuthenticatedCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' + await expect + .poll(() => + checkout + .frames() + .some((frame) => frame.url().startsWith(challengeOrigin)) + ) + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const completeAuthentication = challenge.getByRole('button', { + name: /complete/i + }) + await expect(completeAuthentication).toBeVisible() + await testInfo.attach('3ds-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await completeAuthentication.click() + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + '3ds-completed.png' + ) + } + + async declineCheckout(session: LiveCloudBillingSession, testInfo: TestInfo) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + const observeFailure = async () => { + await expect( + this.page.getByText('Your bank declined this payment', { exact: false }) + ).toBeVisible() + await this.attachScreenshot('checkout-declined.png') + } + await Promise.all([ + observeFailure(), + this.submitCard(checkout, '4000000000000341') + ]) + await expect + .poll(async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status + } + }) + .toMatchObject({ status: 'failed' }) + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.subscription_tier).toBe('FREE') + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + await testInfo.attach('checkout-decline.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros, + status: 'failed', + failureKind: 'card_declined_ui' + }), + contentType: 'application/json' + }) + return { status: 'failed', failureKind: 'card_declined_ui' } + } + + async failAuthenticatedCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' + await expect + .poll(() => + checkout + .frames() + .some((frame) => frame.url().startsWith(challengeOrigin)) + ) + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const failAuthentication = challenge.getByRole('button', { name: /fail/i }) + await expect(failAuthentication).toBeVisible() + await testInfo.attach('3ds-failure-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await failAuthentication.click() + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + declineReason: operation.decline_reason + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ + status: 'failed', + authenticationState: 'failed_retryable' + }) + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.subscription_tier).toBe('FREE') + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + await testInfo.attach('3ds-failure.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros, + authenticationState: 'failed_retryable' + }), + contentType: 'application/json' + }) + return { authenticationState: 'failed_retryable' } + } + + async verifyCancellationRecovery( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const cancel = await session.post( + '/api/billing/subscription/cancel', + { idempotency_key: crypto.randomUUID() }, + z.union([ + zCancelSubscriptionResponse, + zCancelSubscriptionAcceptedResponse + ]), + [200, 202] + ) + await session.expectSucceeded(cancel.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const canceled = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(canceled.cancel_at).toBeTruthy() + const resubscribe = await session.post( + '/api/billing/subscription/resubscribe', + { idempotency_key: crypto.randomUUID() }, + zResubscribeResponse + ) + if (resubscribe.status === 'pending') { + await session.expectSucceeded(resubscribe.billing_op_id) + } + await expect + .poll(async () => { + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + return { + cancelAt: status.cancel_at, + planSlug: status.plan_slug, + subscriptionStatus: status.subscription_status + } + }) + .toMatchObject({ + cancelAt: undefined, + planSlug: 'creator-monthly', + subscriptionStatus: 'active' + }) + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + const result = { + cancelOperationId: cancel.billing_op_id, + reactivateOperationId: resubscribe.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros + } + await testInfo.attach('cancellation-recovery.json', { + body: JSON.stringify(result), + contentType: 'application/json' + }) + return result + } + + async verifyPlanTransitions( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(1) + const balanceBeforeUpgrade = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const upgradePreview = await session.post( + '/api/billing/preview-subscribe', + { plan_slug: 'pro-monthly' }, + zPreviewSubscribeResponse + ) + expect(upgradePreview).toMatchObject({ + allowed: true, + transition_type: 'upgrade' + }) + const upgradeRequest: Record = { + plan_slug: 'pro-monthly' + } + if (upgradePreview.quote_id) { + upgradeRequest.quote_id = upgradePreview.quote_id + } + if (upgradePreview.quote_version !== undefined) { + upgradeRequest.quote_version = upgradePreview.quote_version + } + if (upgradePreview.proration_at) { + upgradeRequest.proration_at = upgradePreview.proration_at + } + const upgrade = await session.post( + '/api/billing/subscribe', + upgradeRequest, + zSubscribeResponse + ) + await session.expectSucceeded(upgrade.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'pro-monthly', + subscription_tier: 'PRO' + }) + const expectedUpgradeBalance = + balanceBeforeUpgrade.amount_micros + + Number(upgradePreview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedUpgradeBalance) + const downgradePreview = await session.post( + '/api/billing/preview-subscribe', + { plan_slug: 'standard-monthly' }, + zPreviewSubscribeResponse + ) + expect(downgradePreview).toMatchObject({ + allowed: true, + transition_type: 'downgrade' + }) + const downgradeRequest: Record = { + plan_slug: 'standard-monthly' + } + if (downgradePreview.quote_id) { + downgradeRequest.quote_id = downgradePreview.quote_id + } + if (downgradePreview.quote_version !== undefined) { + downgradeRequest.quote_version = downgradePreview.quote_version + } + if (downgradePreview.proration_at) { + downgradeRequest.proration_at = downgradePreview.proration_at + } + const downgrade = await session.post( + '/api/billing/subscribe', + downgradeRequest, + zSubscribeResponse + ) + await session.expectSucceeded(downgrade.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'pro-monthly', + scheduled_change: { plan_slug: 'standard-monthly' } + }) + const balanceAfterDowngrade = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfterDowngrade.amount_micros).toBe(expectedUpgradeBalance) + const result = { + upgradeOperationId: upgrade.billing_op_id, + downgradeOperationId: downgrade.billing_op_id, + balanceBeforeUpgradeCents: balanceBeforeUpgrade.amount_micros, + balanceAfterDowngradeCents: balanceAfterDowngrade.amount_micros, + scheduledPlan: 'standard-monthly' + } + await testInfo.attach('plan-transitions.json', { + body: JSON.stringify(result), + contentType: 'application/json' + }) + return result + } + + private async submitCard(checkout: Page, cardNumber: string) { + await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) + await checkout.getByLabel('Expiration', { exact: true }).fill('1230') + await checkout + .getByRole('textbox', { + name: 'Credit or debit card CVC/CVV', + exact: true + }) + .fill('123') + await checkout + .getByPlaceholder('Full name on card', { exact: true }) + .fill('Comfy Billing Test') + await checkout + .getByRole('button', { name: 'Enter address manually', exact: true }) + .click() + await checkout.locator('#billingAddressLine1').fill('123 Test Street') + await checkout.getByLabel('City', { exact: true }).fill('San Francisco') + await checkout.getByLabel('State', { exact: true }).selectOption('CA') + await checkout.getByLabel('ZIP', { exact: true }).fill('94107') + await checkout.locator('#enableStripePass').uncheck() + await checkout.getByRole('button', { name: /^Save/ }).click() + } + + private async verifyCheckoutCompletion( + session: LiveCloudBillingSession, + testInfo: TestInfo, + balanceBeforeCents: number, + preview: ReturnType, + operationId: string, + screenshotName: string + ) { + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + phase: operation.phase + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ status: 'succeeded' }) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + is_active: true, + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const expectedBalance = + balanceBeforeCents + Number(preview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedBalance) + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(1) + await testInfo.attach('checkout-completion.json', { + body: JSON.stringify({ + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + balanceBeforeCents, + balanceAfterCents: expectedBalance, + paymentMethodCount: methods.length + }), + contentType: 'application/json' + }) + await this.page.goto(this.frontend) + await this.attachScreenshot(screenshotName) + return { + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + paymentMethodCount: methods.length + } + } + + private async startCheckout(action = this.subscribe.or(this.resumePayment)) { + await expect(action).toBeEnabled() + const [response, checkout] = await Promise.all([ + this.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/subscribe' + ) && response.request().method() === 'POST' + ), + this.page.waitForEvent('popup'), + action.click() + ]) + expect(response.status()).toBe(200) + const subscription = zSubscribeResponse.parse(await response.json()) + expect(subscription.status).toBe('needs_payment_method') + expect(subscription.billing_op_id).not.toBe('') + const testCheckoutUrl = + /^https:\/\/checkout\.(?:stripe\.com|comfy\.org)\/c\/pay\/cs_test_/ + expect(subscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(checkout).toHaveURL(testCheckoutUrl) + return { checkout, subscription } + } + + async attachScreenshot(name: string) { + await this.comfyPage.attachScreenshot(name, { runInCI: true }) + } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts new file mode 100644 index 00000000000..af892f0373d --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts @@ -0,0 +1,33 @@ +import { randomUUID } from 'node:crypto' + +import type { APIRequestContext } from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +const firebaseApiKey = 'AIzaSyDa_YMeyzV0SkVe92vBZ1tVikWBmOU5KVE' + +const signUpResponseSchema = z.object({ localId: z.string().min(1) }) + +export async function createDisposableCloudAccount(request: APIRequestContext) { + const runId = randomUUID() + const credentials = { + email: `billing-e2e-checkout-completion-${runId}@comfy.org`, + password: `BillingE2E-${randomUUID()}!` + } + const response = await request + .post( + `https://identitytoolkit.googleapis.com/v1/accounts:signUp?key=${firebaseApiKey}`, + { + data: { + ...credentials, + returnSecureToken: true + } + } + ) + .catch(() => { + throw new Error('Disposable Cloud account creation failed') + }) + expect(response.status(), 'Firebase account creation').toBe(200) + const account = signUpResponseSchema.parse(await response.json()) + return { ...credentials, userId: account.localId } +} diff --git a/browser_tests/fixtures/helpers/LiveCloudTopup.ts b/browser_tests/fixtures/helpers/LiveCloudTopup.ts new file mode 100644 index 00000000000..fbc0164eb4b --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudTopup.ts @@ -0,0 +1,168 @@ +import { + zBillingBalanceResponse, + zBillingOpStatusResponse, + zBillingStatusResponse, + zCreateTopupRequest, + zCreateTopupResponse, + zListSavedPaymentMethodsResponse +} from '@comfyorg/ingest-types/zod' +import { expect } from '@playwright/test' +import type { TestInfo } from '@playwright/test' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { matchesBillingResponse } from '@e2e/fixtures/helpers/LiveCloudBilling' + +export class LiveCloudTopup { + private readonly dialog + + constructor( + private readonly comfyPage: ComfyPage, + private readonly session: LiveCloudBillingSession, + private readonly frontend: string + ) { + this.dialog = comfyPage.page.getByRole('dialog') + } + + async open() { + const status = await this.session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.is_active).toBe(true) + expect(status.subscription_tier).toBe('CREATOR') + const methods = await this.session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).not.toHaveLength(0) + await this.comfyPage.page.goto(`${this.frontend}/?topup=1`) + await this.dialog.getByRole('button', { name: '$10', exact: true }).click() + await this.dialog + .getByRole('button', { name: 'Add credits', exact: true }) + .click() + await expect( + this.dialog.getByText( + 'Your saved payment method is charged immediately.', + { exact: true } + ) + ).toBeVisible() + } + + async verifyPurchase(testInfo: TestInfo) { + const before = await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const { topup } = await this.submitPurchase() + await this.attachPurchase(testInfo, topup, before.amount_micros) + await expect( + this.comfyPage.page.getByText('Credits added successfully', { + exact: true + }) + ).toBeVisible() + await this.expectPurchaseComplete( + topup.billing_op_id, + before.amount_micros + Number(topup.amount_cents) + ) + await this.comfyPage.attachScreenshot('saved-card-topup.png', { + runInCI: true + }) + } + + async verifyIdempotentRetry(testInfo: TestInfo) { + const before = await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const { request, topup } = await this.submitPurchase() + expect(request.idempotency_key).toBeTruthy() + if (!request.idempotency_key) { + throw new Error('Missing top-up idempotency key') + } + const retry = await this.session.post( + '/api/billing/topup', + { + amount_cents: Number(request.amount_cents), + idempotency_key: request.idempotency_key + }, + zCreateTopupResponse + ) + expect(retry.billing_op_id).toBe(topup.billing_op_id) + expect(retry.topup_id).toBe(topup.topup_id) + await this.attachPurchase(testInfo, topup, before.amount_micros, { + retryOperationId: retry.billing_op_id, + retryTopupId: retry.topup_id + }) + await this.expectPurchaseComplete( + topup.billing_op_id, + before.amount_micros + Number(topup.amount_cents) + ) + } + + private async submitPurchase() { + const [response] = await Promise.all([ + this.comfyPage.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/topup' + ) && response.request().method() === 'POST' + ), + this.dialog + .getByRole('button', { name: 'Pay $10.00', exact: true }) + .click() + ]) + expect(response.status()).toBe(200) + const request = zCreateTopupRequest.parse(response.request().postDataJSON()) + const topup = zCreateTopupResponse.parse(await response.json()) + expect(topup.amount_cents).toBe(1000n) + expect(topup.billing_op_id).not.toBe('') + return { request, topup } + } + + private async attachPurchase( + testInfo: TestInfo, + topup: ReturnType, + balanceBeforeCents: number, + extra: Record = {} + ) { + await testInfo.attach('topup.json', { + body: JSON.stringify({ + operationId: topup.billing_op_id, + topupId: topup.topup_id, + amountCents: Number(topup.amount_cents), + balanceBeforeCents, + ...extra + }), + contentType: 'application/json' + }) + } + + private async expectPurchaseComplete( + operationId: string, + expectedBalanceCents: number + ) { + await expect + .poll(async () => { + const operation = await this.session.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return { id: operation.id, status: operation.status } + }) + .toMatchObject({ id: operationId, status: 'succeeded' }) + await expect + .poll( + async () => + ( + await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + ).amount_micros + ) + .toBe(expectedBalanceCents) + } +} diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts new file mode 100644 index 00000000000..66ffd1aacc5 --- /dev/null +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -0,0 +1,127 @@ +import type { BrowserContext } from '@playwright/test' +import { expect } from '@playwright/test' + +import { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { + installContextNetworkIsolation, + networkIsolationFixture as base +} from '@e2e/fixtures/networkIsolationFixture' +import { + installLiveCloudBillingRouting, + signInToLiveCloud +} from '@e2e/fixtures/utils/liveCloudBillingContext' +import { + loadLiveCloudBillingConfig, + loadLiveCloudBillingEndpoints +} from '@e2e/fixtures/utils/liveCloudBillingConfig' + +interface FreshBillingSession { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout +} + +export const liveCloudBillingFixture = base.extend<{ + billingSession: LiveCloudBillingSession + comfyPage: ComfyPage + checkout: LiveCloudCheckout + freshBillingSession: () => Promise +}>({ + baseURL: process.env.PLAYWRIGHT_TEST_URL, + networkPolicy: async ({ baseURL }, use, testInfo) => { + const config = loadLiveCloudBillingEndpoints() + const origins = new Set([ + new URL(baseURL ?? config.PLAYWRIGHT_TEST_URL).origin, + config.PLAYWRIGHT_SETUP_API_URL, + ...(config.PLAYWRIGHT_SETUP_API_URL === 'https://testcloud.comfy.org' + ? ['https://testapi.comfy.org'] + : []), + 'https://identitytoolkit.googleapis.com', + 'https://securetoken.googleapis.com', + 'https://dreamboothy-dev.firebaseapp.com', + 'https://checkout.stripe.com', + 'https://checkout.comfy.org', + 'https://testmode-acs.stripe.com', + 'https://hooks.stripe.com', + 'https://api.stripe.com', + 'https://js.stripe.com', + 'https://m.stripe.network', + 'https://m.stripe.com', + 'https://r.stripe.com', + 'https://q.stripe.com', + 'https://b.stripecdn.com' + ]) + const unexpected = new Set() + await use({ origins, unexpected }) + const blocked = [...unexpected] + await testInfo.attach('blocked-egress.json', { + body: JSON.stringify(blocked), + contentType: 'application/json' + }) + expect( + blocked.filter((entry) => /^(API|Navigation) /.test(entry)), + 'Unexpected API or navigation destination' + ).toEqual([]) + }, + context: async ({ context, networkPolicy }, use) => { + await installLiveCloudBillingRouting(context, networkPolicy) + await use(context) + await context.unrouteAll({ behavior: 'ignoreErrors' }) + }, + billingSession: async ({ page }, use) => { + await use(await signInToLiveCloud(page)) + }, + freshBillingSession: async ( + { browser, contextOptions, networkPolicy }, + use + ) => { + const config = loadLiveCloudBillingConfig() + const contexts: BrowserContext[] = [] + try { + await use(async () => { + const context = await browser.newContext({ + ...contextOptions, + baseURL: config.PLAYWRIGHT_TEST_URL, + storageState: { cookies: [], origins: [] }, + serviceWorkers: 'block', + recordVideo: undefined, + recordHar: undefined + }) + contexts.push(context) + await installContextNetworkIsolation( + context, + networkPolicy, + config.PLAYWRIGHT_TEST_URL + ) + await installLiveCloudBillingRouting(context, networkPolicy) + const page = await context.newPage() + const billingSession = await signInToLiveCloud(page) + const comfyPage = new ComfyPage(page, context.request) + return { + billingSession, + checkout: new LiveCloudCheckout(comfyPage, config.PLAYWRIGHT_TEST_URL) + } + }) + } finally { + for (const context of contexts) { + await context.unrouteAll({ behavior: 'ignoreErrors' }) + await context.close() + } + } + }, + comfyPage: async ({ page, request }, use) => { + await use(new ComfyPage(page, request)) + }, + checkout: async ({ comfyPage, billingSession }, use, testInfo) => { + await billingSession.assertNoCardAccount(testInfo) + const checkout = new LiveCloudCheckout( + comfyPage, + loadLiveCloudBillingConfig().PLAYWRIGHT_TEST_URL + ) + await checkout.open() + await use(checkout) + } +}) + +export { expect as comfyExpect } from '@playwright/test' diff --git a/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts new file mode 100644 index 00000000000..74cf953f1df --- /dev/null +++ b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts @@ -0,0 +1,34 @@ +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudBilling' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { createDisposableCloudAccount } from '@e2e/fixtures/helpers/LiveCloudDisposableAccount' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' +import { loadLiveCloudBillingEndpoints } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +interface DisposableCheckout { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout +} + +export const liveCloudDisposableBillingFixture = + liveCloudBillingFixture.extend<{ + disposableCheckout: DisposableCheckout + }>({ + disposableCheckout: async ({ comfyPage, context }, use, testInfo) => { + const account = await createDisposableCloudAccount(context.request) + const billingSession = await signInToLiveCloud(comfyPage.page, account) + await billingSession.assertNoCardAccount(testInfo) + const frontend = loadLiveCloudBillingEndpoints().PLAYWRIGHT_TEST_URL + await billingSession.ensureProvisioned(frontend) + await testInfo.attach('disposable-account.json', { + body: JSON.stringify({ email: account.email, userId: account.userId }), + contentType: 'application/json' + }) + await use({ + billingSession, + checkout: new LiveCloudCheckout(comfyPage, frontend) + }) + } + }) + +export { expect } from '@playwright/test' diff --git a/browser_tests/fixtures/liveCloudPaidBillingFixture.ts b/browser_tests/fixtures/liveCloudPaidBillingFixture.ts new file mode 100644 index 00000000000..aabdb618d05 --- /dev/null +++ b/browser_tests/fixtures/liveCloudPaidBillingFixture.ts @@ -0,0 +1,17 @@ +import { LiveCloudTopup } from '@e2e/fixtures/helpers/LiveCloudTopup' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +export const liveCloudPaidBillingFixture = liveCloudBillingFixture.extend<{ + topup: LiveCloudTopup +}>({ + topup: async ({ comfyPage, billingSession }, use) => { + const topup = new LiveCloudTopup( + comfyPage, + billingSession, + loadLiveCloudBillingConfig().PLAYWRIGHT_TEST_URL + ) + await topup.open() + await use(topup) + } +}) diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index 36635ba123b..bee03306eef 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -1,4 +1,4 @@ -import type { APIRequestContext } from '@playwright/test' +import type { APIRequestContext, BrowserContext } from '@playwright/test' import { expect, test as base } from '@playwright/test' import { config as dotenvConfig } from 'dotenv' @@ -31,8 +31,76 @@ function guardApiRequests( } } +export interface NetworkPolicy { + origins: Set + unexpected: Set +} + +export async function installContextNetworkIsolation( + context: BrowserContext, + { origins, unexpected }: NetworkPolicy, + baseURL?: string +) { + const restore = guardApiRequests( + context.request, + origins, + unexpected, + baseURL + ) + await context.route('**/*', async (route) => { + const url = new URL(route.request().url()) + if (origins.has(url.origin)) { + await route.continue() + return + } + unexpected.add(`${route.request().method()} ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + }) + await context.routeWebSocket( + (url) => !origins.has(url.origin.replace(/^ws/, 'http')), + async (socket) => { + const url = new URL(socket.url()) + unexpected.add(`WebSocket ${url.origin}${url.pathname}`) + await socket.close() + } + ) + await context.route('https://huggingface.co/**/resolve/**', (route) => + route.request().method() === 'HEAD' + ? route.fulfill({ status: 200, body: '' }) + : route.fallback() + ) + await context.route( + 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', + (route) => + route.fulfill({ contentType: 'application/javascript', body: '' }) + ) + await context.route('https://apis.google.com/js/api.js**', (route) => + route.fulfill({ status: 503, body: '' }) + ) + await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => + route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) + ) + for (const slide of HERO_SLIDES) { + await context.route(slide.poster, (route) => + route.fulfill({ path: assetPath('image32x32.webp') }) + ) + await context.route(slide.src, (route) => + route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) + ) + } + await context.route( + 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', + (route) => route.fulfill({ status: 404, body: '' }) + ) + await context.route( + 'https://{api,stagingapi}.comfy.org/nodes{,?*}', + (route) => route.fulfill({ status: 404, body: '' }) + ) + return restore +} + export const networkIsolationFixture = base.extend<{ - networkPolicy: { origins: Set; unexpected: Set } + networkPolicy: NetworkPolicy }>({ serviceWorkers: 'block', networkPolicy: [ @@ -68,61 +136,11 @@ export const networkIsolationFixture = base.extend<{ { context, baseURL, networkPolicy: { origins, unexpected } }, use ) => { - const restore = guardApiRequests( - context.request, - origins, - unexpected, + const restore = await installContextNetworkIsolation( + context, + { origins, unexpected }, baseURL ) - await context.route('**/*', async (route) => { - const url = new URL(route.request().url()) - if (origins.has(url.origin)) { - await route.continue() - return - } - unexpected.add(`${route.request().method()} ${url.origin}${url.pathname}`) - await route.abort('blockedbyclient') - }) - await context.routeWebSocket( - (url) => !origins.has(url.origin.replace(/^ws/, 'http')), - async (socket) => { - const url = new URL(socket.url()) - unexpected.add(`WebSocket ${url.origin}${url.pathname}`) - await socket.close() - } - ) - await context.route('https://huggingface.co/**/resolve/**', (route) => - route.request().method() === 'HEAD' - ? route.fulfill({ status: 200, body: '' }) - : route.fallback() - ) - await context.route( - 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', - (route) => - route.fulfill({ contentType: 'application/javascript', body: '' }) - ) - await context.route('https://apis.google.com/js/api.js**', (route) => - route.fulfill({ status: 503, body: '' }) - ) - await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => - route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) - ) - for (const slide of HERO_SLIDES) { - await context.route(slide.poster, (route) => - route.fulfill({ path: assetPath('image32x32.webp') }) - ) - await context.route(slide.src, (route) => - route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) - ) - } - await context.route( - 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', - (route) => route.fulfill({ status: 404, body: '' }) - ) - await context.route( - 'https://{api,stagingapi}.comfy.org/nodes{,?*}', - (route) => route.fulfill({ status: 404, body: '' }) - ) await use(context) restore() diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts new file mode 100644 index 00000000000..f4f360dc08c --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + liveCloudBillingConfigSchema, + loadLiveCloudBillingConfig +} from '@e2e/fixtures/utils/liveCloudBillingConfig' + +const sandboxConfig = { + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', + CLOUD_ACCOUNT_EMAIL: 'billing-e2e@example.com', + CLOUD_ACCOUNT_PASSWORD: 'test-password' +} + +describe('Live Cloud billing prerequisites', () => { + it.for([ + 'https://cloud.comfy.org', + 'https://testcloud.comfy.org.example.com', + 'https://testcloud.comfy.org/redirect', + 'http://testcloud.comfy.org', + 'not-a-url' + ])('rejects an unsafe backend: %s', (PLAYWRIGHT_SETUP_API_URL) => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL + }).success + ).toBe(false) + }) + + it('rejects a production frontend with sandbox account credentials', () => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL: 'https://cloud.comfy.org' + }).success + ).toBe(false) + }) + + it('accepts a local frontend backed by test Cloud', () => { + expect(liveCloudBillingConfigSchema.safeParse(sandboxConfig).success).toBe( + true + ) + }) + + it('accepts an isolated preview deployment', () => { + const target = 'https://pr-8978.testenvs.comfy.org' + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL: target, + PLAYWRIGHT_SETUP_API_URL: target + }).success + ).toBe(true) + }) + + it('reports missing prerequisites without leaking credentials', () => { + vi.stubEnv('CLOUD_ACCOUNT_PASSWORD', 'private-value') + vi.stubEnv('CLOUD_ACCOUNT_EMAIL', undefined) + expect(loadLiveCloudBillingConfig).toThrow('CLOUD_ACCOUNT_EMAIL') + expect(loadLiveCloudBillingConfig).not.toThrow('private-value') + }) +}) + +describe('Frontend origins', () => { + it.for([ + 'http://localhost:5173/app', + 'http://localhost:5173/?x=1', + 'http://localhost:5173/#x', + 'http://user:pass@localhost:5173' + ])('rejects non-origin frontend %s', (PLAYWRIGHT_TEST_URL) => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL + }).success + ).toBe(false) + }) + + it('normalizes root trailing slashes on both origins', () => { + const config = liveCloudBillingConfigSchema.parse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL: 'http://localhost:5173/', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org/' + }) + expect(config.PLAYWRIGHT_TEST_URL).toBe('http://localhost:5173') + expect(config.PLAYWRIGHT_SETUP_API_URL).toBe('https://testcloud.comfy.org') + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts new file mode 100644 index 00000000000..76ee16679bd --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -0,0 +1,79 @@ +import { z } from 'zod' + +const sandboxURL = z + .string() + .url() + .refine((value) => { + if (!URL.canParse(value)) return false + const url = new URL(value) + return ( + url.protocol === 'https:' && + url.href === `${url.origin}/` && + (url.hostname === 'testcloud.comfy.org' || + url.hostname === 'stagingcloud.comfy.org' || + /^pr-\d+\.testenvs\.comfy\.org$/.test(url.hostname)) + ) + }, 'Use a Cloud test, staging, or PR preview origin') + .transform((value) => new URL(value).origin) + +const liveCloudBillingEndpointsSchema = z + .object({ + PLAYWRIGHT_TEST_URL: z + .string() + .url() + .refine((value) => { + if (!URL.canParse(value)) return false + const url = new URL(value) + return url.href === `${url.origin}/` + }, 'Use an origin without a path, query, credentials, or fragment') + .transform((value) => new URL(value).origin), + PLAYWRIGHT_SETUP_API_URL: sandboxURL + }) + .superRefine((config, ctx) => { + if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return + const frontend = new URL(config.PLAYWRIGHT_TEST_URL) + const local = + ['localhost', '127.0.0.1'].includes(frontend.hostname) && + ['http:', 'https:'].includes(frontend.protocol) + if ( + !local && + config.PLAYWRIGHT_TEST_URL !== config.PLAYWRIGHT_SETUP_API_URL + ) { + ctx.addIssue({ + code: 'custom', + path: ['PLAYWRIGHT_TEST_URL'], + message: 'Use localhost or the selected sandbox origin' + }) + } + }) + +export const liveCloudBillingConfigSchema = liveCloudBillingEndpointsSchema.and( + z.object({ + CLOUD_ACCOUNT_EMAIL: z.string().email(), + CLOUD_ACCOUNT_PASSWORD: z.string().min(1) + }) +) + +export function loadLiveCloudBillingEndpoints() { + const result = liveCloudBillingEndpointsSchema.safeParse(process.env) + if (!result.success) { + throw new Error( + `Cloud billing prerequisites: ${result.error.issues + .map((issue) => issue.path.join('.')) + .join(', ')}. See docs/testing/cloud-billing-e2e.md.` + ) + } + return result.data +} + +export function loadLiveCloudBillingConfig() { + const result = liveCloudBillingConfigSchema.safeParse(process.env) + if (!result.success) { + throw new Error( + `Cloud billing prerequisites: ${result.error.issues + .map((issue) => issue.path.join('.')) + .join(', ')}. See docs/testing/cloud-billing-e2e.md.` + ) + } + return result.data +} diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts new file mode 100644 index 00000000000..bbeaf1f92ec --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -0,0 +1,112 @@ +import type { BrowserContext, Page } from '@playwright/test' +import { expect } from '@playwright/test' + +import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { + LiveCloudBillingSession, + matchesBillingResponse +} from '@e2e/fixtures/helpers/LiveCloudBilling' +import type { NetworkPolicy } from '@e2e/fixtures/networkIsolationFixture' +import { + loadLiveCloudBillingConfig, + loadLiveCloudBillingEndpoints +} from '@e2e/fixtures/utils/liveCloudBillingConfig' + +interface LiveCloudCredentials { + email: string + password: string +} + +export async function installLiveCloudBillingRouting( + context: BrowserContext, + networkPolicy: NetworkPolicy +) { + const config = loadLiveCloudBillingEndpoints() + await context.route('**/*', async (route) => { + const request = route.request() + const url = new URL(request.url()) + if ( + !networkPolicy.origins.has(url.origin) && + url.hostname.endsWith('.comfy.org') && + /^\/(api|customers)(\/|$)/.test(url.pathname) + ) { + networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + request.isNavigationRequest() && + !networkPolicy.origins.has(url.origin) + ) { + networkPolicy.unexpected.add(`Navigation ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + url.origin === config.PLAYWRIGHT_TEST_URL && + /^\/(api|internal)(\/|$)/.test(url.pathname) + ) { + try { + const response = await route.fetch({ + url: new URL( + url.pathname + url.search, + config.PLAYWRIGHT_SETUP_API_URL + ).href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + } catch { + throw new Error(`Cloud proxy request failed: ${url.pathname}`) + } + return + } + await route.fallback() + }) +} + +export async function signInToLiveCloud( + page: Page, + credentials?: LiveCloudCredentials +) { + const endpoints = loadLiveCloudBillingEndpoints() + const account = + credentials ?? + (() => { + const config = loadLiveCloudBillingConfig() + return { + email: config.CLOUD_ACCOUNT_EMAIL, + password: config.CLOUD_ACCOUNT_PASSWORD + } + })() + await new FeatureFlagHelper(page).seedFlags({ + onboarding_survey_enabled: false + }) + await page.goto(`${endpoints.PLAYWRIGHT_TEST_URL}/cloud/login`) + await page + .getByRole('button', { name: 'Use email instead', exact: true }) + .click() + await page + .getByRole('textbox', { name: 'Email', exact: true }) + .fill(account.email) + await page.getByLabel('Password', { exact: true }).fill(account.password) + const [response] = await Promise.all([ + page.waitForResponse( + (response) => + matchesBillingResponse( + response, + endpoints.PLAYWRIGHT_TEST_URL, + '/api/billing/status' + ) && response.status() === 200 + ), + page.getByRole('button', { name: 'Sign in', exact: true }).click() + ]) + await page.goto(endpoints.PLAYWRIGHT_TEST_URL) + const authorization = await response.request().headerValue('authorization') + expect(authorization).toBeTruthy() + if (!authorization) throw new Error('Missing billing authorization') + return new LiveCloudBillingSession( + page.request, + endpoints.PLAYWRIGHT_SETUP_API_URL, + { authorization } + ) +} diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts new file mode 100644 index 00000000000..8ac20230f29 --- /dev/null +++ b/browser_tests/playwrightSelection.test.ts @@ -0,0 +1,63 @@ +import { spawnSync } from 'node:child_process' + +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +describe('Live billing opt-in', () => { + it.for([ + { enabled: '0', project: 'cloud-live', count: 0 }, + { enabled: '1', project: 'cloud-live', count: 3 }, + { enabled: '1', project: 'cloud-live-paid', count: 2 }, + { enabled: '1', project: 'cloud-live-disposable', count: 6 } + ])( + 'collects $project only when explicitly enabled ($enabled)', + { timeout: 90_000 }, + ({ enabled, project, count }) => { + const result = spawnSync( + 'pnpm', + [ + 'exec', + 'playwright', + 'test', + ...(enabled === '1' ? [`--project=${project}`] : ['tests/liveCloud']), + '--list', + '--reporter=json' + ], + { + cwd: process.cwd(), + env: { + ...process.env, + PLAYWRIGHT_CLOUD_LIVE: enabled, + DISTRIBUTION: 'cloud' + }, + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 10_000_000 + } + ) + expect(result.error).toBeUndefined() + expect(result.status).toBe(enabled === '1' ? 0 : 1) + const report = z + .object({ + suites: z.array( + z.object({ + suites: z.array(z.object({ specs: z.array(z.unknown()) })) + }) + ), + config: z.object({ + globalSetup: z.unknown(), + globalTeardown: z.unknown() + }) + }) + .parse(JSON.parse(result.stdout)) + const specs = report.suites.flatMap((suite) => + suite.suites.flatMap((child) => child.specs) + ) + expect(specs).toHaveLength(count) + if (enabled === '1') { + expect(report.config.globalSetup).toBeNull() + expect(report.config.globalTeardown).toBeNull() + } + } + ) +}) diff --git a/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts b/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts new file mode 100644 index 00000000000..36aca0c452c --- /dev/null +++ b/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts @@ -0,0 +1,70 @@ +import { + comfyExpect as expect, + liveCloudBillingFixture as test +} from '@e2e/fixtures/liveCloudBillingFixture' + +test.describe( + 'Real Cloud checkout persistence', + { tag: ['@cloud-live'] }, + () => { + test('resumes the same pending checkout after reloading the page', async ({ + comfyPage, + checkout, + billingSession + }, testInfo) => { + const operationId = + await test.step('Leave the checkout pending', async () => { + const operationId = await checkout.abandonCheckout(testInfo) + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + return operationId + }) + + await test.step('Reload and reopen the checkout', async () => { + await comfyPage.page.reload() + await checkout.open() + await expect(checkout.resumePayment).toBeEnabled() + await checkout.attachScreenshot('checkout-after-reload.png') + }) + + await test.step('Resume the same checkout after reload', async () => { + const resumedId = await checkout.abandonCheckout(testInfo, true) + expect(resumedId).toBe(operationId) + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + }) + }) + + test('reuses the pending operation after signing in from a fresh browser context', async ({ + comfyPage, + checkout, + billingSession, + freshBillingSession + }, testInfo) => { + const operationId = + await test.step('Leave the checkout pending', async () => { + const operationId = await checkout.abandonCheckout(testInfo) + await billingSession.expectPending(operationId) + return operationId + }) + + const fresh = + await test.step('Sign in from a fresh browser context', async () => { + await comfyPage.page.close() + const fresh = await freshBillingSession() + await fresh.billingSession.assertNoCardAccount(testInfo) + await fresh.billingSession.expectPending(operationId) + await fresh.checkout.open() + return fresh + }) + + await test.step('Resume the same checkout in the fresh context', async () => { + const resumedId = await fresh.checkout.abandonCheckout(testInfo) + expect(resumedId).toBe(operationId) + await fresh.billingSession.expectPending(operationId) + await expect(fresh.checkout.resumePayment).toBeEnabled() + await fresh.checkout.attachScreenshot('checkout-in-fresh-context.png') + }) + }) + } +) diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts new file mode 100644 index 00000000000..4d0eaa5d8b3 --- /dev/null +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -0,0 +1,30 @@ +import { + comfyExpect as expect, + liveCloudBillingFixture as test +} from '@e2e/fixtures/liveCloudBillingFixture' + +test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { + test('resumes an abandoned no-card checkout using the same billing operation', async ({ + checkout, + billingSession + }, testInfo) => { + const operationId = + await test.step('Leave the checkout pending', async () => { + await checkout.attachScreenshot('preview.png') + return await checkout.abandonCheckout(testInfo) + }) + + await test.step('See the pending checkout recovery action', async () => { + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await checkout.attachScreenshot('resume-payment.png') + }) + + await test.step('Retry the same checkout', async () => { + const resumedId = await checkout.abandonCheckout(testInfo, true) + expect(resumedId).toBe(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await billingSession.expectPending(operationId) + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts b/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts new file mode 100644 index 00000000000..d7e16065876 --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts @@ -0,0 +1,27 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe( + 'Real Cloud subscription lifecycle', + { tag: ['@cloud-live'] }, + () => { + test.setTimeout(240_000) + + test('cancels and reactivates without changing the balance', async ({ + disposableCheckout + }, testInfo) => { + await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + const recovery = + await disposableCheckout.checkout.verifyCancellationRecovery( + disposableCheckout.billingSession, + testInfo + ) + expect(recovery.balanceAfterCents).toBe(recovery.balanceBeforeCents) + }) + } +) diff --git a/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts b/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts new file mode 100644 index 00000000000..0a0c2ba270b --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts @@ -0,0 +1,19 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud checkout failure', { tag: ['@cloud-live'] }, () => { + test('keeps the account on Free after a card decline', async ({ + disposableCheckout + }, testInfo) => { + const failure = await disposableCheckout.checkout.declineCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(failure).toMatchObject({ + status: 'failed', + failureKind: 'card_declined_ui' + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts b/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts new file mode 100644 index 00000000000..df4ea4e3d5e --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts @@ -0,0 +1,23 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe( + 'Real Cloud checkout completion', + { tag: ['@cloud-live'] }, + () => { + test('activates Creator and grants the quoted credits', async ({ + disposableCheckout + }, testInfo) => { + const completion = await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(completion).toMatchObject({ + planSlug: 'creator-monthly', + paymentMethodCount: 1 + }) + }) + } +) diff --git a/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts b/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts new file mode 100644 index 00000000000..664311a5abd --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts @@ -0,0 +1,22 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud plan transitions', { tag: ['@cloud-live'] }, () => { + test.setTimeout(240_000) + + test('upgrades now and schedules a downgrade', async ({ + disposableCheckout + }, testInfo) => { + await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + const transitions = await disposableCheckout.checkout.verifyPlanTransitions( + disposableCheckout.billingSession, + testInfo + ) + expect(transitions.scheduledPlan).toBe('standard-monthly') + }) +}) diff --git a/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts new file mode 100644 index 00000000000..1515e83430b --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts @@ -0,0 +1,22 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('activates Creator after successful authentication', async ({ + disposableCheckout + }, testInfo) => { + const completion = + await disposableCheckout.checkout.completeAuthenticatedCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(completion).toMatchObject({ + planSlug: 'creator-monthly', + paymentMethodCount: 1 + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts new file mode 100644 index 00000000000..fd82066e16d --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts @@ -0,0 +1,20 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('keeps the account on Free after failed authentication', async ({ + disposableCheckout + }, testInfo) => { + const failure = await disposableCheckout.checkout.failAuthenticatedCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(failure).toMatchObject({ + authenticationState: 'failed_retryable' + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts b/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts new file mode 100644 index 00000000000..e582df49d07 --- /dev/null +++ b/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts @@ -0,0 +1,9 @@ +import { liveCloudPaidBillingFixture as test } from '@e2e/fixtures/liveCloudPaidBillingFixture' + +test.describe('Real Cloud saved-card billing', { tag: ['@cloud-live'] }, () => { + test('purchases credits with the saved test card', async ({ + topup + }, testInfo) => { + await topup.verifyPurchase(testInfo) + }) +}) diff --git a/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts b/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts new file mode 100644 index 00000000000..ae09f759943 --- /dev/null +++ b/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts @@ -0,0 +1,9 @@ +import { liveCloudPaidBillingFixture as test } from '@e2e/fixtures/liveCloudPaidBillingFixture' + +test.describe('Real Cloud saved-card billing', { tag: ['@cloud-live'] }, () => { + test('retries one top-up without granting credits twice', async ({ + topup + }, testInfo) => { + await topup.verifyIdempotentRetry(testInfo) + }) +}) diff --git a/docs/adr/README.md b/docs/adr/README.md index 7fc0f7c491f..98c5d9d70e1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -43,6 +43,7 @@ An Architecture Decision Record captures an important architectural decision mad | [TELEMETRY-DIAGNOSTICS-0019](TELEMETRY-DIAGNOSTICS-0019-recoverable-event-diagnostics.md) | Recoverable Event Diagnostics | Proposed | 2026-08-25 | | [TELEMETRY-ROUTING-0013](TELEMETRY-ROUTING-0013-telemetry-routing-across-consumers.md) | Telemetry Routing Across Consumers | Accepted | 2026-07-28 | | [TELEMETRY-STARTUP-0030](TELEMETRY-STARTUP-0030-production-startup-telemetry.md) | Production Startup Telemetry | Proposed | 2026-09-08 | +| [TESTING-BILLING-0031](TESTING-BILLING-0031-real-cloud-checkout-tests.md) | Real Cloud Checkout Tests | Proposed | 2026-09-11 | | [TESTING-E2E-0027](TESTING-E2E-0027-coverage-measurement-and-test-selection.md) | E2E Coverage Measurement and Test Selection | Proposed | 2026-09-01 | | [WEBSITE-REQUESTS-0030](WEBSITE-REQUESTS-0030-creator-widgets-and-router-request-composition.md) | Creator widgets and Router request composition | Deprecated | 2026-09-09 | | [WIDGET-SERIALIZATION-0006](WIDGET-SERIALIZATION-0006-preserve-primitive-widget-values-across-copy-and-paste.md) | PrimitiveNode Widget Restoration Lifecycle | Accepted | 2026-02-22 | diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md new file mode 100644 index 00000000000..0106c83b813 --- /dev/null +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -0,0 +1,49 @@ +# ADR-TESTING-BILLING-0031: Real Cloud Checkout Tests + +Date: 2026-09-11 + +## Status + +Proposed + +## Context + +Mocked responses cannot detect disagreement between the frontend, Cloud billing, +and Stripe. The first regression test must exercise the browser against a real +sandbox without requiring backend administrator credentials. + +## Decision + +Add an explicitly enabled `cloud-live` project to the existing Playwright runner. +Reuse `.env`, frontend/backend URL conventions, and ComfyPage. Sign in through the +real UI and capture post-login screenshots. Keep normal browser CI independent of +sandbox credentials. + +Use a dedicated no-card account and the public billing API. Close the unpaid +Stripe checkout, resume payment, and assert that the same billing operation is +reused. Record the pending operation instead of resetting backend state. + +A separate JSON configuration and an external reset executable add unnecessary +setup. Embedding database writes and Stripe/Temporal administration in the fixture +also makes browser coverage depend on privileged access and backend internals. +Those approaches are rejected for this test. Backend reset and expiry verification +can be added separately when required for repeatable CI. + +## Consequences + +### Positive + +- The browser test requires only sandbox URLs and account credentials. +- No payment is submitted and no backend billing state is reset. + +### Negative + +- Unpaid operations can remain pending after a run. Account reuse depends on the + backend's ability to resume that state, and concurrent runs must be avoided. +- Cleanup is not a substitute for an assertion about product recovery. +- Payment completion, reload recovery, timeout behavior, and a release gate remain + follow-up work. + +## Notes + +See the [run instructions](../testing/cloud-billing-e2e.md). diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md new file mode 100644 index 00000000000..0e2cccf1ed2 --- /dev/null +++ b/docs/testing/cloud-billing-e2e.md @@ -0,0 +1,72 @@ +# Real Cloud checkout recovery + +Run the opt-in `cloud-live` project through the existing browser-test runner: + +```sh +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 --headed +pnpm exec playwright show-report +``` + +Set these variables using the existing `.env` convention or inject them with +`op run --env-file -- zsh -f`: + +| Variable | Value | +| -------------------------- | ---------------------------------------------------- | +| `PLAYWRIGHT_TEST_URL` | Local frontend URL or deployed sandbox origin | +| `PLAYWRIGHT_SETUP_API_URL` | Matching test, staging, or PR sandbox backend origin | +| `CLOUD_ACCOUNT_EMAIL` | Sandbox account email | +| `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | + +For repeated local runs, open one credential shell and run all test commands +inside it. This reuses credentials already loaded by 1Password instead of +requesting them again for each command: + +```sh +op run --env-file -- zsh -f +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 +``` + +Keep the shell open for the session. Exiting it ends credential reuse. The env +file contains 1Password references, not passwords. + +For a local frontend, start `pnpm dev:cloud` and use its Vite URL with +`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The test routes browser API requests and its API checks directly to the selected +sandbox, independently of the local Vite proxy. Production targets are rejected. + +The fixture disables `onboarding_survey_enabled` through the existing dev-only +feature-flag helper before navigation. Deployed builds ignore this override, so +accounts used against a deployed frontend must have completed the survey. + +The account must own a personal workspace with no active paid subscription or saved +payment method. Use an account reserved for this test. Do not run concurrent +billing tests against that account. + +The test signs in, checks the real preview and payment dialog are ready, opens +Stripe test checkout, closes it without entering a card, and resumes the same +pending billing operation. The report attaches screenshots and the operation ID. +Authentication traces and saved credentials are not retained. + +No database, Stripe secret key, or Temporal access is required. Closing the +browser leaves the unpaid checkout pending in the backend. The test does not +reset billing state or claim that the operation expired. The first action accepts either a fresh Subscribe button or the pending +Complete your payment button, so sequential runs can resume an existing checkout. Backend expiry and fixture reset remain separate work. + +The no-card tests cover checkout handoff, retry, page reload, and signing in +from an empty browser context. Payment completion, the 24-hour timeout, and +CI integration remain follow-up work. A +successful sign-in alone does not establish billing coverage. + +## Saved-card billing + +Run the separate `cloud-live-paid` project with: + +```sh +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live-paid --workers=1 +``` + +Inject the saved-card account credentials into `CLOUD_ACCOUNT_EMAIL` +and `CLOUD_ACCOUNT_PASSWORD`. This project requires an active Creator plan and +at least one saved sandbox payment method. It purchases $10 of test credits and +checks the completed billing operation, exact balance increase, and success UI. +The balance API returns cents despite its `amount_micros` field name. Never +point this command at the no-card account. diff --git a/playwright.config.ts b/playwright.config.ts index f7675fc06a2..10492d68be2 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -42,6 +42,7 @@ const maybeLocalOptions: PlaywrightTestConfig = process.env.PLAYWRIGHT_LOCAL export default defineConfig({ testDir: './browser_tests', testIgnore: [ + '**/liveCloud/**', '**/*.test.ts', // Untransformed recorder output — still bare codegen, not a runnable spec '**/*.raw.spec.ts', @@ -56,10 +57,46 @@ export default defineConfig({ reporter: process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR ? 'blob' : 'html', ...maybeLocalOptions, - globalSetup: './browser_tests/globalSetup.ts', - globalTeardown: './browser_tests/globalTeardown.ts', + globalSetup: + process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? undefined + : './browser_tests/globalSetup.ts', + globalTeardown: + process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? undefined + : './browser_tests/globalTeardown.ts', projects: [ + ...(process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? [ + { name: 'cloud-live', testMatch: '**/tests/liveCloud/*.spec.ts' }, + { + name: 'cloud-live-paid', + testMatch: '**/tests/liveCloud/paid/*.spec.ts' + }, + { + name: 'cloud-live-disposable', + testMatch: '**/tests/liveCloud/disposable/*.spec.ts' + } + ].map( + (project) => + ({ + ...project, + testIgnore: [], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + }) satisfies NonNullable[number] + ) + : []), { name: 'chromium', use: { ...devices['Desktop Chrome'] }, diff --git a/src/composables/billing/types.ts b/src/composables/billing/types.ts index 84719722b10..31a7da5cce5 100644 --- a/src/composables/billing/types.ts +++ b/src/composables/billing/types.ts @@ -77,7 +77,10 @@ export interface BillingActions { * Pass-through by design: the caller owns the completed/pending follow-up * (balance refresh or billing-op polling), so this does not refresh. */ - topup: (amountCents: number) => Promise + topup: ( + amountCents: number, + idempotencyKey?: string + ) => Promise fetchPlans: () => Promise /** * Ensures billing is initialized and subscription is active. diff --git a/src/composables/billing/useBillingContext.test.ts b/src/composables/billing/useBillingContext.test.ts index eff32f5f06b..91426ed1b55 100644 --- a/src/composables/billing/useBillingContext.test.ts +++ b/src/composables/billing/useBillingContext.test.ts @@ -352,9 +352,12 @@ describe('useBillingContext', () => { vi.clearAllMocks() expect(context.type.value).toBe('workspace') - await context.topup(500) + await context.topup(500, 'topup-attempt-1') - expect(workspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(workspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(mockPurchaseCredits).not.toHaveBeenCalled() }) diff --git a/src/composables/billing/useBillingContext.ts b/src/composables/billing/useBillingContext.ts index 4d25ba306ff..ccb9b29b0ea 100644 --- a/src/composables/billing/useBillingContext.ts +++ b/src/composables/billing/useBillingContext.ts @@ -315,7 +315,7 @@ function useBillingContextInternal(): BillingContext { return activeContext.value.resubscribe(options) } - async function topup(amountCents: number) { + async function topup(amountCents: number, idempotencyKey?: string) { if ( !Number.isInteger(amountCents) || amountCents <= 0 || @@ -325,7 +325,7 @@ function useBillingContextInternal(): BillingContext { 'Top-up amount must be a positive whole-dollar cent value' ) } - return activeContext.value.topup(amountCents) + return activeContext.value.topup(amountCents, idempotencyKey) } async function fetchPlans() { diff --git a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts index 96d4058fdd6..67f1e2c0241 100644 --- a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts +++ b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts @@ -19,12 +19,20 @@ const mockFetchBalance = vi.fn() const mockFetchStatus = vi.fn() const mockManageSubscription = vi.fn<() => Promise>() const mockReportError = vi.hoisted(() => vi.fn()) +const mockCreateUuidv4 = vi.hoisted(() => vi.fn()) + +vi.mock(import('@/utils/uuid'), () => ({ createUuidv4: mockCreateUuidv4 })) vi.mock(import('@/platform/telemetry/reportError'), () => ({ reportError: mockReportError })) const mockTopup = - vi.fn<(amountCents: number) => Promise>() + vi.fn< + ( + amountCents: number, + idempotencyKey: string + ) => Promise + >() const mockShowSettings = vi.fn() const mockToastAdd = vi.fn() @@ -60,7 +68,8 @@ vi.mock(import('@/composables/billing/useBillingContext'), () => ({ fetchBalance: mockFetchBalance, fetchStatus: mockFetchStatus, manageSubscription: mockManageSubscription, - topup: (amountCents: number) => mockTopup(amountCents) + topup: (amountCents: number, idempotencyKey: string) => + mockTopup(amountCents, idempotencyKey) }) })) @@ -177,6 +186,7 @@ async function clickAddCredits() { } beforeEach(() => { + mockCreateUuidv4.mockReturnValue('topup-attempt-1') vi.mocked(useDialogStore().closeDialog).mockImplementation(() => {}) }) @@ -747,6 +757,43 @@ describe('TopUpCreditsDialogContentWorkspace', () => { ) }) + it('reuses the idempotency key after an uncertain request failure', async () => { + mockTopup + .mockRejectedValueOnce(new Error('connection lost')) + .mockResolvedValueOnce(topupResponse('pending')) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + + renderDialog() + await clickAddCredits() + const payButton = screen.getByRole('button', { name: 'Pay $50.00' }) + await userEvent.click(payButton) + await waitFor(() => expect(payButton).toBeEnabled()) + await userEvent.click(payButton) + + expect(mockTopup).toHaveBeenNthCalledWith(1, 5000, 'topup-attempt-1') + expect(mockTopup).toHaveBeenNthCalledWith(2, 5000, 'topup-attempt-1') + consoleError.mockRestore() + }) + + it('uses a new idempotency key after a definitive failure', async () => { + mockCreateUuidv4 + .mockReturnValueOnce('topup-attempt-1') + .mockReturnValueOnce('topup-attempt-2') + mockTopup + .mockResolvedValueOnce(topupResponse('failed')) + .mockResolvedValueOnce(topupResponse('pending')) + + renderDialog() + await clickAddCredits() + const payButton = screen.getByRole('button', { name: 'Pay $50.00' }) + await userEvent.click(payButton) + await waitFor(() => expect(payButton).toBeEnabled()) + await userEvent.click(payButton) + + expect(mockTopup).toHaveBeenNthCalledWith(1, 5000, 'topup-attempt-1') + expect(mockTopup).toHaveBeenNthCalledWith(2, 5000, 'topup-attempt-2') + }) + it('does not top up after the server capability is revoked', async () => { renderDialog() await clickAddCredits() diff --git a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue index ef3a8aa83fd..f53ba22a6e1 100644 --- a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue +++ b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue @@ -305,6 +305,7 @@ import { useBillingCapabilities } from '@/platform/workspace/composables/useBill import { useHasSavedPaymentMethod } from '@/platform/workspace/composables/useHasSavedPaymentMethod' import { useBillingOperationStore } from '@/platform/workspace/stores/billingOperationStore' import { useDialogStore } from '@/stores/dialogStore' +import { createUuidv4 } from '@/utils/uuid' import { cn } from '@comfyorg/tailwind-utils' const { isInsufficientCredits = false } = defineProps<{ @@ -356,6 +357,7 @@ const payAmount = ref(50) const showCeilingWarning = ref(false) const loading = ref(false) const paymentSubmitted = ref(false) +let topupIdempotencyKey: string | undefined const step = ref<'amount' | 'confirm' | 'verifying'>( topupOperation.value && canTopUp.value ? 'verifying' : 'amount' ) @@ -378,6 +380,7 @@ const creditsModel = computed({ set: (newCredits: number) => { payAmount.value = Math.round(creditsToUsd(newCredits)) selectedPreset.value = null + topupIdempotencyKey = undefined } }) @@ -439,12 +442,14 @@ function handlePayAmountChange(value: number) { payAmount.value = value selectedPreset.value = null showCeilingWarning.value = false + topupIdempotencyKey = undefined } function handlePresetClick(amount: number) { showCeilingWarning.value = false payAmount.value = amount selectedPreset.value = amount + topupIdempotencyKey = undefined } function handlePrimaryAction() { @@ -481,6 +486,7 @@ function startOverTopup() { const operation = topupOperation.value if (operation) billingOperationStore.dismissOperation(operation.opId) paymentSubmitted.value = false + topupIdempotencyKey = undefined step.value = 'amount' } @@ -514,7 +520,8 @@ async function handleBuy() { }) const amountCents = payAmount.value * 100 - const response = await topup(amountCents) + topupIdempotencyKey ??= createUuidv4() + const response = await topup(amountCents, topupIdempotencyKey) if (!response) { paymentSubmitted.value = false telemetry?.trackBillingEvent({ @@ -574,6 +581,7 @@ async function handleBuy() { } else { // Synchronous 'failed' here means the charge was declined, not rejected pre-attempt. paymentSubmitted.value = false + topupIdempotencyKey = undefined telemetry?.trackBillingEvent({ operation: 'topup', stage: 'failed', diff --git a/src/platform/workspace/composables/useWorkspaceBilling.test.ts b/src/platform/workspace/composables/useWorkspaceBilling.test.ts index a0789802746..22c64bae06c 100644 --- a/src/platform/workspace/composables/useWorkspaceBilling.test.ts +++ b/src/platform/workspace/composables/useWorkspaceBilling.test.ts @@ -1547,9 +1547,12 @@ describe('useWorkspaceBilling', () => { mockWorkspaceApi.createTopup.mockResolvedValue(topupResponse) const billing = setupBilling() - const result = await billing.topup(500) + const result = await billing.topup(500, 'topup-attempt-1') - expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(result).toBe(topupResponse) expect(mockWorkspaceApi.getBillingStatus).not.toHaveBeenCalled() expect(mockWorkspaceApi.getBillingBalance).not.toHaveBeenCalled() @@ -1634,9 +1637,12 @@ describe('useWorkspaceBilling', () => { mockWorkspaceApi.createTopup.mockResolvedValue(topupResponse) const billing = setupBilling() - const result = await billing.topup(500) + const result = await billing.topup(500, 'topup-attempt-1') - expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(result).toBe(topupResponse) expect(mockWorkspaceApi.getBillingStatus).not.toHaveBeenCalled() expect(mockWorkspaceApi.getBillingBalance).not.toHaveBeenCalled() diff --git a/src/platform/workspace/composables/useWorkspaceBilling.ts b/src/platform/workspace/composables/useWorkspaceBilling.ts index 1f4bdf07235..4c07ede821c 100644 --- a/src/platform/workspace/composables/useWorkspaceBilling.ts +++ b/src/platform/workspace/composables/useWorkspaceBilling.ts @@ -535,11 +535,14 @@ export function useWorkspaceBilling(): BillingState & BillingActions { } } - async function topup(amountCents: number): Promise { + async function topup( + amountCents: number, + idempotencyKey?: string + ): Promise { isLoading.value = true error.value = null try { - return await workspaceApi.createTopup(amountCents) + return await workspaceApi.createTopup(amountCents, idempotencyKey) } catch (err) { error.value = err instanceof Error ? err.message : 'Failed to top up credits' diff --git a/src/platform/workspace/stores/billingOperationStore.test.ts b/src/platform/workspace/stores/billingOperationStore.test.ts index cb8c5db563b..d9e56410a9c 100644 --- a/src/platform/workspace/stores/billingOperationStore.test.ts +++ b/src/platform/workspace/stores/billingOperationStore.test.ts @@ -152,17 +152,19 @@ describe('billingOperationStore', () => { return store } - it('carries a parked-on-checkout phase onto the operation', async () => { + it('exposes checkout recovery after polling without an action URL', async () => { const store = await pollPhase('awaiting_payment_method') await vi.waitFor(() => - expect(store.getOperation('op-phase')?.phase).toBe( - 'awaiting_payment_method' - ) + expect(store.subscriptionActionOperation).toMatchObject({ + opId: 'op-phase', + phase: 'awaiting_payment_method', + actionUrl: null + }) ) }) - it('carries a parked-on-invoice phase onto the operation', async () => { + it('does not expose checkout recovery while awaiting invoice payment', async () => { const store = await pollPhase('awaiting_invoice_payment') await vi.waitFor(() => @@ -170,6 +172,7 @@ describe('billingOperationStore', () => { 'awaiting_invoice_payment' ) ) + expect(store.subscriptionActionOperation).toBeUndefined() }) it('leaves the phase null when the server reports none', async () => { diff --git a/src/platform/workspace/stores/billingOperationStore.ts b/src/platform/workspace/stores/billingOperationStore.ts index 939bc1988a0..0947964eab0 100644 --- a/src/platform/workspace/stores/billingOperationStore.ts +++ b/src/platform/workspace/stores/billingOperationStore.ts @@ -171,6 +171,7 @@ export const useBillingOperationStore = defineStore('billingOperation', () => { op.workspaceId === workspaceStore.activeWorkspaceId && ((op.status === 'pending' && (op.actionUrl !== null || + op.phase === 'awaiting_payment_method' || op.authenticationState === 'requires_action' || op.authenticationState === 'failed_retryable')) || op.status === 'reconciliation_needed')