From 8ec03e687272fcbafb30c5228747a7609ae389a4 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:33:27 -0700 Subject: [PATCH 01/23] fix: restore checkout recovery after reload --- .../workspace/stores/billingOperationStore.test.ts | 13 ++++++++----- .../workspace/stores/billingOperationStore.ts | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/platform/workspace/stores/billingOperationStore.test.ts b/src/platform/workspace/stores/billingOperationStore.test.ts index cb8c5db563b8..d9e56410a9c4 100644 --- a/src/platform/workspace/stores/billingOperationStore.test.ts +++ b/src/platform/workspace/stores/billingOperationStore.test.ts @@ -152,17 +152,19 @@ describe('billingOperationStore', () => { return store } - it('carries a parked-on-checkout phase onto the operation', async () => { + it('exposes checkout recovery after polling without an action URL', async () => { const store = await pollPhase('awaiting_payment_method') await vi.waitFor(() => - expect(store.getOperation('op-phase')?.phase).toBe( - 'awaiting_payment_method' - ) + expect(store.subscriptionActionOperation).toMatchObject({ + opId: 'op-phase', + phase: 'awaiting_payment_method', + actionUrl: null + }) ) }) - it('carries a parked-on-invoice phase onto the operation', async () => { + it('does not expose checkout recovery while awaiting invoice payment', async () => { const store = await pollPhase('awaiting_invoice_payment') await vi.waitFor(() => @@ -170,6 +172,7 @@ describe('billingOperationStore', () => { 'awaiting_invoice_payment' ) ) + expect(store.subscriptionActionOperation).toBeUndefined() }) it('leaves the phase null when the server reports none', async () => { diff --git a/src/platform/workspace/stores/billingOperationStore.ts b/src/platform/workspace/stores/billingOperationStore.ts index 939bc1988a0e..0947964eab09 100644 --- a/src/platform/workspace/stores/billingOperationStore.ts +++ b/src/platform/workspace/stores/billingOperationStore.ts @@ -171,6 +171,7 @@ export const useBillingOperationStore = defineStore('billingOperation', () => { op.workspaceId === workspaceStore.activeWorkspaceId && ((op.status === 'pending' && (op.actionUrl !== null || + op.phase === 'awaiting_payment_method' || op.authenticationState === 'requires_action' || op.authenticationState === 'failed_retryable')) || op.status === 'reconciliation_needed') From 4ac239c2b30b34a7ab68de1f47271a3be7adebd5 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:50:16 -0700 Subject: [PATCH 02/23] docs: propose real Cloud billing E2E coverage --- browser_tests/README.md | 4 + docs/testing/cloud-billing-e2e.md | 120 ++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 docs/testing/cloud-billing-e2e.md diff --git a/browser_tests/README.md b/browser_tests/README.md index e3cbbcc57beb..531995e3d44c 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -22,6 +22,10 @@ to a section below. ## Prerequisites +For the proposed real Cloud backend billing scenarios, environment requirements, +and release criteria, see the [Cloud billing E2E coverage proposal](../docs/testing/cloud-billing-e2e.md). +It is a draft plan; executable coverage and its CI gate remain to be implemented. + **CRITICAL**: Start the ComfyUI backend with `--multi-user`: ```bash diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md new file mode 100644 index 000000000000..774351235819 --- /dev/null +++ b/docs/testing/cloud-billing-e2e.md @@ -0,0 +1,120 @@ +# Cloud billing E2E coverage proposal + +Status: Draft for scope review. This document adds no executable tests or CI gate. + +## Problem and current coverage + +A no-card customer who did not finish hosted Stripe Checkout reached a +verification timeout and repeated same-plan 409 responses. Investigation also +found that backend timeout cleanup could leave `billing_status` at +`awaiting_payment_method` after the checkout operation ended. +Sources: [incident thread](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789104046181919) +and [backend investigation](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789107331270719). + +The existing [checkout recovery spec](../../browser_tests/tests/dialogs/checkoutRecovery.spec.ts) +seeds pending checkout state and mocks billing responses. It covers frontend +reconciliation behavior, but cannot establish that a real subscription, billing +operation, and workspace converge after Stripe and Temporal finish. + +The [backend timeout fix](https://github.com/Comfy-Org/cloud/pull/8978) merged on +2026-09-11. Deployment and cleanup of the existing cohort are separate evidence +requirements; this proposal does not establish either one. + +## Scope and owners + +The [thread confirms E2E as ShihChi's scope](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169), +with [Anupreet's confirmation](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789165368584359). +The additional assignments below remain open for review. + +| Work | Owner / status | +| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| FE to Cloud backend E2E coverage | ShihChi; implementation scope proposed below | +| Backend billing design plan | Luke with Wei, per thread | +| Dashboard and monitor updates | Luke, per [his reply](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789146868800469) | +| FE resume/retry behavior and timeout telemetry implementation | Owner and tracking issue to confirm | +| Sandbox identity, Stripe configuration, state inspection, and test cleanup | Backend partner to confirm with Luke | +| Existing stranded workspace cleanup | Track [BE-12602](https://linear.app/comfyorg/issue/BE-12602); confirm executor and completion evidence | +| CI failures and release decision | ShihChi proposes test triage; release owner and escalation path to confirm | + +## First executable slice + +Use a candidate frontend with a real Cloud test backend and Stripe test mode. +Start with personal, no-card, classic hosted checkout (`embedded_checkout=false`), +the path reported in the incident. Exercise the actual Subscribe click, preview +request, displayed preview, confirmed write, and resulting operation. + +Do not fulfill billing/auth requests with fixtures, inject a pending checkout +record, or seed a completed subscription to satisfy these scenarios. Setup may +provision an isolated test identity and empty workspace through an agreed backend +test interface. Existing mocked tests continue to provide fast frontend feedback. + +| Scenario | Browser assertion | Authoritative state assertion | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Complete payment | Subscribe opens the hosted payment flow; successful payment returns to a usable subscribed UI | Operation succeeds, subscription is active, workspace is `paid`; exactly one subscription/payment for the attempt | +| Abandon, return, resume | Close Checkout before payment, return or reload the app, and use a visible action to reopen the pending payment flow | Attempt remains resumable while pending; eventual payment converges to the successful state above | +| Retry the same plan | Repeated Subscribe attempts lead to a usable pending checkout or an actionable recovery path | No duplicate subscription/payment; a conflict response must not strand the user | +| Expire an abandoned attempt | After backend expiry, return and subscribe again without a permanent verification spinner | Original op is terminal, reserved subscription ended, workspace `inactive` for this no-other-subscription fixture; a fresh attempt succeeds | +| Popup blocked | Payment action remains accessible when the browser blocks automatic opening | Existing operation can still complete through that action without a duplicate attempt | + +The browser polling deadline and the backend payment-method wait are different +timeouts. Moving the browser clock does not advance Temporal. Verify the backend +timeout branch using Temporal's workflow test clock and full terminal-state +assertions; a live timeout journey additionally needs an agreed non-production +expiry mechanism that exercises the same timeout branch. An abandonment signal +alone does not prove the timer branch. Until that mechanism exists, mark the live +timeout row blocked and retain the backend regression test as separate evidence. + +Follow with saved-card, payment failure, delayed webhook, cross-workspace/user +recovery isolation, and supported embedded-checkout cases. Record the tested +billing rail and feature flags; a hosted-flow pass does not cover embedded checkout. + +## Environment and lifecycle contract to agree before implementation + +- Select the exact test frontend/backend deployment and verify Stripe test mode + before creating a checkout. Record expected and served frontend SHA, backend + revision, billing rail, and effective feature flags for each run. +- Provide real browser authentication for a dedicated test identity, plus a fresh + workspace/customer with no card, active subscription, or pending operation. + Use a per-run identity or exclusive lease so parallel runs cannot share billing + state. Define how expired credentials fail setup. +- Provide a supported readback for operation, subscription, workspace status, and + Stripe payment/session IDs. UI success alone is insufficient. Verify the fixture + starts clean before driving the user flow. +- Define bounded readiness deadlines for webhook delivery and state convergence. + Fail with the last observed state when exceeded; do not add fixed sleeps or + turn missing setup into a skipped passing test. +- Run cleanup even after failure. Cancel test subscriptions and expire remaining + sessions/operations through supported test mechanisms, then verify cleanup. + Cleanup failure fails the run and prevents reuse of the fixture. +- Preserve browser traces and redacted request/state evidence. Keep auth tokens, + payment links, and customer data out of public artifacts. + +## CI and release criteria (proposed, not enabled) + +1. Start with an explicitly triggered sandbox run against a recorded FE/BE pair. + Prove the test fails on a known-broken recovery or teardown implementation and + passes on the fixed pair before treating it as regression protection. +2. Run the successful payment and abandon/resume/retry cases for billing changes + in both repositories and before promoting a Cloud release candidate. Add a + scheduled run for deployed integration drift. Agree the shared trigger and + check owner before making it required. +3. Require a successful result for the exact candidate and flags being promoted. + Scenario failures, setup failures, skipped required cases, missing evidence, + and cleanup failures cannot authorize promotion. Record retries separately so + a pass after failure remains visible for triage. +4. Keep timeout workflow proof and any blocked live scenario explicit in the + result. A narrower smoke pass must not be reported as full matrix completion. + +## Observability and rollout completion + +Luke's monitoring work should independently verify ingestion and alerting for +stuck operations and workspace drift. Frontend event emission and actual RUM +ingestion need separate checks. Confirm whether `billing.operation.timeout` is +the agreed event contract before adding an assertion; a sampled-out session must +not become a flaky payment-test assertion. + +The gap is closed when the agreed matrix has executable tests, a recorded live +pass and known-bad rejection, a functioning CI/release gate, and a named failure +owner. Separately track the FE recovery fix, backend deployment, monitoring, and +BE-12602 cleanup with their own completion evidence. Merging this plan completes +none of those implementation or rollout steps. From 6cda24d6bf670ba7860b374b3c5483da7a7fd5f8 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:59:39 -0700 Subject: [PATCH 03/23] test: add real Cloud checkout recovery suite --- browser_tests/README.md | 7 +- .../fixtures/liveCloudBillingFixture.ts | 148 ++++++++++++++++++ .../utils/liveCloudBillingConfig.test.ts | 52 ++++++ .../fixtures/utils/liveCloudBillingConfig.ts | 50 ++++++ .../tests/liveCloud/checkoutRecovery.spec.ts | 109 +++++++++++++ docs/adr/README.md | 1 + ...-BILLING-0031-real-cloud-checkout-tests.md | 53 +++++++ docs/testing/cloud-billing-e2e.md | 86 +++++++++- package.json | 1 + playwright.cloud-billing.config.ts | 30 ++++ playwright.config.ts | 1 + tsconfig.json | 1 + 12 files changed, 532 insertions(+), 7 deletions(-) create mode 100644 browser_tests/fixtures/liveCloudBillingFixture.ts create mode 100644 browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts create mode 100644 browser_tests/fixtures/utils/liveCloudBillingConfig.ts create mode 100644 browser_tests/tests/liveCloud/checkoutRecovery.spec.ts create mode 100644 docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md create mode 100644 playwright.cloud-billing.config.ts diff --git a/browser_tests/README.md b/browser_tests/README.md index 531995e3d44c..94f59d462645 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -22,9 +22,10 @@ to a section below. ## Prerequisites -For the proposed real Cloud backend billing scenarios, environment requirements, -and release criteria, see the [Cloud billing E2E coverage proposal](../docs/testing/cloud-billing-e2e.md). -It is a draft plan; executable coverage and its CI gate remain to be implemented. +For real Cloud billing checkout recovery, sandbox setup, and remaining release +criteria, see [Cloud billing E2E coverage](../docs/testing/cloud-billing-e2e.md). +Its dedicated runner requires real authentication and a backend reset hook; +the live suite is excluded from the ordinary browser test configurations. **CRITICAL**: Start the ComfyUI backend with `--multi-user`: diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts new file mode 100644 index 000000000000..d46265cdfd2e --- /dev/null +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -0,0 +1,148 @@ +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' + +import { + zBillingOpStatusResponse, + zBillingStatusResponse, + zCurrentWorkspaceResponse, + zListSavedPaymentMethodsResponse, + zPreviewSubscribeResponse +} from '@comfyorg/ingest-types/zod' +import { z } from 'zod' + +import { + ComfyPage, + comfyExpect as expect, + comfyPageFixture as base +} from '@e2e/fixtures/ComfyPage' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +const execFileAsync = promisify(execFile) +const sandbox = loadLiveCloudBillingConfig() +const resetResultSchema = z.object({ + workspace_id: z.literal(sandbox.workspaceId), + stripe_livemode: z.literal(false), + pending_operations: z.literal(0) +}) + +async function resetSandbox() { + let stdout: string + try { + const result = await execFileAsync( + sandbox.resetScript, + [sandbox.baseURL, sandbox.workspaceId], + { timeout: 60_000, maxBuffer: 16_384 } + ) + stdout = result.stdout + } catch { + throw new Error('Sandbox billing reset failed; do not reuse this workspace') + } + resetResultSchema.parse(JSON.parse(stdout)) +} + +interface BillingSandbox { + preview: z.infer + readOperation: ( + operationId: string + ) => Promise> +} + +export const liveCloudBillingFixture = base.extend<{ + billingSandbox: BillingSandbox +}>({ + networkPolicy: async ({ baseURL }, use) => { + expect(baseURL).toBe(sandbox.baseURL) + const origins = new Set([sandbox.baseURL, ...sandbox.allowedOrigins]) + const unexpected = new Set() + await use({ origins, unexpected }) + expect( + [...unexpected], + 'Add the required sandbox dependency origin' + ).toEqual([]) + }, + billingSandbox: async ({ page }, use, testInfo) => { + let verifyCleanup: (() => Promise) | undefined + try { + await resetSandbox() + const [workspaceResponse, documentResponse] = await Promise.all([ + page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/workspaces/current' + ), + page.goto(sandbox.baseURL) + ]) + expect(workspaceResponse.status()).toBe(200) + const workspace = zCurrentWorkspaceResponse.parse( + await workspaceResponse.json() + ) + expect(workspace.id).toBe(sandbox.workspaceId) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const headers = await workspaceResponse.request().allHeaders() + + async function read(path: string, schema: z.ZodType): Promise { + const response = await page.request.get( + new URL(path, sandbox.baseURL).href, + { headers } + ) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } + + async function assertCleanWorkspace() { + const [status, methods] = await Promise.all([ + read('/api/billing/status', zBillingStatusResponse), + read('/api/billing/payment-methods', zListSavedPaymentMethodsResponse) + ]) + expect(status.is_active).toBe(false) + expect(status.billing_status).toBe('inactive') + expect(methods).toHaveLength(0) + } + + verifyCleanup = assertCleanWorkspace + await assertCleanWorkspace() + const [previewResponse] = await Promise.all([ + page.waitForResponse( + (response) => + new URL(response.url()).pathname === + '/api/billing/preview-subscribe' + ), + page.goto( + `${sandbox.baseURL}/cloud/subscribe?tier=creator&cycle=monthly` + ) + ]) + expect(previewResponse.status()).toBe(200) + const preview = zPreviewSubscribeResponse.parse( + await previewResponse.json() + ) + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await testInfo.attach('sandbox.json', { + body: JSON.stringify({ + baseURL: sandbox.baseURL, + workspaceId: workspace.id, + frontendVersion: + documentResponse?.headers()['x-frontend-version'] ?? null + }), + contentType: 'application/json' + }) + await use({ + preview, + readOperation: (operationId) => + read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + }) + } finally { + await resetSandbox() + await verifyCleanup?.() + } + }, + comfyPage: async ({ page, request, billingSandbox }, use) => { + expect(billingSandbox.preview.allowed).toBe(true) + await use(new ComfyPage(page, request)) + } +}) + +export { comfyExpect } from '@e2e/fixtures/ComfyPage' diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts new file mode 100644 index 000000000000..d51fea7e91b8 --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + liveCloudBillingConfigSchema, + loadLiveCloudBillingConfig +} from '@e2e/fixtures/utils/liveCloudBillingConfig' + +const sandboxConfig = { + baseURL: 'https://testcloud.comfy.org', + storageState: '/private/cloud-billing-auth.json', + workspaceId: '12345678-1234-4234-8234-123456789abc', + resetScript: '/private/reset-cloud-billing', + allowedOrigins: ['https://checkout.stripe.com'] +} + +describe('Live Cloud billing target validation', () => { + it.for([ + 'https://cloud.comfy.org', + 'https://testcloud.comfy.org.example.com', + 'https://testcloud.comfy.org/redirect', + 'http://testcloud.comfy.org', + 'not-a-url' + ])('rejects an unsafe target: %s', (baseURL) => { + expect( + liveCloudBillingConfigSchema.safeParse({ ...sandboxConfig, baseURL }) + .success + ).toBe(false) + }) + + it('rejects production Cloud as an allowed dependency', () => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + allowedOrigins: ['https://cloud.comfy.org'] + }).success + ).toBe(false) + }) + + it('accepts an isolated preview deployment', () => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + baseURL: 'https://pr-8978.testenvs.comfy.org' + }).success + ).toBe(true) + }) + + it('fails when sandbox configuration is missing', () => { + vi.stubEnv('CLOUD_BILLING_CONFIG', undefined) + expect(loadLiveCloudBillingConfig).toThrow('Set CLOUD_BILLING_CONFIG') + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts new file mode 100644 index 000000000000..40b71ba93b63 --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -0,0 +1,50 @@ +import { accessSync, constants, readFileSync } from 'node:fs' +import { isAbsolute } from 'node:path' + +import { z } from 'zod' + +const absolutePath = z.string().refine(isAbsolute, 'Use an absolute path') +const origin = z + .string() + .url() + .refine((value) => { + if (!URL.canParse(value)) return false + const url = new URL(value) + return url.protocol === 'https:' && url.origin === value + }, 'Use an HTTPS origin without a path') + +export const liveCloudBillingConfigSchema = z.object({ + baseURL: origin.refine((value) => { + if (!URL.canParse(value)) return false + const { hostname } = new URL(value) + return ( + hostname === 'testcloud.comfy.org' || + hostname === 'stagingcloud.comfy.org' || + /^pr-\d+\.testenvs\.comfy\.org$/.test(hostname) + ) + }, 'Use a Cloud test, staging, or PR preview deployment'), + storageState: absolutePath, + workspaceId: z.string().uuid(), + resetScript: absolutePath, + allowedOrigins: z + .array(origin) + .refine( + (values) => !values.includes('https://cloud.comfy.org'), + 'Production Cloud is not a sandbox dependency' + ) +}) + +export function loadLiveCloudBillingConfig() { + const path = process.env.CLOUD_BILLING_CONFIG + if (!path) { + throw new Error( + 'Set CLOUD_BILLING_CONFIG to a sandbox config. See docs/testing/cloud-billing-e2e.md.' + ) + } + const config = liveCloudBillingConfigSchema.parse( + JSON.parse(readFileSync(path, 'utf8')) + ) + accessSync(config.storageState, constants.R_OK) + accessSync(config.resetScript, constants.X_OK) + return config +} diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts new file mode 100644 index 000000000000..1f5892b3855b --- /dev/null +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -0,0 +1,109 @@ +import { zSubscribeResponse } from '@comfyorg/ingest-types/zod' + +import { + comfyExpect as expect, + liveCloudBillingFixture as test +} from '@e2e/fixtures/liveCloudBillingFixture' + +const testCheckoutUrl = /^https:\/\/checkout\.stripe\.com\/c\/pay\/cs_test_/ + +test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { + test('resumes an abandoned no-card checkout using the same billing operation', async ({ + comfyPage, + billingSandbox + }) => { + const preview = billingSandbox.preview + const currency = + preview.amount_due_cents === undefined ? 'USD' : preview.currency + if (!currency) throw new Error('The live billing preview has no currency') + const amountDue = new Intl.NumberFormat('en-US', { + style: 'currency', + currency + }).format( + Number(preview.amount_due_cents ?? preview.cost_today_cents) / 100 + ) + + await expect( + comfyPage.page.getByText(`Total due today ${amountDue}`, { exact: true }) + ).toBeVisible() + + const [subscribeResponse, checkout] = await Promise.all([ + comfyPage.page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/billing/subscribe' && + response.request().method() === 'POST' + ), + comfyPage.page.waitForEvent('popup'), + comfyPage.page + .getByRole('button', { name: 'Subscribe to Creator', exact: true }) + .click() + ]) + expect(subscribeResponse.status()).toBe(200) + const subscription = zSubscribeResponse.parse( + await subscribeResponse.json() + ) + expect(subscription.status).toBe('needs_payment_method') + expect(subscription.billing_op_id).not.toBe('') + expect(subscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(checkout).toHaveURL(testCheckoutUrl) + await checkout.close() + + await expect + .poll(async () => { + const operation = await billingSandbox.readOperation( + subscription.billing_op_id + ) + return { + id: operation.id, + status: operation.status, + phase: operation.phase + } + }) + .toEqual({ + id: subscription.billing_op_id, + status: 'pending', + phase: 'awaiting_payment_method' + }) + + const resumePayment = comfyPage.page.getByRole('button', { + name: 'Complete your payment', + exact: true + }) + await expect(resumePayment).toBeEnabled() + const [resumeResponse, resumedCheckout] = await Promise.all([ + comfyPage.page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/billing/subscribe' && + response.request().method() === 'POST' + ), + comfyPage.page.waitForEvent('popup'), + resumePayment.click() + ]) + expect(resumeResponse.status()).toBe(200) + const resumedSubscription = zSubscribeResponse.parse( + await resumeResponse.json() + ) + expect(resumedSubscription.status).toBe('needs_payment_method') + expect(resumedSubscription.billing_op_id).toBe(subscription.billing_op_id) + expect(resumedSubscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(resumedCheckout).toHaveURL(testCheckoutUrl) + await resumedCheckout.close() + await expect(resumePayment).toBeEnabled() + await expect + .poll(async () => { + const operation = await billingSandbox.readOperation( + subscription.billing_op_id + ) + return { + id: operation.id, + status: operation.status, + phase: operation.phase + } + }) + .toEqual({ + id: subscription.billing_op_id, + status: 'pending', + phase: 'awaiting_payment_method' + }) + }) +}) diff --git a/docs/adr/README.md b/docs/adr/README.md index 7fc0f7c491fa..98c5d9d70e15 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -43,6 +43,7 @@ An Architecture Decision Record captures an important architectural decision mad | [TELEMETRY-DIAGNOSTICS-0019](TELEMETRY-DIAGNOSTICS-0019-recoverable-event-diagnostics.md) | Recoverable Event Diagnostics | Proposed | 2026-08-25 | | [TELEMETRY-ROUTING-0013](TELEMETRY-ROUTING-0013-telemetry-routing-across-consumers.md) | Telemetry Routing Across Consumers | Accepted | 2026-07-28 | | [TELEMETRY-STARTUP-0030](TELEMETRY-STARTUP-0030-production-startup-telemetry.md) | Production Startup Telemetry | Proposed | 2026-09-08 | +| [TESTING-BILLING-0031](TESTING-BILLING-0031-real-cloud-checkout-tests.md) | Real Cloud Checkout Tests | Proposed | 2026-09-11 | | [TESTING-E2E-0027](TESTING-E2E-0027-coverage-measurement-and-test-selection.md) | E2E Coverage Measurement and Test Selection | Proposed | 2026-09-01 | | [WEBSITE-REQUESTS-0030](WEBSITE-REQUESTS-0030-creator-widgets-and-router-request-composition.md) | Creator widgets and Router request composition | Deprecated | 2026-09-09 | | [WIDGET-SERIALIZATION-0006](WIDGET-SERIALIZATION-0006-preserve-primitive-widget-values-across-copy-and-paste.md) | PrimitiveNode Widget Restoration Lifecycle | Accepted | 2026-02-22 | diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md new file mode 100644 index 000000000000..b7a9a6940975 --- /dev/null +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -0,0 +1,53 @@ +# ADR-TESTING-BILLING-0031: Real Cloud Checkout Tests + +Date: 2026-09-11 + +## Status + +Proposed + +## Context + +Mocked checkout responses cannot detect disagreement between the frontend, +Cloud billing operations, and Stripe. The abandoned no-card checkout incident +requires an integration regression test, while ordinary browser CI must remain +independent of real billing credentials and mutable customer state. + +## Decision + +Use a separate, explicitly invoked Playwright configuration for real Cloud billing +tests. Reuse the Comfy page object and network guard, but replace the mock-based +Cloud setup with real authenticated browser storage and an exact sandbox origin +allowlist. Keep these tests out of ordinary browser test collection. + +Require a dedicated personal workspace and an idempotent backend-owned reset +executable. The frontend repository owns browser assertions and reads billing state +through the public API; it does not embed database writes or Temporal admin logic. +Reset runs before and after the test, and failure prevents reuse of the fixture. +The reset implementation remains a prerequisite for live validation. + +Retain mocked tests for fast frontend feedback. Extending them alone was rejected +because supplied responses cannot expose a broken backend contract. Reusing a +shared customer without reset was rejected because a failed run would change the +next run's preconditions. Hardcoding backend admin cleanup here was rejected +because the backend owns subscription and workflow lifecycle rules. + +## Consequences + +### Positive + +- Real Subscribe responses and pending operations can expose integration failures. +- Ordinary CI does not gain a dependency on billing credentials or Stripe. +- The backend can implement cleanup without duplicating lifecycle rules in tests. + +### Negative + +- Sandbox auth, reset implementation, and cross-run workspace exclusion must be + provisioned before execution; the draft cannot claim live coverage yet. +- The first recovery test does not cover payment completion or Temporal expiry. +- A required cross-repository release gate remains follow-up work. + +## Notes + +See the [scenario matrix and run instructions](../testing/cloud-billing-e2e.md) +and [scope discussion](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169). diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 774351235819..79a58ea6b0d6 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -1,6 +1,7 @@ -# Cloud billing E2E coverage proposal +# Cloud billing E2E coverage -Status: Draft for scope review. This document adds no executable tests or CI gate. +Status: Draft implementation. One live checkout recovery test is included; +sandbox execution and a CI gate remain pending. ## Problem and current coverage @@ -38,6 +39,83 @@ The additional assignments below remain open for review. ## First executable slice +The [live recovery spec](../../browser_tests/tests/liveCloud/checkoutRecovery.spec.ts) +uses real browser authentication and Cloud billing responses. It checks the +displayed preview, starts a no-card checkout, closes the Stripe test popup, +retries through the payment button, and verifies that the backend returns the +same pending operation. It does not enter card details or prove payment success, +reload recovery, duplicate-charge prevention, or backend expiry. + +The dedicated runner excludes this suite from ordinary mocked browser runs. It +requires an explicit sandbox config, authenticated browser storage, and a reset +executable; missing prerequisites fail setup. Sandbox execution has not yet been +verified. + +### Run the live recovery test + +Store configuration and browser credentials outside the repository. Example +configuration (replace the workspace ID and paths): + +```json +{ + "baseURL": "https://testcloud.comfy.org", + "storageState": "/absolute/private/cloud-billing-auth.json", + "workspaceId": "12345678-1234-4234-8234-123456789abc", + "resetScript": "/absolute/private/reset-cloud-billing", + "allowedOrigins": [ + "https://identitytoolkit.googleapis.com", + "https://securetoken.googleapis.com", + "https://checkout.stripe.com", + "https://js.stripe.com" + ] +} +``` + +Use storage captured after a real sandbox login with Playwright's +`context.storageState({ path, indexedDB: true })`, including Firebase IndexedDB. +The account must own the configured personal workspace, have finished onboarding, +and use English UI settings. Configure the deployment for classic hosted checkout. +The allowed origins above are a starting set, not a verified dependency inventory; +add exact sandbox dependency origins reported by the network guard. Billing and +auth remain real; the shared fixture retains its unrelated media/metadata mocks. + +```bash +CLOUD_BILLING_CONFIG=/absolute/private/cloud-billing-config.json \ + pnpm test:browser:cloud-billing +``` + +The reset executable is an integration requirement, **not an existing backend +endpoint or an implementation supplied by this PR**. The backend partner must +provide it before the suite can run. It receives two arguments, the sandbox +origin and workspace ID, and must: + +1. Reject any non-test Stripe account or workspace outside its dedicated allowlist. +2. Idempotently end test subscriptions, clear test payment methods, and terminate + pending operations/sessions using supported backend mechanisms. Do not delete + the identity or invalidate its browser credentials. +3. Verify the workspace is inactive with no active subscription, saved payment + method, or pending operation. Exit nonzero if any step fails. +4. Write only this JSON receipt to stdout after verification: + +```json +{ + "workspace_id": "12345678-1234-4234-8234-123456789abc", + "stripe_livemode": false, + "pending_operations": 0 +} +``` + +The runner invokes reset before and after each test, including failure paths. +It also reads real billing status and saved payment methods before the scenario +and after cleanup when authenticated setup succeeded. Run only one job per +workspace across machines; `workers: 1` only serializes a single invocation. +Retries are disabled. Trace, screenshot, and video capture are disabled because +they can contain auth and checkout credentials. The small `sandbox.json` +attachment records the target and frontend version header if available; it is +not a complete FE/BE release-evidence record. + +### Remaining scenario matrix + Use a candidate frontend with a real Cloud test backend and Stripe test mode. Start with personal, no-card, classic hosted checkout (`embedded_checkout=false`), the path reported in the incident. Exercise the actual Subscribe click, preview @@ -116,5 +194,5 @@ not become a flaky payment-test assertion. The gap is closed when the agreed matrix has executable tests, a recorded live pass and known-bad rejection, a functioning CI/release gate, and a named failure owner. Separately track the FE recovery fix, backend deployment, monitoring, and -BE-12602 cleanup with their own completion evidence. Merging this plan completes -none of those implementation or rollout steps. +BE-12602 cleanup with their own completion evidence. Merging the first test does +not complete those remaining scenarios or rollout steps. diff --git a/package.json b/package.json index aa24d109d6c3..8a577dbbebbb 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", "test:browser": "pnpm exec playwright test", + "test:browser:cloud-billing": "pnpm exec playwright test --config playwright.cloud-billing.config.ts", "test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser", "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", diff --git a/playwright.cloud-billing.config.ts b/playwright.cloud-billing.config.ts new file mode 100644 index 000000000000..e5d325dd7a11 --- /dev/null +++ b/playwright.cloud-billing.config.ts @@ -0,0 +1,30 @@ +import { defineConfig, devices } from '@playwright/test' + +import { loadLiveCloudBillingConfig } from './browser_tests/fixtures/utils/liveCloudBillingConfig' + +const sandbox = loadLiveCloudBillingConfig() + +Object.assign(globalThis, { + __DISTRIBUTION__: 'cloud', + __IS_NIGHTLY__: false +}) + +export default defineConfig({ + testDir: './browser_tests/tests/liveCloud', + fullyParallel: false, + workers: 1, + retries: 0, + forbidOnly: true, + timeout: 120_000, + expect: { timeout: 30_000 }, + reporter: 'list', + use: { + ...devices['Desktop Chrome'], + baseURL: sandbox.baseURL, + storageState: sandbox.storageState, + locale: 'en-US', + trace: 'off', + screenshot: 'off', + video: 'off' + } +}) diff --git a/playwright.config.ts b/playwright.config.ts index f7675fc06a23..043c94683447 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -42,6 +42,7 @@ const maybeLocalOptions: PlaywrightTestConfig = process.env.PLAYWRIGHT_LOCAL export default defineConfig({ testDir: './browser_tests', testIgnore: [ + '**/liveCloud/**', '**/*.test.ts', // Untransformed recorder output — still bare codegen, not a runnable spec '**/*.raw.spec.ts', diff --git a/tsconfig.json b/tsconfig.json index 39955cbe8d6d..c61aa50f545b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -55,6 +55,7 @@ "src/**/*", "src/types/**/*.d.ts", "playwright.config.ts", + "playwright.cloud-billing.config.ts", "playwright.i18n.config.ts", "vite.config.mts", "vitest.config.ts" From 2a75d9c96fc681e587854a23956188ea5dce4baa Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:02:24 -0700 Subject: [PATCH 04/23] test: load sandbox credentials only for live tests --- browser_tests/fixtures/liveCloudBillingFixture.ts | 2 ++ playwright.cloud-billing.config.ts | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index d46265cdfd2e..c1df2e868585 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -50,6 +50,8 @@ interface BillingSandbox { export const liveCloudBillingFixture = base.extend<{ billingSandbox: BillingSandbox }>({ + baseURL: sandbox.baseURL, + storageState: sandbox.storageState, networkPolicy: async ({ baseURL }, use) => { expect(baseURL).toBe(sandbox.baseURL) const origins = new Set([sandbox.baseURL, ...sandbox.allowedOrigins]) diff --git a/playwright.cloud-billing.config.ts b/playwright.cloud-billing.config.ts index e5d325dd7a11..a13bfa0b2d41 100644 --- a/playwright.cloud-billing.config.ts +++ b/playwright.cloud-billing.config.ts @@ -1,9 +1,5 @@ import { defineConfig, devices } from '@playwright/test' -import { loadLiveCloudBillingConfig } from './browser_tests/fixtures/utils/liveCloudBillingConfig' - -const sandbox = loadLiveCloudBillingConfig() - Object.assign(globalThis, { __DISTRIBUTION__: 'cloud', __IS_NIGHTLY__: false @@ -20,8 +16,6 @@ export default defineConfig({ reporter: 'list', use: { ...devices['Desktop Chrome'], - baseURL: sandbox.baseURL, - storageState: sandbox.storageState, locale: 'en-US', trace: 'off', screenshot: 'off', From 496f80874aca4c997b3c4f7e98fa251ba9105ce2 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:30:53 -0700 Subject: [PATCH 05/23] test: use existing Cloud browser runner and sandbox cleanup --- .../fixtures/liveCloudBillingFixture.ts | 140 ++++--- .../fixtures/utils/liveCloudBillingCleanup.ts | 196 ++++++++++ .../utils/liveCloudBillingConfig.test.ts | 55 ++- .../fixtures/utils/liveCloudBillingConfig.ts | 78 ++-- .../tests/liveCloud/checkoutRecovery.spec.ts | 15 +- ...-BILLING-0031-real-cloud-checkout-tests.md | 57 ++- docs/testing/cloud-billing-e2e.md | 272 ++++--------- package.json | 5 +- playwright.cloud-billing.config.ts | 24 -- playwright.config.ts | 20 + pnpm-lock.yaml | 359 +++++++++++++----- pnpm-workspace.yaml | 3 + tsconfig.json | 1 - 13 files changed, 746 insertions(+), 479 deletions(-) create mode 100644 browser_tests/fixtures/utils/liveCloudBillingCleanup.ts delete mode 100644 playwright.cloud-billing.config.ts diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index c1df2e868585..610f0b199523 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -1,6 +1,3 @@ -import { execFile } from 'node:child_process' -import { promisify } from 'node:util' - import { zBillingOpStatusResponse, zBillingStatusResponse, @@ -8,38 +5,16 @@ import { zListSavedPaymentMethodsResponse, zPreviewSubscribeResponse } from '@comfyorg/ingest-types/zod' -import { z } from 'zod' +import type { z } from 'zod' import { ComfyPage, comfyExpect as expect, comfyPageFixture as base } from '@e2e/fixtures/ComfyPage' +import { withBillingCleanup } from '@e2e/fixtures/utils/liveCloudBillingCleanup' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' -const execFileAsync = promisify(execFile) -const sandbox = loadLiveCloudBillingConfig() -const resetResultSchema = z.object({ - workspace_id: z.literal(sandbox.workspaceId), - stripe_livemode: z.literal(false), - pending_operations: z.literal(0) -}) - -async function resetSandbox() { - let stdout: string - try { - const result = await execFileAsync( - sandbox.resetScript, - [sandbox.baseURL, sandbox.workspaceId], - { timeout: 60_000, maxBuffer: 16_384 } - ) - stdout = result.stdout - } catch { - throw new Error('Sandbox billing reset failed; do not reuse this workspace') - } - resetResultSchema.parse(JSON.parse(stdout)) -} - interface BillingSandbox { preview: z.infer readOperation: ( @@ -50,11 +25,24 @@ interface BillingSandbox { export const liveCloudBillingFixture = base.extend<{ billingSandbox: BillingSandbox }>({ - baseURL: sandbox.baseURL, - storageState: sandbox.storageState, + baseURL: process.env.PLAYWRIGHT_TEST_URL, networkPolicy: async ({ baseURL }, use) => { - expect(baseURL).toBe(sandbox.baseURL) - const origins = new Set([sandbox.baseURL, ...sandbox.allowedOrigins]) + const sandbox = loadLiveCloudBillingConfig() + const origins = new Set([ + new URL(baseURL ?? sandbox.PLAYWRIGHT_TEST_URL).origin, + sandbox.PLAYWRIGHT_SETUP_API_URL, + 'https://identitytoolkit.googleapis.com', + 'https://securetoken.googleapis.com', + 'https://dreamboothy-dev.firebaseapp.com', + 'https://checkout.stripe.com', + 'https://api.stripe.com', + 'https://js.stripe.com', + 'https://m.stripe.network', + 'https://m.stripe.com', + 'https://r.stripe.com', + 'https://q.stripe.com', + 'https://b.stripecdn.com' + ]) const unexpected = new Set() await use({ origins, unexpected }) expect( @@ -63,46 +51,52 @@ export const liveCloudBillingFixture = base.extend<{ ).toEqual([]) }, billingSandbox: async ({ page }, use, testInfo) => { - let verifyCleanup: (() => Promise) | undefined - try { - await resetSandbox() - const [workspaceResponse, documentResponse] = await Promise.all([ - page.waitForResponse( - (response) => - new URL(response.url()).pathname === '/api/workspaces/current' - ), - page.goto(sandbox.baseURL) - ]) - expect(workspaceResponse.status()).toBe(200) - const workspace = zCurrentWorkspaceResponse.parse( - await workspaceResponse.json() - ) - expect(workspace.id).toBe(sandbox.workspaceId) - expect(workspace.type).toBe('personal') - expect(workspace.role).toBe('owner') - const headers = await workspaceResponse.request().allHeaders() + const sandbox = loadLiveCloudBillingConfig() + const documentResponse = await page.goto( + `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/login` + ) + await page + .getByRole('textbox', { name: 'Email', exact: true }) + .fill(sandbox.CLOUD_ACCOUNT_EMAIL) + await page + .getByLabel('Password', { exact: true }) + .fill(sandbox.CLOUD_ACCOUNT_PASSWORD) + const [workspaceResponse] = await Promise.all([ + page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/workspaces/current' + ), + page.getByRole('button', { name: 'Sign in', exact: true }).click() + ]) + expect(workspaceResponse.status()).toBe(200) + const workspace = zCurrentWorkspaceResponse.parse( + await workspaceResponse.json() + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const headers = await workspaceResponse.request().allHeaders() - async function read(path: string, schema: z.ZodType): Promise { - const response = await page.request.get( - new URL(path, sandbox.baseURL).href, - { headers } - ) - expect(response.status(), `GET ${path}`).toBe(200) - return schema.parse(await response.json()) - } + async function read(path: string, schema: z.ZodType): Promise { + const response = await page.request.get( + new URL(path, sandbox.PLAYWRIGHT_TEST_URL).href, + { headers } + ) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } - async function assertCleanWorkspace() { - const [status, methods] = await Promise.all([ - read('/api/billing/status', zBillingStatusResponse), - read('/api/billing/payment-methods', zListSavedPaymentMethodsResponse) - ]) - expect(status.is_active).toBe(false) - expect(status.billing_status).toBe('inactive') - expect(methods).toHaveLength(0) - } + async function assertCleanWorkspace() { + const [status, methods] = await Promise.all([ + read('/api/billing/status', zBillingStatusResponse), + read('/api/billing/payment-methods', zListSavedPaymentMethodsResponse) + ]) + expect(status.is_active).toBe(false) + expect(status.billing_status).toBe('inactive') + expect(methods).toHaveLength(0) + } - verifyCleanup = assertCleanWorkspace - await assertCleanWorkspace() + await assertCleanWorkspace() + await withBillingCleanup(sandbox, workspace.id, async () => { const [previewResponse] = await Promise.all([ page.waitForResponse( (response) => @@ -110,7 +104,7 @@ export const liveCloudBillingFixture = base.extend<{ '/api/billing/preview-subscribe' ), page.goto( - `${sandbox.baseURL}/cloud/subscribe?tier=creator&cycle=monthly` + `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/subscribe?tier=creator&cycle=monthly` ) ]) expect(previewResponse.status()).toBe(200) @@ -121,7 +115,7 @@ export const liveCloudBillingFixture = base.extend<{ expect(preview.new_plan.slug).toBe('creator-monthly') await testInfo.attach('sandbox.json', { body: JSON.stringify({ - baseURL: sandbox.baseURL, + baseURL: sandbox.PLAYWRIGHT_TEST_URL, workspaceId: workspace.id, frontendVersion: documentResponse?.headers()['x-frontend-version'] ?? null @@ -136,10 +130,8 @@ export const liveCloudBillingFixture = base.extend<{ zBillingOpStatusResponse ) }) - } finally { - await resetSandbox() - await verifyCleanup?.() - } + }) + await assertCleanWorkspace() }, comfyPage: async ({ page, request, billingSandbox }, use) => { expect(billingSandbox.preview.allowed).toBe(true) diff --git a/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts b/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts new file mode 100644 index 000000000000..980fc48771ab --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts @@ -0,0 +1,196 @@ +import { expect } from '@playwright/test' +import { Client as TemporalClient, Connection } from '@temporalio/client' +import { Client } from 'pg' +import { z } from 'zod' + +import type { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +const operationSchema = z.object({ + id: z.string(), + type: z.literal('initial_subscription'), + engine: z.literal('temporal'), + phase: z.literal('awaiting_payment_method'), + status: z.enum(['pending', 'failed']), + metronome_commit_id: z.null(), + to_subscription_id: z.string() +}) +const sessionSchema = z.object({ + id: z.string().startsWith('cs_test_'), + customer: z.string(), + livemode: z.literal(false), + status: z.enum(['open', 'expired']) +}) + +type Config = z.infer + +export async function withBillingCleanup( + config: Config, + workspaceId: string, + run: () => Promise +) { + const db = new Client({ + connectionString: config.SMOKE_DB_DSN, + connectionTimeoutMillis: 10_000, + statement_timeout: 10_000 + }) + await db.connect() + let connection: Connection | undefined + try { + const lock = await db.query( + 'SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked', + [workspaceId] + ) + expect( + z.object({ locked: z.literal(true) }).parse(lock.rows[0]).locked, + 'Another billing test owns this workspace' + ).toBe(true) + const state = await db.query( + `SELECT stripe_customer_id, billing_status, active_subscription_id + FROM workspaces WHERE id=$1`, + [workspaceId] + ) + const workspace = z + .object({ + stripe_customer_id: z.string().startsWith('cus_'), + billing_status: z.literal('inactive'), + active_subscription_id: z.null() + }) + .parse(state.rows[0]) + const pending = await db.query( + `SELECT id FROM billing_ops + WHERE workspace_id=$1 AND status IN ('pending', 'paid_ungrantable')`, + [workspaceId] + ) + expect( + pending.rows, + 'Workspace has pre-existing billing operations' + ).toEqual([]) + const subscriptions = await db.query( + `SELECT id FROM workspace_subscriptions + WHERE workspace_id=$1 AND status <> 'ended'`, + [workspaceId] + ) + expect( + subscriptions.rows, + 'Workspace has pre-existing subscriptions' + ).toEqual([]) + + async function stripe(path: string, method = 'GET'): Promise { + const response = await fetch(`https://api.stripe.com/v1/${path}`, { + method, + headers: { Authorization: `Bearer ${config.SMOKE_STRIPE_TEST_KEY}` }, + signal: AbortSignal.timeout(15_000) + }) + expect(response.status, `Stripe ${method} ${path.split('?')[0]}`).toBe( + 200 + ) + return response.json() + } + z.object({ + id: z.literal(workspace.stripe_customer_id), + livemode: z.literal(false) + }).parse(await stripe(`customers/${workspace.stripe_customer_id}`)) + connection = await Connection.connect({ + address: config.TEMPORAL_ADDRESS, + ...(config.TEMPORAL_API_KEY + ? { apiKey: config.TEMPORAL_API_KEY, tls: true } + : {}) + }) + const temporal = new TemporalClient({ + connection, + namespace: config.TEMPORAL_NAMESPACE + }) + await temporal.workflowService.describeNamespace({ + namespace: config.TEMPORAL_NAMESPACE + }) + const before = await db.query( + 'SELECT id FROM billing_ops WHERE workspace_id=$1', + [workspaceId] + ) + const ids = z + .array(z.object({ id: z.string() })) + .parse(before.rows) + .map(({ id }) => id) + const startedAt = Math.floor(Date.now() / 1000) + async function cleanup() { + const sessions = z + .object({ data: z.array(sessionSchema), has_more: z.literal(false) }) + .parse( + await stripe( + `checkout/sessions?customer=${workspace.stripe_customer_id}&created[gte]=${startedAt}&limit=100` + ) + ) + for (const session of sessions.data) { + expect(session.customer).toBe(workspace.stripe_customer_id) + if (session.status === 'open') { + expect( + sessionSchema.parse( + await stripe(`checkout/sessions/${session.id}/expire`, 'POST') + ).status + ).toBe('expired') + } + } + const created = await db.query( + `SELECT id, type, engine, phase, status, metronome_commit_id, to_subscription_id + FROM billing_ops WHERE workspace_id=$1 AND NOT (id = ANY($2::varchar[]))`, + [workspaceId, ids] + ) + const operations = z.array(operationSchema).parse(created.rows) + for (const operation of operations) { + const workflow = temporal.workflow.getHandle(operation.id) + if (operation.status === 'pending') { + await workflow.signal('subscription_checkout_abandoned', {}) + } + await expect + .poll(async () => (await workflow.describe()).status.name, { + timeout: 60_000 + }) + .toBe('FAILED') + const ended = await db.query( + `SELECT o.status AS operation_status, s.status AS subscription_status + FROM billing_ops o JOIN workspace_subscriptions s ON s.id=o.to_subscription_id + WHERE o.id=$1 AND o.workspace_id=$2`, + [operation.id, workspaceId] + ) + z.object({ + operation_status: z.literal('failed'), + subscription_status: z.literal('ended') + }).parse(ended.rows[0]) + } + if (operations.length > 0) { + await db.query('BEGIN') + try { + const restored = await db.query( + `UPDATE workspaces SET billing_status='inactive', update_time=NOW() + WHERE id=$1 AND active_subscription_id IS NULL + AND billing_status IN ('inactive', 'awaiting_payment_method') + AND NOT EXISTS (SELECT 1 FROM billing_ops WHERE workspace_id=$1 AND status IN ('pending', 'paid_ungrantable')) + AND NOT EXISTS (SELECT 1 FROM workspace_subscriptions WHERE workspace_id=$1 AND status <> 'ended') + RETURNING id`, + [workspaceId] + ) + expect( + restored.rowCount, + 'Refusing cleanup of unexpected billing state' + ).toBe(1) + await db.query( + `INSERT INTO billing_outboxes (create_time, update_time, event_type, workspace_id, processed) + VALUES (NOW(), NOW(), 'projection_dirty', $1, false)`, + [workspaceId] + ) + await db.query('COMMIT') + } catch (error) { + await db.query('ROLLBACK') + throw error + } + } + } + try { + await run() + } finally { + await cleanup() + } + } finally { + await Promise.all([connection?.close(), db.end()]) + } +} diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index d51fea7e91b8..f319cbbe7529 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -6,47 +6,72 @@ import { } from '@e2e/fixtures/utils/liveCloudBillingConfig' const sandboxConfig = { - baseURL: 'https://testcloud.comfy.org', - storageState: '/private/cloud-billing-auth.json', - workspaceId: '12345678-1234-4234-8234-123456789abc', - resetScript: '/private/reset-cloud-billing', - allowedOrigins: ['https://checkout.stripe.com'] + PLAYWRIGHT_TEST_URL: 'http://localhost:5173', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', + CLOUD_ACCOUNT_EMAIL: 'billing-e2e@example.com', + CLOUD_ACCOUNT_PASSWORD: 'test-password', + SMOKE_DB_DSN: 'postgresql://localhost/sandbox', + SMOKE_STRIPE_TEST_KEY: 'sk_test_example', + TEMPORAL_ADDRESS: 'localhost:7233', + TEMPORAL_NAMESPACE: 'default' } -describe('Live Cloud billing target validation', () => { +describe('Live Cloud billing prerequisites', () => { it.for([ 'https://cloud.comfy.org', 'https://testcloud.comfy.org.example.com', 'https://testcloud.comfy.org/redirect', 'http://testcloud.comfy.org', 'not-a-url' - ])('rejects an unsafe target: %s', (baseURL) => { + ])('rejects an unsafe backend: %s', (PLAYWRIGHT_SETUP_API_URL) => { expect( - liveCloudBillingConfigSchema.safeParse({ ...sandboxConfig, baseURL }) - .success + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_SETUP_API_URL + }).success + ).toBe(false) + }) + + it('rejects a production frontend even with sandbox cleanup credentials', () => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL: 'https://cloud.comfy.org' + }).success ).toBe(false) }) - it('rejects production Cloud as an allowed dependency', () => { + it('rejects a live Stripe key', () => { expect( liveCloudBillingConfigSchema.safeParse({ ...sandboxConfig, - allowedOrigins: ['https://cloud.comfy.org'] + SMOKE_STRIPE_TEST_KEY: 'sk_live_example' }).success ).toBe(false) }) + it('accepts a local frontend backed by test Cloud', () => { + expect(liveCloudBillingConfigSchema.safeParse(sandboxConfig).success).toBe( + true + ) + }) + it('accepts an isolated preview deployment', () => { + const target = 'https://pr-8978.testenvs.comfy.org' expect( liveCloudBillingConfigSchema.safeParse({ ...sandboxConfig, - baseURL: 'https://pr-8978.testenvs.comfy.org' + PLAYWRIGHT_TEST_URL: target, + PLAYWRIGHT_SETUP_API_URL: target }).success ).toBe(true) }) - it('fails when sandbox configuration is missing', () => { - vi.stubEnv('CLOUD_BILLING_CONFIG', undefined) - expect(loadLiveCloudBillingConfig).toThrow('Set CLOUD_BILLING_CONFIG') + it('reports missing prerequisites without leaking credentials', () => { + vi.stubEnv('CLOUD_ACCOUNT_PASSWORD', 'private-value') + vi.stubEnv('SMOKE_STRIPE_TEST_KEY', 'sk_live_private-value') + vi.stubEnv('CLOUD_ACCOUNT_EMAIL', undefined) + expect(loadLiveCloudBillingConfig).toThrow('CLOUD_ACCOUNT_EMAIL') + expect(loadLiveCloudBillingConfig).not.toThrow('private-value') }) }) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 40b71ba93b63..9d70636ec88d 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -1,50 +1,58 @@ -import { accessSync, constants, readFileSync } from 'node:fs' -import { isAbsolute } from 'node:path' - import { z } from 'zod' -const absolutePath = z.string().refine(isAbsolute, 'Use an absolute path') -const origin = z +const sandboxURL = z .string() .url() .refine((value) => { if (!URL.canParse(value)) return false const url = new URL(value) - return url.protocol === 'https:' && url.origin === value - }, 'Use an HTTPS origin without a path') - -export const liveCloudBillingConfigSchema = z.object({ - baseURL: origin.refine((value) => { - if (!URL.canParse(value)) return false - const { hostname } = new URL(value) return ( - hostname === 'testcloud.comfy.org' || - hostname === 'stagingcloud.comfy.org' || - /^pr-\d+\.testenvs\.comfy\.org$/.test(hostname) + url.protocol === 'https:' && + url.origin === value && + (url.hostname === 'testcloud.comfy.org' || + url.hostname === 'stagingcloud.comfy.org' || + /^pr-\d+\.testenvs\.comfy\.org$/.test(url.hostname)) ) - }, 'Use a Cloud test, staging, or PR preview deployment'), - storageState: absolutePath, - workspaceId: z.string().uuid(), - resetScript: absolutePath, - allowedOrigins: z - .array(origin) - .refine( - (values) => !values.includes('https://cloud.comfy.org'), - 'Production Cloud is not a sandbox dependency' - ) -}) + }, 'Use a Cloud test, staging, or PR preview origin') + +export const liveCloudBillingConfigSchema = z + .object({ + PLAYWRIGHT_TEST_URL: z.string().url(), + PLAYWRIGHT_SETUP_API_URL: sandboxURL, + CLOUD_ACCOUNT_EMAIL: z.string().email(), + CLOUD_ACCOUNT_PASSWORD: z.string().min(1), + SMOKE_DB_DSN: z.string().min(1), + SMOKE_STRIPE_TEST_KEY: z.string().startsWith('sk_test_'), + TEMPORAL_ADDRESS: z.string().min(1), + TEMPORAL_NAMESPACE: z.string().min(1), + TEMPORAL_API_KEY: z.string().min(1).optional() + }) + .superRefine((config, ctx) => { + if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return + const frontend = new URL(config.PLAYWRIGHT_TEST_URL) + const local = + ['localhost', '127.0.0.1'].includes(frontend.hostname) && + ['http:', 'https:'].includes(frontend.protocol) + if ( + !local && + config.PLAYWRIGHT_TEST_URL !== config.PLAYWRIGHT_SETUP_API_URL + ) { + ctx.addIssue({ + code: 'custom', + path: ['PLAYWRIGHT_TEST_URL'], + message: 'Use localhost or the selected sandbox origin' + }) + } + }) export function loadLiveCloudBillingConfig() { - const path = process.env.CLOUD_BILLING_CONFIG - if (!path) { + const result = liveCloudBillingConfigSchema.safeParse(process.env) + if (!result.success) { throw new Error( - 'Set CLOUD_BILLING_CONFIG to a sandbox config. See docs/testing/cloud-billing-e2e.md.' + `Cloud billing prerequisites: ${result.error.issues + .map((issue) => issue.path.join('.')) + .join(', ')}. See docs/testing/cloud-billing-e2e.md.` ) } - const config = liveCloudBillingConfigSchema.parse( - JSON.parse(readFileSync(path, 'utf8')) - ) - accessSync(config.storageState, constants.R_OK) - accessSync(config.resetScript, constants.X_OK) - return config + return result.data } diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts index 1f5892b3855b..b29159f6580a 100644 --- a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -11,7 +11,7 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { test('resumes an abandoned no-card checkout using the same billing operation', async ({ comfyPage, billingSandbox - }) => { + }, testInfo) => { const preview = billingSandbox.preview const currency = preview.amount_due_cents === undefined ? 'USD' : preview.currency @@ -27,6 +27,11 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { comfyPage.page.getByText(`Total due today ${amountDue}`, { exact: true }) ).toBeVisible() + await testInfo.attach('preview.png', { + body: await comfyPage.page.screenshot(), + contentType: 'image/png' + }) + const [subscribeResponse, checkout] = await Promise.all([ comfyPage.page.waitForResponse( (response) => @@ -46,6 +51,10 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { expect(subscription.billing_op_id).not.toBe('') expect(subscription.payment_method_url).toMatch(testCheckoutUrl) await expect(checkout).toHaveURL(testCheckoutUrl) + await testInfo.attach('stripe-checkout.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) await checkout.close() await expect @@ -70,6 +79,10 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { exact: true }) await expect(resumePayment).toBeEnabled() + await testInfo.attach('resume-payment.png', { + body: await comfyPage.page.screenshot(), + contentType: 'image/png' + }) const [resumeResponse, resumedCheckout] = await Promise.all([ comfyPage.page.waitForResponse( (response) => diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md index b7a9a6940975..9cfacc28ac0d 100644 --- a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -9,45 +9,40 @@ Proposed ## Context Mocked checkout responses cannot detect disagreement between the frontend, -Cloud billing operations, and Stripe. The abandoned no-card checkout incident -requires an integration regression test, while ordinary browser CI must remain -independent of real billing credentials and mutable customer state. +Cloud billing operations, and Stripe. Ordinary browser CI must remain independent +of real billing credentials and mutable customer state. ## Decision -Use a separate, explicitly invoked Playwright configuration for real Cloud billing -tests. Reuse the Comfy page object and network guard, but replace the mock-based -Cloud setup with real authenticated browser storage and an exact sandbox origin -allowlist. Keep these tests out of ordinary browser test collection. +Add an explicitly enabled `cloud-live` project to the existing Playwright runner. +Reuse `.env`, frontend/backend URL conventions, and ComfyPage, replacing the +mock-based Cloud setup with real UI authentication. Capture post-login screenshots. -Require a dedicated personal workspace and an idempotent backend-owned reset -executable. The frontend repository owns browser assertions and reads billing state -through the public API; it does not embed database writes or Temporal admin logic. -Reset runs before and after the test, and failure prevents reuse of the fixture. -The reset implementation remains a prerequisite for live validation. +Use a dedicated inactive personal workspace. Reuse Cloud E2E account variables +and billing smoke database/Stripe variables. Bundle teardown with the fixture: +expire test-created Stripe sessions, signal the existing Temporal abandonment +path, verify terminal operations, and restore inactive billing with a projection +outbox event. Guard cleanup with workspace state checks and an advisory lock. -Retain mocked tests for fast frontend feedback. Extending them alone was rejected -because supplied responses cannot expose a broken backend contract. Reusing a -shared customer without reset was rejected because a failed run would change the -next run's preconditions. Hardcoding backend admin cleanup here was rejected -because the backend owns subscription and workflow lifecycle rules. +A separate JSON configuration and an unimplemented reset executable were rejected: +they add another setup contract without making the test runnable. Extending mocked +tests alone cannot expose a broken backend contract. A public-API-only reset is +not available for a pending checkout. Adding a new backend admin endpoint would +expand the first test's scope. The fixture therefore depends on existing privileged +sandbox interfaces, like Cloud's billing smoke tests. ## Consequences -### Positive - -- Real Subscribe responses and pending operations can expose integration failures. -- Ordinary CI does not gain a dependency on billing credentials or Stripe. -- The backend can implement cleanup without duplicating lifecycle rules in tests. - -### Negative - -- Sandbox auth, reset implementation, and cross-run workspace exclusion must be - provisioned before execution; the draft cannot claim live coverage yet. -- The first recovery test does not cover payment completion or Temporal expiry. -- A required cross-repository release gate remains follow-up work. +- Normal browser test collection remains independent of sandbox credentials. +- Test setup can reject missing access before creating a billing operation. +- The frontend test now depends on backend schema and workflow cleanup contracts; + changes to these contracts require updating this fixture. +- Cleanup is not a product recovery assertion and cannot count as timeout coverage. +- A dedicated identity and backend credentials remain required for live validation. +- Payment completion, reload recovery, timeout behavior, and a release gate remain + follow-up work. ## Notes -See the [scenario matrix and run instructions](../testing/cloud-billing-e2e.md) -and [scope discussion](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169). +See the [run instructions](../testing/cloud-billing-e2e.md) and +[scope discussion](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169). diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 79a58ea6b0d6..e3497b12973e 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -1,198 +1,84 @@ -# Cloud billing E2E coverage - -Status: Draft implementation. One live checkout recovery test is included; -sandbox execution and a CI gate remain pending. - -## Problem and current coverage - -A no-card customer who did not finish hosted Stripe Checkout reached a -verification timeout and repeated same-plan 409 responses. Investigation also -found that backend timeout cleanup could leave `billing_status` at -`awaiting_payment_method` after the checkout operation ended. -Sources: [incident thread](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789104046181919) -and [backend investigation](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789107331270719). - -The existing [checkout recovery spec](../../browser_tests/tests/dialogs/checkoutRecovery.spec.ts) -seeds pending checkout state and mocks billing responses. It covers frontend -reconciliation behavior, but cannot establish that a real subscription, billing -operation, and workspace converge after Stripe and Temporal finish. - -The [backend timeout fix](https://github.com/Comfy-Org/cloud/pull/8978) merged on -2026-09-11. Deployment and cleanup of the existing cohort are separate evidence -requirements; this proposal does not establish either one. - -## Scope and owners - -The [thread confirms E2E as ShihChi's scope](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169), -with [Anupreet's confirmation](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789165368584359). -The additional assignments below remain open for review. - -| Work | Owner / status | -| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| FE to Cloud backend E2E coverage | ShihChi; implementation scope proposed below | -| Backend billing design plan | Luke with Wei, per thread | -| Dashboard and monitor updates | Luke, per [his reply](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789146868800469) | -| FE resume/retry behavior and timeout telemetry implementation | Owner and tracking issue to confirm | -| Sandbox identity, Stripe configuration, state inspection, and test cleanup | Backend partner to confirm with Luke | -| Existing stranded workspace cleanup | Track [BE-12602](https://linear.app/comfyorg/issue/BE-12602); confirm executor and completion evidence | -| CI failures and release decision | ShihChi proposes test triage; release owner and escalation path to confirm | - -## First executable slice - -The [live recovery spec](../../browser_tests/tests/liveCloud/checkoutRecovery.spec.ts) -uses real browser authentication and Cloud billing responses. It checks the -displayed preview, starts a no-card checkout, closes the Stripe test popup, -retries through the payment button, and verifies that the backend returns the -same pending operation. It does not enter card details or prove payment success, -reload recovery, duplicate-charge prevention, or backend expiry. - -The dedicated runner excludes this suite from ordinary mocked browser runs. It -requires an explicit sandbox config, authenticated browser storage, and a reset -executable; missing prerequisites fail setup. Sandbox execution has not yet been -verified. - -### Run the live recovery test - -Store configuration and browser credentials outside the repository. Example -configuration (replace the workspace ID and paths): - -```json -{ - "baseURL": "https://testcloud.comfy.org", - "storageState": "/absolute/private/cloud-billing-auth.json", - "workspaceId": "12345678-1234-4234-8234-123456789abc", - "resetScript": "/absolute/private/reset-cloud-billing", - "allowedOrigins": [ - "https://identitytoolkit.googleapis.com", - "https://securetoken.googleapis.com", - "https://checkout.stripe.com", - "https://js.stripe.com" - ] -} -``` +# Real Cloud checkout recovery + +Run this through the existing `browser_tests` Playwright runner. The `cloud` +project supplies mock billing responses; `cloud-live` signs in through the real +Cloud login page and sends real preview, subscribe, and operation-status requests. -Use storage captured after a real sandbox login with Playwright's -`context.storageState({ path, indexedDB: true })`, including Firebase IndexedDB. -The account must own the configured personal workspace, have finished onboarding, -and use English UI settings. Configure the deployment for classic hosted checkout. -The allowed origins above are a starting set, not a verified dependency inventory; -add exact sandbox dependency origins reported by the network guard. Billing and -auth remain real; the shared fixture retains its unrelated media/metadata mocks. - -```bash -CLOUD_BILLING_CONFIG=/absolute/private/cloud-billing-config.json \ - pnpm test:browser:cloud-billing +The opt-in command enables only `cloud-live`, with one worker and no retries: + +```sh +pnpm test:browser:cloud-billing --headed +pnpm exec playwright show-report ``` -The reset executable is an integration requirement, **not an existing backend -endpoint or an implementation supplied by this PR**. The backend partner must -provide it before the suite can run. It receives two arguments, the sandbox -origin and workspace ID, and must: - -1. Reject any non-test Stripe account or workspace outside its dedicated allowlist. -2. Idempotently end test subscriptions, clear test payment methods, and terminate - pending operations/sessions using supported backend mechanisms. Do not delete - the identity or invalidate its browser credentials. -3. Verify the workspace is inactive with no active subscription, saved payment - method, or pending operation. Exit nonzero if any step fails. -4. Write only this JSON receipt to stdout after verification: - -```json -{ - "workspace_id": "12345678-1234-4234-8234-123456789abc", - "stripe_livemode": false, - "pending_operations": 0 -} +The test checks the displayed amount against the real preview, opens Stripe test +checkout without entering a card, closes it, and resumes the same pending billing +operation. The HTML report includes screenshots of the preview, Stripe checkout, +and resume button. Authentication traces, videos, and storage files are not saved. +This does not test payment completion, page-reload recovery, or the 24-hour timeout. + +## Environment + +Use the existing `.env` convention. Do not commit credentials. + +| Variable | Value | +| -------------------------- | ------------------------------------------------------------------------- | +| `PLAYWRIGHT_TEST_URL` | Frontend URL, such as `http://localhost:5173` | +| `PLAYWRIGHT_SETUP_API_URL` | `https://testcloud.comfy.org`, staging, or a PR sandbox origin | +| `CLOUD_ACCOUNT_EMAIL` | Dedicated sandbox account, following Cloud's E2E account convention | +| `CLOUD_ACCOUNT_PASSWORD` | That account's password | +| `SMOKE_DB_DSN` | Database for the same sandbox, following Cloud's billing smoke convention | +| `SMOKE_STRIPE_TEST_KEY` | Stripe `sk_test_` key for that sandbox | +| `TEMPORAL_ADDRESS` | That sandbox's Temporal endpoint | +| `TEMPORAL_NAMESPACE` | That sandbox's Temporal namespace | +| `TEMPORAL_API_KEY` | Required for Temporal Cloud; omit for a local connection | + +For a local frontend connected to test Cloud: + +```sh +pnpm dev:cloud ``` -The runner invokes reset before and after each test, including failure paths. -It also reads real billing status and saved payment methods before the scenario -and after cleanup when authenticated setup succeeded. Run only one job per -workspace across machines; `workers: 1` only serializes a single invocation. -Retries are disabled. Trace, screenshot, and video capture are disabled because -they can contain auth and checkout credentials. The small `sandbox.json` -attachment records the target and frontend version header if available; it is -not a complete FE/BE release-evidence record. - -### Remaining scenario matrix - -Use a candidate frontend with a real Cloud test backend and Stripe test mode. -Start with personal, no-card, classic hosted checkout (`embedded_checkout=false`), -the path reported in the incident. Exercise the actual Subscribe click, preview -request, displayed preview, confirmed write, and resulting operation. - -Do not fulfill billing/auth requests with fixtures, inject a pending checkout -record, or seed a completed subscription to satisfy these scenarios. Setup may -provision an isolated test identity and empty workspace through an agreed backend -test interface. Existing mocked tests continue to provide fast frontend feedback. - -| Scenario | Browser assertion | Authoritative state assertion | -| --------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| Complete payment | Subscribe opens the hosted payment flow; successful payment returns to a usable subscribed UI | Operation succeeds, subscription is active, workspace is `paid`; exactly one subscription/payment for the attempt | -| Abandon, return, resume | Close Checkout before payment, return or reload the app, and use a visible action to reopen the pending payment flow | Attempt remains resumable while pending; eventual payment converges to the successful state above | -| Retry the same plan | Repeated Subscribe attempts lead to a usable pending checkout or an actionable recovery path | No duplicate subscription/payment; a conflict response must not strand the user | -| Expire an abandoned attempt | After backend expiry, return and subscribe again without a permanent verification spinner | Original op is terminal, reserved subscription ended, workspace `inactive` for this no-other-subscription fixture; a fresh attempt succeeds | -| Popup blocked | Payment action remains accessible when the browser blocks automatic opening | Existing operation can still complete through that action without a duplicate attempt | - -The browser polling deadline and the backend payment-method wait are different -timeouts. Moving the browser clock does not advance Temporal. Verify the backend -timeout branch using Temporal's workflow test clock and full terminal-state -assertions; a live timeout journey additionally needs an agreed non-production -expiry mechanism that exercises the same timeout branch. An abandonment signal -alone does not prove the timer branch. Until that mechanism exists, mark the live -timeout row blocked and retain the backend regression test as separate evidence. - -Follow with saved-card, payment failure, delayed webhook, cross-workspace/user -recovery isolation, and supported embedded-checkout cases. Record the tested -billing rail and feature flags; a hosted-flow pass does not cover embedded checkout. - -## Environment and lifecycle contract to agree before implementation - -- Select the exact test frontend/backend deployment and verify Stripe test mode - before creating a checkout. Record expected and served frontend SHA, backend - revision, billing rail, and effective feature flags for each run. -- Provide real browser authentication for a dedicated test identity, plus a fresh - workspace/customer with no card, active subscription, or pending operation. - Use a per-run identity or exclusive lease so parallel runs cannot share billing - state. Define how expired credentials fail setup. -- Provide a supported readback for operation, subscription, workspace status, and - Stripe payment/session IDs. UI success alone is insufficient. Verify the fixture - starts clean before driving the user flow. -- Define bounded readiness deadlines for webhook delivery and state convergence. - Fail with the last observed state when exceeded; do not add fixed sleeps or - turn missing setup into a skipped passing test. -- Run cleanup even after failure. Cancel test subscriptions and expire remaining - sessions/operations through supported test mechanisms, then verify cleanup. - Cleanup failure fails the run and prevents reuse of the fixture. -- Preserve browser traces and redacted request/state evidence. Keep auth tokens, - payment links, and customer data out of public artifacts. - -## CI and release criteria (proposed, not enabled) - -1. Start with an explicitly triggered sandbox run against a recorded FE/BE pair. - Prove the test fails on a known-broken recovery or teardown implementation and - passes on the fixed pair before treating it as regression protection. -2. Run the successful payment and abandon/resume/retry cases for billing changes - in both repositories and before promoting a Cloud release candidate. Add a - scheduled run for deployed integration drift. Agree the shared trigger and - check owner before making it required. -3. Require a successful result for the exact candidate and flags being promoted. - Scenario failures, setup failures, skipped required cases, missing evidence, - and cleanup failures cannot authorize promotion. Record retries separately so - a pass after failure remains visible for triage. -4. Keep timeout workflow proof and any blocked live scenario explicit in the - result. A narrower smoke pass must not be reported as full matrix completion. - -## Observability and rollout completion - -Luke's monitoring work should independently verify ingestion and alerting for -stuck operations and workspace drift. Frontend event emission and actual RUM -ingestion need separate checks. Confirm whether `billing.operation.timeout` is -the agreed event contract before adding an assertion; a sampled-out session must -not become a flaky payment-test assertion. - -The gap is closed when the agreed matrix has executable tests, a recorded live -pass and known-bad rejection, a functioning CI/release gate, and a named failure -owner. Separately track the FE recovery fix, backend deployment, monitoring, and -BE-12602 cleanup with their own completion evidence. Merging the first test does -not complete those remaining scenarios or rollout steps. +In another terminal, run the test with `PLAYWRIGHT_TEST_URL` set to that Vite +server and `PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The frontend +server must proxy to this same backend. For staging or preview, use the matching +backend when starting the dev server, or point both URLs at the deployed sandbox. +Production targets and live Stripe keys are rejected. + +The sandbox must use the Temporal billing engine with checkout-abandonment signal +support. The account must own a personal workspace with inactive billing, no saved cards, +no active or scheduled subscriptions, and no pending billing operation. Provision +this identity separately from shared smoke accounts. The test refuses to repair +pre-existing state. GitHub frontend secrets include `CLOUD_TEST_EMAIL` and +`CLOUD_TEST_PASSWORD`, but their account suitability has not been verified. + +## Cleanup + +The fixture takes a database advisory lock for the workspace. Before allowing +Subscribe, it checks the database state, verifies the Stripe customer is in test +mode, and checks access to the Temporal namespace. A second run using this fixture +cannot use the same workspace concurrently. Other tools must not use this account. + +After the test, including assertion failures, cleanup: + +1. Expires only Stripe test checkout sessions created for this customer during + the run. +2. Finds operations created during the run and requires unpaid initial + subscriptions owned by Temporal. +3. Sends `subscription_checkout_abandoned` to those operations' workflows, waits + for terminal failure, and verifies their subscriptions ended. +4. Restores inactive workspace billing only if no pending operations or non-ended + subscriptions remain, emitting the backend's `projection_dirty` outbox event + in the same transaction. +5. Reads the public billing API again to verify inactive billing and no cards. + +Cleanup refuses unexpected paid state. A cleanup failure fails the run and means +the account requires investigation before reuse. Process termination can prevent +teardown, so do not share the identity. Cleanup is fixture maintenance, not evidence +that the product's timeout or abandonment behavior is correct. + +## Validation status + +The live checkout journey has not yet passed. Local sandbox credentials and +backend cleanup access are still required. Test collection and prerequisite +validation do not establish real Cloud coverage. diff --git a/package.json b/package.json index 8a577dbbebbb..026df7fdeeb8 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", "test:browser": "pnpm exec playwright test", - "test:browser:cloud-billing": "pnpm exec playwright test --config playwright.cloud-billing.config.ts", + "test:browser:cloud-billing": "cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1", "test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser", "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", @@ -169,6 +169,7 @@ "@storybook/vue3": "catalog:", "@storybook/vue3-vite": "catalog:", "@tailwindcss/vite": "catalog:", + "@temporalio/client": "catalog:", "@testing-library/jest-dom": "catalog:", "@testing-library/user-event": "catalog:", "@testing-library/vue": "catalog:", @@ -176,6 +177,7 @@ "@types/fs-extra": "catalog:", "@types/jsdom": "catalog:", "@types/node": "catalog:", + "@types/pg": "catalog:", "@types/semver": "catalog:", "@types/three": "catalog:", "@vitejs/plugin-vue": "catalog:", @@ -211,6 +213,7 @@ "oxfmt": "catalog:", "oxlint": "catalog:", "oxlint-tsgolint": "catalog:", + "pg": "catalog:", "picocolors": "catalog:", "postcss-html": "catalog:", "pretty-bytes": "catalog:", diff --git a/playwright.cloud-billing.config.ts b/playwright.cloud-billing.config.ts deleted file mode 100644 index a13bfa0b2d41..000000000000 --- a/playwright.cloud-billing.config.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { defineConfig, devices } from '@playwright/test' - -Object.assign(globalThis, { - __DISTRIBUTION__: 'cloud', - __IS_NIGHTLY__: false -}) - -export default defineConfig({ - testDir: './browser_tests/tests/liveCloud', - fullyParallel: false, - workers: 1, - retries: 0, - forbidOnly: true, - timeout: 120_000, - expect: { timeout: 30_000 }, - reporter: 'list', - use: { - ...devices['Desktop Chrome'], - locale: 'en-US', - trace: 'off', - screenshot: 'off', - video: 'off' - } -}) diff --git a/playwright.config.ts b/playwright.config.ts index 043c94683447..50aa8b30e5b5 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -61,6 +61,26 @@ export default defineConfig({ globalTeardown: './browser_tests/globalTeardown.ts', projects: [ + ...(process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? [ + { + name: 'cloud-live', + testMatch: '**/tests/liveCloud/**/*.spec.ts', + testIgnore: [], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off' as const, + video: 'off' as const, + screenshot: 'off' as const + } + } + ] + : []), { name: 'chromium', use: { ...devices['Desktop Chrome'] }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9be46e513ee9..d151e2ed8746 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -114,6 +114,9 @@ catalogs: '@tanstack/vue-virtual': specifier: ^3.13.12 version: 3.13.32 + '@temporalio/client': + specifier: ^1.23.0 + version: 1.23.0 '@testing-library/jest-dom': specifier: ^6.9.1 version: 6.9.1 @@ -147,6 +150,9 @@ catalogs: '@types/node': specifier: ^25.9.5 version: 25.9.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 '@types/semver': specifier: ^7.8.0 version: 7.8.0 @@ -327,6 +333,9 @@ catalogs: parse5: specifier: 8.0.1 version: 8.0.1 + pg: + specifier: ^8.23.0 + version: 8.23.0 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -579,7 +588,7 @@ importers: version: 14.4.0(vue@3.5.42(typescript@6.0.3)) '@vueuse/integrations': specifier: 'catalog:' - version: 14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3)) + version: 14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3)) '@vueuse/router': specifier: ^14.4.0 version: 14.4.0(vue-router@5.2.0(@vue/compiler-sfc@3.5.42)(esbuild@0.28.1)(pinia@4.0.2(@vue/devtools-api@8.1.5)(typescript@6.0.3)(vue@3.5.42(typescript@6.0.3)))(rolldown@1.2.6)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)))(vue@3.5.42(typescript@6.0.3)) @@ -731,6 +740,9 @@ importers: '@tailwindcss/vite': specifier: 'catalog:' version: 4.3.2(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) + '@temporalio/client': + specifier: 'catalog:' + version: 1.23.0 '@testing-library/jest-dom': specifier: 'catalog:' version: 6.9.1 @@ -739,7 +751,7 @@ importers: version: 14.6.1(@testing-library/dom@10.4.1) '@testing-library/vue': specifier: 'catalog:' - version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) + version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) '@total-typescript/shoehorn': specifier: 'catalog:' version: 0.1.2 @@ -752,6 +764,9 @@ importers: '@types/node': specifier: 'catalog:' version: 25.9.5 + '@types/pg': + specifier: 'catalog:' + version: 8.23.1 '@types/semver': specifier: 'catalog:' version: 7.8.0 @@ -857,6 +872,9 @@ importers: oxlint-tsgolint: specifier: 'catalog:' version: 7.0.2001 + pg: + specifier: 'catalog:' + version: 8.23.0 picocolors: specifier: 'catalog:' version: 1.1.1 @@ -1055,7 +1073,7 @@ importers: version: 14.6.1(@testing-library/dom@10.4.1) '@testing-library/vue': specifier: 'catalog:' - version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) + version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) '@total-typescript/shoehorn': specifier: 'catalog:' version: 0.1.2 @@ -2349,6 +2367,10 @@ packages: '@formkit/auto-animate@0.9.0': resolution: {integrity: sha512-VhP4zEAacXS3dfTpJpJ88QdLqMTcabMg0jwpOSxZ/VzfQVfl3GkZSCZThhGC5uhq/TxPHPzW0dzr4H9Bb1OgKA==} + '@grpc/grpc-js@1.14.4': + resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} + engines: {node: '>=12.10.0'} + '@grpc/grpc-js@1.9.16': resolution: {integrity: sha512-wE4Ut/olIzfKqp631XrG+wbF0v1vWFN4YL9FyXC2LJiG33DsV7PLzURjrCvY/6je2ntdRkeLpPDluzSRGaVltQ==} engines: {node: ^8.13.0 || >=10.10.0} @@ -2358,6 +2380,11 @@ packages: engines: {node: '>=6'} hasBin: true + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@hapi/address@5.1.1': resolution: {integrity: sha512-A+po2d/dVoY7cYajycYI43ZbYMXukuopIsqCjh5QzsBCipDtdofHntljDlpccMjIfTy6UOkg+5KPriwYch2bXA==} engines: {node: '>=14.0.0'} @@ -2653,15 +2680,15 @@ packages: '@jridgewell/source-map@0.3.11': resolution: {integrity: sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==} - '@jridgewell/sourcemap-codec@1.5.5': - resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - '@jridgewell/sourcemap-codec@1.6.0': resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@jsdevtools/ono@7.1.3': resolution: {integrity: sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==} @@ -4098,6 +4125,18 @@ packages: peerDependencies: vue: ^2.7.0 || ^3.0.0 + '@temporalio/client@1.23.0': + resolution: {integrity: sha512-E9nXZPiKDGEDbPpRi0EyQ9i1x70fEIG26aMXZO88kKslKZnEKNPphs06xIic7bhuDoSrPsgVfdDap6o3l6IO2A==} + engines: {node: '>= 20.3.0'} + + '@temporalio/common@1.23.0': + resolution: {integrity: sha512-TaZYb/ruzgzJ8jd42EqRUCI4ispLZETt2Jigc9szES6q9edDmfncRw143HyvPi7b8n9naohYmA/IxTe9COH/dg==} + engines: {node: '>= 20.3.0'} + + '@temporalio/proto@1.23.0': + resolution: {integrity: sha512-90JWnYEs1wyq97/SXkQGknRVMr/BCV9sZ0pOnK3fHTedVinfvz3/PBVFkeAJyU3CJqMQGnzRs9IIfsBTT1icrA==} + engines: {node: '>= 20.3.0'} + '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -4384,6 +4423,9 @@ packages: '@types/node@25.9.5': resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + '@types/pg@8.23.1': + resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + '@types/react-dom@19.3.0': resolution: {integrity: sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==} peerDependencies: @@ -4905,27 +4947,15 @@ packages: peerDependencies: '@babel/core': ^7.0.0-0 - '@vue/compiler-core@3.5.41': - resolution: {integrity: sha512-q0Xtv/F9w2YO/7htQhtiL+Ev2WCJbe5N2hc+XfgyKkEKqWpSxknmT8QOuGdEKNdjPq0c3F7rNpFkTo3Kfrm7pg==} - '@vue/compiler-core@3.5.42': resolution: {integrity: sha512-2Ye1ilMtKXxl8qZUrQ5j0CdgenFp/HFQmta6rfRyfEsTG69L6Wk+tWuNoHYHMx9E8tF2Slvdg1FuwDvAXdy1LQ==} - '@vue/compiler-dom@3.5.41': - resolution: {integrity: sha512-oKacVfNglLvGjnS6BXOlGL7EyG2h8X03pqXCjzotRZUaXGjbrTJUnVAQjrCqUnS+lyu31nwQjZY/d817GmCnfw==} - '@vue/compiler-dom@3.5.42': resolution: {integrity: sha512-qbhQZEFmycr+ni/qyuccS4sucNN7VAbDfbkvNxWOX2VfgFm90MNs3/UhRNKoPMEIVn0F8gdlYjLPvqxHwHeQOA==} - '@vue/compiler-sfc@3.5.41': - resolution: {integrity: sha512-XJhip7R2wy6vX3knCxdZN4KracFaZUef58s1KYewqluedHIJaPIVfXoYT7MF1F8nCvv6k8bWWxDC8opMkg1VTQ==} - '@vue/compiler-sfc@3.5.42': resolution: {integrity: sha512-fkCAFB4okcAANGMThboWnScp/gzWjU0ZSkVnjTIiplmMDq2uq0tIB3j+xVu4rhv5rvOgBySCysudmbMd6xRRqw==} - '@vue/compiler-ssr@3.5.41': - resolution: {integrity: sha512-U3v5OejKEGqOI0Wy0+Sz7hGuIFZHA4LSXzrNM3IMIeDyJEBBfTpX26n3SDgToRpP2bLc9FfI2j/kSgcJ8Emq5A==} - '@vue/compiler-ssr@3.5.42': resolution: {integrity: sha512-xmLk3wLkbizPAiLyomjgFFosf2ys9b5Ghb+oh/k2tnvipNz8OFrQOiTcWCzyK7MpBp9KkyGtfvgfLUivbmuGYA==} @@ -4973,9 +5003,6 @@ packages: '@vue/server-renderer@3.5.42': resolution: {integrity: sha512-2++5dUyYS4gvo7xQXSECUDhB7TS0aOl5SeVfC5qSq1Jgfhjvegw1zqhwTIR3imZ+QYPJQw9gfcFvXGAjGZ7ajQ==} - '@vue/shared@3.5.41': - resolution: {integrity: sha512-IOnwSCma8j+9xJT6b8H0dEYidC80NsYmNMlZxRsukYcSoGaDBohog5hDxzeUXdFeGWFA++vWvxqOmrr96VlqMA==} - '@vue/shared@3.5.42': resolution: {integrity: sha512-2rPxex1jQf4jvl9MOHl6YaXCPcrNqz/FstMOEh3QWY+/OME9nQTvl9WYeCwhW7AFjaR0SnngZGlp/wkR6rkI6g==} @@ -5080,6 +5107,10 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -6099,6 +6130,10 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -7171,9 +7206,6 @@ packages: magic-string@1.1.1: resolution: {integrity: sha512-qFemKPzc3ttrYVaMmnSkGtGc5nE6Ncl4bj7c9IE6C9OUIRXjf6PzJ+UZ1xhVIjYc7dolHq3qKzpAJPZUbvNj+A==} - magicast@0.5.3: - resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} - magicast@0.5.4: resolution: {integrity: sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==} @@ -7485,6 +7517,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + ms@3.0.0-canary.1: + resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} + engines: {node: '>=12.13'} + muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} @@ -7512,6 +7548,10 @@ packages: new-date@1.0.3: resolution: {integrity: sha512-0fsVvQPbo2I18DT2zVHpezmeeNYV2JaJSrseiHLc17GNOxJzUdx5mvSigPu8LtIfZSij5i1wXnXFspEs2CD6hA==} + nexus-rpc@0.0.3: + resolution: {integrity: sha512-ksIBeYHDv6FHAR1h3Vwk0kQ6REhdo1STqDHXMzu/yn/VN/XzgOor2IUjewG6paoA7IgxlIPZoOU/PD78PC6HTg==} + engines: {node: '>= 20.0.0'} + nlcst-to-string@4.0.0: resolution: {integrity: sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==} @@ -7786,6 +7826,40 @@ packages: perfect-debounce@2.1.0: resolution: {integrity: sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.16.0: + resolution: {integrity: sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.23.0: + resolution: {integrity: sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -7860,6 +7934,22 @@ packages: resolution: {integrity: sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + posthog-js@1.402.3: resolution: {integrity: sha512-MCn4O9wJc1BGBbb820MlmYifDb6EhFBFhfR1K9294gfgFRcxin+NSlypswVBdt1ZO4XD/P7EPmwApyVX/EYlUw==} @@ -7969,6 +8059,10 @@ packages: resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} engines: {node: '>=12.0.0'} + protobufjs@8.8.0: + resolution: {integrity: sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==} + engines: {node: '>=12.0.0'} + proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -8410,6 +8504,10 @@ packages: resolution: {integrity: sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==} engines: {node: '>=0.10.0'} + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} @@ -9336,9 +9434,6 @@ packages: vscode-nls@5.2.0: resolution: {integrity: sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==} - vscode-uri@3.1.0: - resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} - vscode-uri@3.2.0: resolution: {integrity: sha512-m2gXo3bn0G1kT9InzMf07fTbqMbGtyckj3bH5ktLO+1Ssv+yiATZ4dhwaQv9UZWxJh6E9IFGnQyjgWVDWVBDrg==} @@ -9597,6 +9692,10 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + xxhash-wasm@1.1.0: resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} @@ -9868,7 +9967,7 @@ snapshots: dependencies: '@astrojs/compiler': 2.13.1 '@astrojs/yaml2ts': 0.2.4 - '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/sourcemap-codec': 1.6.0 '@volar/kit': 2.4.28(typescript@6.0.3) '@volar/language-core': 2.4.28 '@volar/language-server': 2.4.28 @@ -9883,7 +9982,7 @@ snapshots: volar-service-typescript-twoslash-queries: 0.0.71(@volar/language-service@2.4.28) volar-service-yaml: 0.0.71(@volar/language-service@2.4.28) vscode-html-languageservice: 5.6.2 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: prettier: 3.9.5 transitivePeerDependencies: @@ -9962,7 +10061,7 @@ snapshots: dependencies: '@vitejs/plugin-vue': 6.0.8(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) '@vitejs/plugin-vue-jsx': 5.1.6(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) - '@vue/compiler-sfc': 3.5.41 + '@vue/compiler-sfc': 3.5.42 astro: 7.2.1(@astrojs/markdown-remark@7.2.2)(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@25.9.5)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vite: 8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vite-plugin-vue-devtools: 8.2.1(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) @@ -10961,6 +11060,11 @@ snapshots: '@formkit/auto-animate@0.9.0': {} + '@grpc/grpc-js@1.14.4': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + '@grpc/grpc-js@1.9.16': dependencies: '@grpc/proto-loader': 0.7.15 @@ -10973,6 +11077,13 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.3 + '@hapi/address@5.1.1': dependencies: '@hapi/hoek': 11.0.7 @@ -11272,8 +11383,6 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 - '@jridgewell/sourcemap-codec@1.5.5': {} - '@jridgewell/sourcemap-codec@1.6.0': {} '@jridgewell/trace-mapping@0.3.31': @@ -11281,6 +11390,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@js-sdsl/ordered-map@4.4.2': {} + '@jsdevtools/ono@7.1.3': {} '@keyv/bigmap@1.3.1(keyv@5.6.0)': @@ -12392,6 +12503,29 @@ snapshots: '@tanstack/virtual-core': 3.17.4 vue: 3.5.42(typescript@6.0.3) + '@temporalio/client@1.23.0': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@temporalio/common': 1.23.0 + '@temporalio/proto': 1.23.0 + abort-controller: 3.0.0 + long: 5.3.2 + nexus-rpc: 0.0.3 + uuid: 11.1.1 + + '@temporalio/common@1.23.0': + dependencies: + '@temporalio/proto': 1.23.0 + long: 5.3.2 + ms: 3.0.0-canary.1 + nexus-rpc: 0.0.3 + protobufjs: 8.8.0 + + '@temporalio/proto@1.23.0': + dependencies: + long: 5.3.2 + protobufjs: 8.8.0 + '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -12427,18 +12561,6 @@ snapshots: dependencies: '@testing-library/dom': 10.4.1 - '@testing-library/vue@8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': - dependencies: - '@babel/runtime': 7.29.7 - '@testing-library/dom': 9.3.4 - '@vue/test-utils': 2.4.11(@vue/compiler-dom@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) - vue: 3.5.42(typescript@6.0.3) - optionalDependencies: - '@vue/compiler-sfc': 3.5.41 - transitivePeerDependencies: - - '@vue/compiler-dom' - - '@vue/server-renderer' - '@testing-library/vue@8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': dependencies: '@babel/runtime': 7.29.7 @@ -12725,6 +12847,12 @@ snapshots: dependencies: undici-types: 7.24.6 + '@types/pg@8.23.1': + dependencies: + '@types/node': 25.9.5 + pg-protocol: 1.16.0 + pg-types: 2.2.0 + '@types/react-dom@19.3.0(@types/react@19.3.0)': dependencies: '@types/react': 19.3.0 @@ -13128,7 +13256,7 @@ snapshots: typesafe-path: 0.2.2 typescript: 6.0.3 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 '@volar/language-core@2.4.28': dependencies: @@ -13144,14 +13272,14 @@ snapshots: vscode-languageserver: 9.0.1 vscode-languageserver-protocol: 3.18.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 '@volar/language-service@2.4.28': dependencies: '@volar/language-core': 2.4.28 vscode-languageserver-protocol: 3.18.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 '@volar/source-map@2.4.28': {} @@ -13167,7 +13295,7 @@ snapshots: jsonc-parser: 2.3.1 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 '@vscode/l10n@0.0.18': {} @@ -13239,14 +13367,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@vue/compiler-core@3.5.41': - dependencies: - '@babel/parser': 7.29.8 - '@vue/shared': 3.5.41 - entities: 7.0.1 - estree-walker: 2.0.2 - source-map-js: 1.2.1 - '@vue/compiler-core@3.5.42': dependencies: '@babel/parser': 7.29.8 @@ -13255,28 +13375,11 @@ snapshots: estree-walker: 2.0.2 source-map-js: 1.2.1 - '@vue/compiler-dom@3.5.41': - dependencies: - '@vue/compiler-core': 3.5.41 - '@vue/shared': 3.5.41 - '@vue/compiler-dom@3.5.42': dependencies: '@vue/compiler-core': 3.5.42 '@vue/shared': 3.5.42 - '@vue/compiler-sfc@3.5.41': - dependencies: - '@babel/parser': 7.29.8 - '@vue/compiler-core': 3.5.41 - '@vue/compiler-dom': 3.5.41 - '@vue/compiler-ssr': 3.5.41 - '@vue/shared': 3.5.41 - estree-walker: 2.0.2 - magic-string: 0.30.21 - postcss: 8.5.26 - source-map-js: 1.2.1 - '@vue/compiler-sfc@3.5.42': dependencies: '@babel/parser': 7.29.8 @@ -13289,11 +13392,6 @@ snapshots: postcss: 8.5.26 source-map-js: 1.2.1 - '@vue/compiler-ssr@3.5.41': - dependencies: - '@vue/compiler-dom': 3.5.41 - '@vue/shared': 3.5.41 - '@vue/compiler-ssr@3.5.42': dependencies: '@vue/compiler-dom': 3.5.42 @@ -13380,8 +13478,6 @@ snapshots: '@vue/runtime-dom': 3.5.42 '@vue/shared': 3.5.42 - '@vue/shared@3.5.41': {} - '@vue/shared@3.5.42': {} '@vue/test-utils@2.4.11(@vue/compiler-dom@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': @@ -13409,7 +13505,7 @@ snapshots: '@vueuse/shared': 14.4.0(vue@3.5.42(typescript@6.0.3)) vue: 3.5.42(typescript@6.0.3) - '@vueuse/integrations@14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3))': + '@vueuse/integrations@14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3))': dependencies: '@vueuse/core': 14.4.0(vue@3.5.42(typescript@6.0.3)) '@vueuse/shared': 14.4.0(vue@3.5.42(typescript@6.0.3)) @@ -13452,6 +13548,10 @@ snapshots: abbrev@2.0.0: {} + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + acorn-jsx@5.3.2(acorn@8.17.0): dependencies: acorn: 8.17.0 @@ -13644,7 +13744,7 @@ snapshots: js-yaml: 4.3.1 jsonc-parser: 3.3.1 magic-string: 1.1.1 - magicast: 0.5.3 + magicast: 0.5.4 mrmime: 2.0.1 neotraverse: 1.0.1 obug: 2.1.4 @@ -14606,6 +14706,8 @@ snapshots: esutils@2.0.3: {} + event-target-shim@5.0.1: {} + eventemitter3@5.0.4: {} eventsource-parser@4.1.0: {} @@ -15754,7 +15856,7 @@ snapshots: magic-string@0.30.21: dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/sourcemap-codec': 1.6.0 magic-string@0.30.8: dependencies: @@ -15762,13 +15864,7 @@ snapshots: magic-string@1.1.1: dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - - magicast@0.5.3: - dependencies: - '@babel/parser': 7.29.8 - '@babel/types': 7.29.8 - source-map-js: 1.2.1 + '@jridgewell/sourcemap-codec': 1.6.0 magicast@0.5.4: dependencies: @@ -16362,6 +16458,8 @@ snapshots: ms@2.1.3: {} + ms@3.0.0-canary.1: {} + muggle-string@0.4.1: {} multi-buffer-data-view@6.0.27: @@ -16381,6 +16479,8 @@ snapshots: dependencies: '@segment/isodate': 1.0.3 + nexus-rpc@0.0.3: {} + nlcst-to-string@4.0.0: dependencies: '@types/nlcst': 2.0.3 @@ -16760,6 +16860,41 @@ snapshots: perfect-debounce@2.1.0: {} + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.23.0): + dependencies: + pg: 8.23.0 + + pg-protocol@1.16.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.23.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.23.0) + pg-protocol: 1.16.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -16828,6 +16963,16 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + posthog-js@1.402.3: dependencies: '@posthog/core': 1.42.1 @@ -16978,6 +17123,10 @@ snapshots: '@types/node': 25.9.5 long: 5.3.2 + protobufjs@8.8.0: + dependencies: + long: 5.3.2 + proxy-from-env@1.1.0: {} proxy-from-env@2.1.0: {} @@ -17568,6 +17717,8 @@ snapshots: speakingurl@14.0.1: {} + split2@4.2.0: {} + sprintf-js@1.0.3: optional: true @@ -18433,7 +18584,7 @@ snapshots: dependencies: vscode-css-languageservice: 6.3.10 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18442,7 +18593,7 @@ snapshots: '@emmetio/css-parser': 0.4.1 '@emmetio/html-matcher': 1.3.0 '@vscode/emmet-helper': 2.11.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18450,20 +18601,20 @@ snapshots: dependencies: vscode-html-languageservice: 5.6.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 volar-service-prettier@0.0.71(@volar/language-service@2.4.28)(prettier@3.9.5): dependencies: - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 prettier: 3.9.5 volar-service-typescript-twoslash-queries@0.0.71(@volar/language-service@2.4.28): dependencies: - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18474,13 +18625,13 @@ snapshots: typescript-auto-import-cache: 0.3.6 vscode-languageserver-textdocument: 1.0.12 vscode-nls: 5.2.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 optionalDependencies: '@volar/language-service': 2.4.28 volar-service-yaml@0.0.71(@volar/language-service@2.4.28): dependencies: - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 yaml-language-server: 1.23.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18490,14 +18641,14 @@ snapshots: '@vscode/l10n': 0.0.18 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.17.5 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 vscode-html-languageservice@5.6.2: dependencies: '@vscode/l10n': 0.0.18 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 vscode-json-languageservice@4.1.8: dependencies: @@ -18505,7 +18656,7 @@ snapshots: vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 vscode-nls: 5.2.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 vscode-jsonrpc@8.2.0: {} @@ -18533,8 +18684,6 @@ snapshots: vscode-nls@5.2.0: {} - vscode-uri@3.1.0: {} - vscode-uri@3.2.0: {} vue-component-meta@3.3.7(typescript@5.9.3): @@ -18556,7 +18705,7 @@ snapshots: '@babel/parser': 7.29.8 '@babel/types': 7.29.8 '@vue/compiler-dom': 3.5.42 - '@vue/compiler-sfc': 3.5.41 + '@vue/compiler-sfc': 3.5.42 ast-types: 0.16.1 esm-resolve: 1.0.11 hash-sum: 2.0.0 @@ -18819,6 +18968,8 @@ snapshots: xmlchars@2.2.0: {} + xtend@4.0.2: {} + xxhash-wasm@1.1.0: {} y18n@5.0.8: {} @@ -18842,7 +18993,7 @@ snapshots: vscode-languageserver: 9.0.1 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.1.0 + vscode-uri: 3.2.0 yaml: 2.9.0 yaml@2.9.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 49b01eef5b83..833f2bbf9973 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -47,6 +47,7 @@ catalog: '@stripe/stripe-js': ^9.12.0 '@tailwindcss/vite': ^4.3.2 '@tanstack/vue-virtual': ^3.13.12 + '@temporalio/client': ^1.23.0 '@testing-library/jest-dom': ^6.9.1 '@testing-library/user-event': ^14.6.1 '@testing-library/vue': ^8.1.0 @@ -59,6 +60,7 @@ catalog: '@types/jsdom': ^21.1.7 '@types/leaflet': ^1.9.20 '@types/node': ^25.9.5 + '@types/pg': ^8.23.1 '@types/semver': ^7.8.0 '@types/three': ^0.184.1 '@vee-validate/zod': ^4.15.1 @@ -119,6 +121,7 @@ catalog: oxlint: ^1.77.0 oxlint-tsgolint: ^7.0.2001 parse5: 8.0.1 + pg: ^8.23.0 picocolors: ^1.1.1 pinia: ^4.0.2 postcss-html: ^1.8.0 diff --git a/tsconfig.json b/tsconfig.json index c61aa50f545b..39955cbe8d6d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -55,7 +55,6 @@ "src/**/*", "src/types/**/*.d.ts", "playwright.config.ts", - "playwright.cloud-billing.config.ts", "playwright.i18n.config.ts", "vite.config.mts", "vitest.config.ts" From d784cf6bcb19ae842ccb75b256bc749043bb0129 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:02:05 -0700 Subject: [PATCH 06/23] test: open the Cloud email sign-in form --- browser_tests/fixtures/liveCloudBillingFixture.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 610f0b199523..b06559f92559 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -55,6 +55,9 @@ export const liveCloudBillingFixture = base.extend<{ const documentResponse = await page.goto( `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/login` ) + await page + .getByRole('button', { name: 'Use email instead', exact: true }) + .click() await page .getByRole('textbox', { name: 'Email', exact: true }) .fill(sandbox.CLOUD_ACCOUNT_EMAIL) From 049ccc4c0576e7303f97956f9fc088020a97f27d Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:15:41 -0700 Subject: [PATCH 07/23] test: run Cloud billing journey without admin credentials --- .../fixtures/liveCloudBillingFixture.ts | 118 +++++---- .../fixtures/utils/liveCloudBillingCleanup.ts | 196 --------------- .../utils/liveCloudBillingConfig.test.ts | 18 +- .../fixtures/utils/liveCloudBillingConfig.ts | 7 +- .../tests/liveCloud/checkoutRecovery.spec.ts | 7 + ...-BILLING-0031-real-cloud-checkout-tests.md | 48 ++-- docs/testing/cloud-billing-e2e.md | 109 +++------ package.json | 3 - pnpm-lock.yaml | 229 ------------------ pnpm-workspace.yaml | 3 - 10 files changed, 136 insertions(+), 602 deletions(-) delete mode 100644 browser_tests/fixtures/utils/liveCloudBillingCleanup.ts diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index b06559f92559..cde5b115f3d2 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -12,7 +12,6 @@ import { comfyExpect as expect, comfyPageFixture as base } from '@e2e/fixtures/ComfyPage' -import { withBillingCleanup } from '@e2e/fixtures/utils/liveCloudBillingCleanup' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' interface BillingSandbox { @@ -31,6 +30,16 @@ export const liveCloudBillingFixture = base.extend<{ const origins = new Set([ new URL(baseURL ?? sandbox.PLAYWRIGHT_TEST_URL).origin, sandbox.PLAYWRIGHT_SETUP_API_URL, + 'https://testapi.comfy.org', + 'https://t.comfy.org', + 'https://mp.comfy.org', + 'https://browser-intake-us5-datadoghq.com', + 'https://cdn.sy-d.io', + 'https://e2.sy-d.io', + 'https://consumer.cloud.gist.build', + 'https://realtime.cloud.gist.build', + 'https://cdp.customer.io', + 'https://o4507954455314432.ingest.us.sentry.io', 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', 'https://dreamboothy-dev.firebaseapp.com', @@ -64,20 +73,15 @@ export const liveCloudBillingFixture = base.extend<{ await page .getByLabel('Password', { exact: true }) .fill(sandbox.CLOUD_ACCOUNT_PASSWORD) - const [workspaceResponse] = await Promise.all([ + const [billingResponse] = await Promise.all([ page.waitForResponse( (response) => - new URL(response.url()).pathname === '/api/workspaces/current' + new URL(response.url()).pathname === '/api/billing/status' && + response.status() === 200 ), page.getByRole('button', { name: 'Sign in', exact: true }).click() ]) - expect(workspaceResponse.status()).toBe(200) - const workspace = zCurrentWorkspaceResponse.parse( - await workspaceResponse.json() - ) - expect(workspace.type).toBe('personal') - expect(workspace.role).toBe('owner') - const headers = await workspaceResponse.request().allHeaders() + const headers = await billingResponse.request().allHeaders() async function read(path: string, schema: z.ZodType): Promise { const response = await page.request.get( @@ -88,53 +92,73 @@ export const liveCloudBillingFixture = base.extend<{ return schema.parse(await response.json()) } - async function assertCleanWorkspace() { + const workspace = await read( + '/api/workspaces/current', + zCurrentWorkspaceResponse + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + + async function assertNoCardAccount() { const [status, methods] = await Promise.all([ read('/api/billing/status', zBillingStatusResponse), read('/api/billing/payment-methods', zListSavedPaymentMethodsResponse) ]) - expect(status.is_active).toBe(false) - expect(status.billing_status).toBe('inactive') - expect(methods).toHaveLength(0) - } - - await assertCleanWorkspace() - await withBillingCleanup(sandbox, workspace.id, async () => { - const [previewResponse] = await Promise.all([ - page.waitForResponse( - (response) => - new URL(response.url()).pathname === - '/api/billing/preview-subscribe' - ), - page.goto( - `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/subscribe?tier=creator&cycle=monthly` - ) - ]) - expect(previewResponse.status()).toBe(200) - const preview = zPreviewSubscribeResponse.parse( - await previewResponse.json() - ) - expect(preview.allowed).toBe(true) - expect(preview.new_plan.slug).toBe('creator-monthly') - await testInfo.attach('sandbox.json', { + await testInfo.attach('billing-preflight.json', { body: JSON.stringify({ - baseURL: sandbox.PLAYWRIGHT_TEST_URL, workspaceId: workspace.id, - frontendVersion: - documentResponse?.headers()['x-frontend-version'] ?? null + isActive: status.is_active, + billingStatus: status.billing_status, + subscriptionTier: status.subscription_tier, + planSlug: status.plan_slug, + paymentMethodCount: methods.length }), contentType: 'application/json' }) - await use({ - preview, - readOperation: (operationId) => - read( - `/api/billing/ops/${encodeURIComponent(operationId)}`, - zBillingOpStatusResponse - ) - }) + if (status.is_active) { + expect(status.subscription_tier).toBe('FREE') + } else { + expect(['inactive', 'awaiting_payment_method']).toContain( + status.billing_status + ) + } + expect(methods).toHaveLength(0) + } + + await assertNoCardAccount() + + const [previewResponse] = await Promise.all([ + page.waitForResponse( + (response) => + new URL(response.url()).pathname === '/api/billing/preview-subscribe' + ), + page.goto( + `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/subscribe?tier=creator&cycle=monthly` + ) + ]) + expect(previewResponse.status()).toBe(200) + const preview = zPreviewSubscribeResponse.parse( + await previewResponse.json() + ) + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await testInfo.attach('sandbox.json', { + body: JSON.stringify({ + baseURL: sandbox.PLAYWRIGHT_TEST_URL, + workspaceId: workspace.id, + frontendVersion: + documentResponse?.headers()['x-frontend-version'] ?? null + }), + contentType: 'application/json' + }) + await use({ + preview, + readOperation: (operationId) => + read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) }) - await assertCleanWorkspace() }, comfyPage: async ({ page, request, billingSandbox }, use) => { expect(billingSandbox.preview.allowed).toBe(true) diff --git a/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts b/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts deleted file mode 100644 index 980fc48771ab..000000000000 --- a/browser_tests/fixtures/utils/liveCloudBillingCleanup.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { expect } from '@playwright/test' -import { Client as TemporalClient, Connection } from '@temporalio/client' -import { Client } from 'pg' -import { z } from 'zod' - -import type { liveCloudBillingConfigSchema } from '@e2e/fixtures/utils/liveCloudBillingConfig' - -const operationSchema = z.object({ - id: z.string(), - type: z.literal('initial_subscription'), - engine: z.literal('temporal'), - phase: z.literal('awaiting_payment_method'), - status: z.enum(['pending', 'failed']), - metronome_commit_id: z.null(), - to_subscription_id: z.string() -}) -const sessionSchema = z.object({ - id: z.string().startsWith('cs_test_'), - customer: z.string(), - livemode: z.literal(false), - status: z.enum(['open', 'expired']) -}) - -type Config = z.infer - -export async function withBillingCleanup( - config: Config, - workspaceId: string, - run: () => Promise -) { - const db = new Client({ - connectionString: config.SMOKE_DB_DSN, - connectionTimeoutMillis: 10_000, - statement_timeout: 10_000 - }) - await db.connect() - let connection: Connection | undefined - try { - const lock = await db.query( - 'SELECT pg_try_advisory_lock(hashtextextended($1, 0)) AS locked', - [workspaceId] - ) - expect( - z.object({ locked: z.literal(true) }).parse(lock.rows[0]).locked, - 'Another billing test owns this workspace' - ).toBe(true) - const state = await db.query( - `SELECT stripe_customer_id, billing_status, active_subscription_id - FROM workspaces WHERE id=$1`, - [workspaceId] - ) - const workspace = z - .object({ - stripe_customer_id: z.string().startsWith('cus_'), - billing_status: z.literal('inactive'), - active_subscription_id: z.null() - }) - .parse(state.rows[0]) - const pending = await db.query( - `SELECT id FROM billing_ops - WHERE workspace_id=$1 AND status IN ('pending', 'paid_ungrantable')`, - [workspaceId] - ) - expect( - pending.rows, - 'Workspace has pre-existing billing operations' - ).toEqual([]) - const subscriptions = await db.query( - `SELECT id FROM workspace_subscriptions - WHERE workspace_id=$1 AND status <> 'ended'`, - [workspaceId] - ) - expect( - subscriptions.rows, - 'Workspace has pre-existing subscriptions' - ).toEqual([]) - - async function stripe(path: string, method = 'GET'): Promise { - const response = await fetch(`https://api.stripe.com/v1/${path}`, { - method, - headers: { Authorization: `Bearer ${config.SMOKE_STRIPE_TEST_KEY}` }, - signal: AbortSignal.timeout(15_000) - }) - expect(response.status, `Stripe ${method} ${path.split('?')[0]}`).toBe( - 200 - ) - return response.json() - } - z.object({ - id: z.literal(workspace.stripe_customer_id), - livemode: z.literal(false) - }).parse(await stripe(`customers/${workspace.stripe_customer_id}`)) - connection = await Connection.connect({ - address: config.TEMPORAL_ADDRESS, - ...(config.TEMPORAL_API_KEY - ? { apiKey: config.TEMPORAL_API_KEY, tls: true } - : {}) - }) - const temporal = new TemporalClient({ - connection, - namespace: config.TEMPORAL_NAMESPACE - }) - await temporal.workflowService.describeNamespace({ - namespace: config.TEMPORAL_NAMESPACE - }) - const before = await db.query( - 'SELECT id FROM billing_ops WHERE workspace_id=$1', - [workspaceId] - ) - const ids = z - .array(z.object({ id: z.string() })) - .parse(before.rows) - .map(({ id }) => id) - const startedAt = Math.floor(Date.now() / 1000) - async function cleanup() { - const sessions = z - .object({ data: z.array(sessionSchema), has_more: z.literal(false) }) - .parse( - await stripe( - `checkout/sessions?customer=${workspace.stripe_customer_id}&created[gte]=${startedAt}&limit=100` - ) - ) - for (const session of sessions.data) { - expect(session.customer).toBe(workspace.stripe_customer_id) - if (session.status === 'open') { - expect( - sessionSchema.parse( - await stripe(`checkout/sessions/${session.id}/expire`, 'POST') - ).status - ).toBe('expired') - } - } - const created = await db.query( - `SELECT id, type, engine, phase, status, metronome_commit_id, to_subscription_id - FROM billing_ops WHERE workspace_id=$1 AND NOT (id = ANY($2::varchar[]))`, - [workspaceId, ids] - ) - const operations = z.array(operationSchema).parse(created.rows) - for (const operation of operations) { - const workflow = temporal.workflow.getHandle(operation.id) - if (operation.status === 'pending') { - await workflow.signal('subscription_checkout_abandoned', {}) - } - await expect - .poll(async () => (await workflow.describe()).status.name, { - timeout: 60_000 - }) - .toBe('FAILED') - const ended = await db.query( - `SELECT o.status AS operation_status, s.status AS subscription_status - FROM billing_ops o JOIN workspace_subscriptions s ON s.id=o.to_subscription_id - WHERE o.id=$1 AND o.workspace_id=$2`, - [operation.id, workspaceId] - ) - z.object({ - operation_status: z.literal('failed'), - subscription_status: z.literal('ended') - }).parse(ended.rows[0]) - } - if (operations.length > 0) { - await db.query('BEGIN') - try { - const restored = await db.query( - `UPDATE workspaces SET billing_status='inactive', update_time=NOW() - WHERE id=$1 AND active_subscription_id IS NULL - AND billing_status IN ('inactive', 'awaiting_payment_method') - AND NOT EXISTS (SELECT 1 FROM billing_ops WHERE workspace_id=$1 AND status IN ('pending', 'paid_ungrantable')) - AND NOT EXISTS (SELECT 1 FROM workspace_subscriptions WHERE workspace_id=$1 AND status <> 'ended') - RETURNING id`, - [workspaceId] - ) - expect( - restored.rowCount, - 'Refusing cleanup of unexpected billing state' - ).toBe(1) - await db.query( - `INSERT INTO billing_outboxes (create_time, update_time, event_type, workspace_id, processed) - VALUES (NOW(), NOW(), 'projection_dirty', $1, false)`, - [workspaceId] - ) - await db.query('COMMIT') - } catch (error) { - await db.query('ROLLBACK') - throw error - } - } - } - try { - await run() - } finally { - await cleanup() - } - } finally { - await Promise.all([connection?.close(), db.end()]) - } -} diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index f319cbbe7529..99c34841f172 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -9,11 +9,7 @@ const sandboxConfig = { PLAYWRIGHT_TEST_URL: 'http://localhost:5173', PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org', CLOUD_ACCOUNT_EMAIL: 'billing-e2e@example.com', - CLOUD_ACCOUNT_PASSWORD: 'test-password', - SMOKE_DB_DSN: 'postgresql://localhost/sandbox', - SMOKE_STRIPE_TEST_KEY: 'sk_test_example', - TEMPORAL_ADDRESS: 'localhost:7233', - TEMPORAL_NAMESPACE: 'default' + CLOUD_ACCOUNT_PASSWORD: 'test-password' } describe('Live Cloud billing prerequisites', () => { @@ -32,7 +28,7 @@ describe('Live Cloud billing prerequisites', () => { ).toBe(false) }) - it('rejects a production frontend even with sandbox cleanup credentials', () => { + it('rejects a production frontend with sandbox account credentials', () => { expect( liveCloudBillingConfigSchema.safeParse({ ...sandboxConfig, @@ -41,15 +37,6 @@ describe('Live Cloud billing prerequisites', () => { ).toBe(false) }) - it('rejects a live Stripe key', () => { - expect( - liveCloudBillingConfigSchema.safeParse({ - ...sandboxConfig, - SMOKE_STRIPE_TEST_KEY: 'sk_live_example' - }).success - ).toBe(false) - }) - it('accepts a local frontend backed by test Cloud', () => { expect(liveCloudBillingConfigSchema.safeParse(sandboxConfig).success).toBe( true @@ -69,7 +56,6 @@ describe('Live Cloud billing prerequisites', () => { it('reports missing prerequisites without leaking credentials', () => { vi.stubEnv('CLOUD_ACCOUNT_PASSWORD', 'private-value') - vi.stubEnv('SMOKE_STRIPE_TEST_KEY', 'sk_live_private-value') vi.stubEnv('CLOUD_ACCOUNT_EMAIL', undefined) expect(loadLiveCloudBillingConfig).toThrow('CLOUD_ACCOUNT_EMAIL') expect(loadLiveCloudBillingConfig).not.toThrow('private-value') diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 9d70636ec88d..62cdb31a9ce4 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -20,12 +20,7 @@ export const liveCloudBillingConfigSchema = z PLAYWRIGHT_TEST_URL: z.string().url(), PLAYWRIGHT_SETUP_API_URL: sandboxURL, CLOUD_ACCOUNT_EMAIL: z.string().email(), - CLOUD_ACCOUNT_PASSWORD: z.string().min(1), - SMOKE_DB_DSN: z.string().min(1), - SMOKE_STRIPE_TEST_KEY: z.string().startsWith('sk_test_'), - TEMPORAL_ADDRESS: z.string().min(1), - TEMPORAL_NAMESPACE: z.string().min(1), - TEMPORAL_API_KEY: z.string().min(1).optional() + CLOUD_ACCOUNT_PASSWORD: z.string().min(1) }) .superRefine((config, ctx) => { if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts index b29159f6580a..351ffe0aaad9 100644 --- a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -47,6 +47,13 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { const subscription = zSubscribeResponse.parse( await subscribeResponse.json() ) + await testInfo.attach('billing-operation.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + status: subscription.status + }), + contentType: 'application/json' + }) expect(subscription.status).toBe('needs_payment_method') expect(subscription.billing_op_id).not.toBe('') expect(subscription.payment_method_url).toMatch(testCheckoutUrl) diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md index 9cfacc28ac0d..801ecbf39cc9 100644 --- a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -8,41 +8,37 @@ Proposed ## Context -Mocked checkout responses cannot detect disagreement between the frontend, -Cloud billing operations, and Stripe. Ordinary browser CI must remain independent -of real billing credentials and mutable customer state. +Mocked responses cannot detect disagreement between the frontend, Cloud billing, +and Stripe. The first regression test must exercise the browser against a real +sandbox without requiring backend administrator credentials. ## Decision Add an explicitly enabled `cloud-live` project to the existing Playwright runner. -Reuse `.env`, frontend/backend URL conventions, and ComfyPage, replacing the -mock-based Cloud setup with real UI authentication. Capture post-login screenshots. - -Use a dedicated inactive personal workspace. Reuse Cloud E2E account variables -and billing smoke database/Stripe variables. Bundle teardown with the fixture: -expire test-created Stripe sessions, signal the existing Temporal abandonment -path, verify terminal operations, and restore inactive billing with a projection -outbox event. Guard cleanup with workspace state checks and an advisory lock. - -A separate JSON configuration and an unimplemented reset executable were rejected: -they add another setup contract without making the test runnable. Extending mocked -tests alone cannot expose a broken backend contract. A public-API-only reset is -not available for a pending checkout. Adding a new backend admin endpoint would -expand the first test's scope. The fixture therefore depends on existing privileged -sandbox interfaces, like Cloud's billing smoke tests. +Reuse `.env`, frontend/backend URL conventions, and ComfyPage. Sign in through the +real UI and capture post-login screenshots. Keep normal browser CI independent of +sandbox credentials. + +Use a dedicated no-card account and the public billing API. Close the unpaid +Stripe checkout, resume payment, and assert that the same billing operation is +reused. Record the pending operation instead of resetting backend state. + +A separate JSON configuration and an external reset executable add unnecessary +setup. Embedding database writes and Stripe/Temporal administration in the fixture +also makes browser coverage depend on privileged access and backend internals. +Those approaches are rejected for this test. Backend reset and expiry verification +can be added separately when required for repeatable CI. ## Consequences -- Normal browser test collection remains independent of sandbox credentials. -- Test setup can reject missing access before creating a billing operation. -- The frontend test now depends on backend schema and workflow cleanup contracts; - changes to these contracts require updating this fixture. -- Cleanup is not a product recovery assertion and cannot count as timeout coverage. -- A dedicated identity and backend credentials remain required for live validation. +- The browser test requires only sandbox URLs and account credentials. +- No payment is submitted and no backend billing state is reset. +- Unpaid operations can remain pending after a run. Account reuse depends on the + backend's ability to resume that state, and concurrent runs must be avoided. +- Cleanup is not a substitute for an assertion about product recovery. - Payment completion, reload recovery, timeout behavior, and a release gate remain follow-up work. ## Notes -See the [run instructions](../testing/cloud-billing-e2e.md) and -[scope discussion](https://comfy-organization.slack.com/archives/C0BNGQG2LCW/p1789164154382169). +See the [run instructions](../testing/cloud-billing-e2e.md). diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index e3497b12973e..a2cfbc15ca3c 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -1,84 +1,41 @@ # Real Cloud checkout recovery -Run this through the existing `browser_tests` Playwright runner. The `cloud` -project supplies mock billing responses; `cloud-live` signs in through the real -Cloud login page and sends real preview, subscribe, and operation-status requests. - -The opt-in command enables only `cloud-live`, with one worker and no retries: +Run the opt-in `cloud-live` project through the existing browser-test runner: ```sh pnpm test:browser:cloud-billing --headed pnpm exec playwright show-report ``` -The test checks the displayed amount against the real preview, opens Stripe test -checkout without entering a card, closes it, and resumes the same pending billing -operation. The HTML report includes screenshots of the preview, Stripe checkout, -and resume button. Authentication traces, videos, and storage files are not saved. -This does not test payment completion, page-reload recovery, or the 24-hour timeout. - -## Environment - -Use the existing `.env` convention. Do not commit credentials. - -| Variable | Value | -| -------------------------- | ------------------------------------------------------------------------- | -| `PLAYWRIGHT_TEST_URL` | Frontend URL, such as `http://localhost:5173` | -| `PLAYWRIGHT_SETUP_API_URL` | `https://testcloud.comfy.org`, staging, or a PR sandbox origin | -| `CLOUD_ACCOUNT_EMAIL` | Dedicated sandbox account, following Cloud's E2E account convention | -| `CLOUD_ACCOUNT_PASSWORD` | That account's password | -| `SMOKE_DB_DSN` | Database for the same sandbox, following Cloud's billing smoke convention | -| `SMOKE_STRIPE_TEST_KEY` | Stripe `sk_test_` key for that sandbox | -| `TEMPORAL_ADDRESS` | That sandbox's Temporal endpoint | -| `TEMPORAL_NAMESPACE` | That sandbox's Temporal namespace | -| `TEMPORAL_API_KEY` | Required for Temporal Cloud; omit for a local connection | - -For a local frontend connected to test Cloud: - -```sh -pnpm dev:cloud -``` - -In another terminal, run the test with `PLAYWRIGHT_TEST_URL` set to that Vite -server and `PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The frontend -server must proxy to this same backend. For staging or preview, use the matching -backend when starting the dev server, or point both URLs at the deployed sandbox. -Production targets and live Stripe keys are rejected. - -The sandbox must use the Temporal billing engine with checkout-abandonment signal -support. The account must own a personal workspace with inactive billing, no saved cards, -no active or scheduled subscriptions, and no pending billing operation. Provision -this identity separately from shared smoke accounts. The test refuses to repair -pre-existing state. GitHub frontend secrets include `CLOUD_TEST_EMAIL` and -`CLOUD_TEST_PASSWORD`, but their account suitability has not been verified. - -## Cleanup - -The fixture takes a database advisory lock for the workspace. Before allowing -Subscribe, it checks the database state, verifies the Stripe customer is in test -mode, and checks access to the Temporal namespace. A second run using this fixture -cannot use the same workspace concurrently. Other tools must not use this account. - -After the test, including assertion failures, cleanup: - -1. Expires only Stripe test checkout sessions created for this customer during - the run. -2. Finds operations created during the run and requires unpaid initial - subscriptions owned by Temporal. -3. Sends `subscription_checkout_abandoned` to those operations' workflows, waits - for terminal failure, and verifies their subscriptions ended. -4. Restores inactive workspace billing only if no pending operations or non-ended - subscriptions remain, emitting the backend's `projection_dirty` outbox event - in the same transaction. -5. Reads the public billing API again to verify inactive billing and no cards. - -Cleanup refuses unexpected paid state. A cleanup failure fails the run and means -the account requires investigation before reuse. Process termination can prevent -teardown, so do not share the identity. Cleanup is fixture maintenance, not evidence -that the product's timeout or abandonment behavior is correct. - -## Validation status - -The live checkout journey has not yet passed. Local sandbox credentials and -backend cleanup access are still required. Test collection and prerequisite -validation do not establish real Cloud coverage. +Set these variables using the existing `.env` convention or inject them with +`op run --env-file -- pnpm test:browser:cloud-billing`: + +| Variable | Value | +| -------------------------- | ---------------------------------------------------- | +| `PLAYWRIGHT_TEST_URL` | Local frontend URL or deployed sandbox origin | +| `PLAYWRIGHT_SETUP_API_URL` | Matching test, staging, or PR sandbox backend origin | +| `CLOUD_ACCOUNT_EMAIL` | Sandbox account email | +| `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | + +For a local frontend, start `pnpm dev:cloud` and use its Vite URL with +`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The local server must +proxy to the selected sandbox. Production targets are rejected. + +The account must own a personal workspace with no active paid subscription or saved +payment method. Use an account reserved for this test. Do not run concurrent +billing tests against that account. + +The test signs in, checks the displayed amount against the real preview, opens +Stripe test checkout, closes it without entering a card, and resumes the same +pending billing operation. The report attaches screenshots and the operation ID. +Authentication traces and saved credentials are not retained. + +No database, Stripe secret key, or Temporal access is required. Closing the +browser leaves the unpaid checkout pending in the backend. The test does not +reset billing state or claim that the operation expired. A subsequent run may +resume the pending operation if the backend permits it; otherwise use a fresh +no-card test account. Backend expiry and fixture reset remain separate work. + +The first test covers checkout handoff and retry. Payment completion, page-reload +recovery, the 24-hour timeout, and CI integration remain follow-up work. A +successful sign-in alone does not establish billing coverage. diff --git a/package.json b/package.json index 026df7fdeeb8..c4c746866064 100644 --- a/package.json +++ b/package.json @@ -169,7 +169,6 @@ "@storybook/vue3": "catalog:", "@storybook/vue3-vite": "catalog:", "@tailwindcss/vite": "catalog:", - "@temporalio/client": "catalog:", "@testing-library/jest-dom": "catalog:", "@testing-library/user-event": "catalog:", "@testing-library/vue": "catalog:", @@ -177,7 +176,6 @@ "@types/fs-extra": "catalog:", "@types/jsdom": "catalog:", "@types/node": "catalog:", - "@types/pg": "catalog:", "@types/semver": "catalog:", "@types/three": "catalog:", "@vitejs/plugin-vue": "catalog:", @@ -213,7 +211,6 @@ "oxfmt": "catalog:", "oxlint": "catalog:", "oxlint-tsgolint": "catalog:", - "pg": "catalog:", "picocolors": "catalog:", "postcss-html": "catalog:", "pretty-bytes": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d151e2ed8746..e2c411b7b94d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -114,9 +114,6 @@ catalogs: '@tanstack/vue-virtual': specifier: ^3.13.12 version: 3.13.32 - '@temporalio/client': - specifier: ^1.23.0 - version: 1.23.0 '@testing-library/jest-dom': specifier: ^6.9.1 version: 6.9.1 @@ -150,9 +147,6 @@ catalogs: '@types/node': specifier: ^25.9.5 version: 25.9.5 - '@types/pg': - specifier: ^8.23.1 - version: 8.23.1 '@types/semver': specifier: ^7.8.0 version: 7.8.0 @@ -333,9 +327,6 @@ catalogs: parse5: specifier: 8.0.1 version: 8.0.1 - pg: - specifier: ^8.23.0 - version: 8.23.0 picocolors: specifier: ^1.1.1 version: 1.1.1 @@ -740,9 +731,6 @@ importers: '@tailwindcss/vite': specifier: 'catalog:' version: 4.3.2(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0)) - '@temporalio/client': - specifier: 'catalog:' - version: 1.23.0 '@testing-library/jest-dom': specifier: 'catalog:' version: 6.9.1 @@ -764,9 +752,6 @@ importers: '@types/node': specifier: 'catalog:' version: 25.9.5 - '@types/pg': - specifier: 'catalog:' - version: 8.23.1 '@types/semver': specifier: 'catalog:' version: 7.8.0 @@ -872,9 +857,6 @@ importers: oxlint-tsgolint: specifier: 'catalog:' version: 7.0.2001 - pg: - specifier: 'catalog:' - version: 8.23.0 picocolors: specifier: 'catalog:' version: 1.1.1 @@ -2367,10 +2349,6 @@ packages: '@formkit/auto-animate@0.9.0': resolution: {integrity: sha512-VhP4zEAacXS3dfTpJpJ88QdLqMTcabMg0jwpOSxZ/VzfQVfl3GkZSCZThhGC5uhq/TxPHPzW0dzr4H9Bb1OgKA==} - '@grpc/grpc-js@1.14.4': - resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} - engines: {node: '>=12.10.0'} - '@grpc/grpc-js@1.9.16': resolution: {integrity: sha512-wE4Ut/olIzfKqp631XrG+wbF0v1vWFN4YL9FyXC2LJiG33DsV7PLzURjrCvY/6je2ntdRkeLpPDluzSRGaVltQ==} engines: {node: ^8.13.0 || >=10.10.0} @@ -2380,11 +2358,6 @@ packages: engines: {node: '>=6'} hasBin: true - '@grpc/proto-loader@0.8.1': - resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} - engines: {node: '>=6'} - hasBin: true - '@hapi/address@5.1.1': resolution: {integrity: sha512-A+po2d/dVoY7cYajycYI43ZbYMXukuopIsqCjh5QzsBCipDtdofHntljDlpccMjIfTy6UOkg+5KPriwYch2bXA==} engines: {node: '>=14.0.0'} @@ -2686,9 +2659,6 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@js-sdsl/ordered-map@4.4.2': - resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} - '@jsdevtools/ono@7.1.3': resolution: {integrity: sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==} @@ -4125,18 +4095,6 @@ packages: peerDependencies: vue: ^2.7.0 || ^3.0.0 - '@temporalio/client@1.23.0': - resolution: {integrity: sha512-E9nXZPiKDGEDbPpRi0EyQ9i1x70fEIG26aMXZO88kKslKZnEKNPphs06xIic7bhuDoSrPsgVfdDap6o3l6IO2A==} - engines: {node: '>= 20.3.0'} - - '@temporalio/common@1.23.0': - resolution: {integrity: sha512-TaZYb/ruzgzJ8jd42EqRUCI4ispLZETt2Jigc9szES6q9edDmfncRw143HyvPi7b8n9naohYmA/IxTe9COH/dg==} - engines: {node: '>= 20.3.0'} - - '@temporalio/proto@1.23.0': - resolution: {integrity: sha512-90JWnYEs1wyq97/SXkQGknRVMr/BCV9sZ0pOnK3fHTedVinfvz3/PBVFkeAJyU3CJqMQGnzRs9IIfsBTT1icrA==} - engines: {node: '>= 20.3.0'} - '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -4423,9 +4381,6 @@ packages: '@types/node@25.9.5': resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} - '@types/pg@8.23.1': - resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} - '@types/react-dom@19.3.0': resolution: {integrity: sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==} peerDependencies: @@ -5107,10 +5062,6 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - abort-controller@3.0.0: - resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} - engines: {node: '>=6.5'} - acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -6130,10 +6081,6 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} - event-target-shim@5.0.1: - resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} - engines: {node: '>=6'} - eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} @@ -7517,10 +7464,6 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - ms@3.0.0-canary.1: - resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} - engines: {node: '>=12.13'} - muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} @@ -7548,10 +7491,6 @@ packages: new-date@1.0.3: resolution: {integrity: sha512-0fsVvQPbo2I18DT2zVHpezmeeNYV2JaJSrseiHLc17GNOxJzUdx5mvSigPu8LtIfZSij5i1wXnXFspEs2CD6hA==} - nexus-rpc@0.0.3: - resolution: {integrity: sha512-ksIBeYHDv6FHAR1h3Vwk0kQ6REhdo1STqDHXMzu/yn/VN/XzgOor2IUjewG6paoA7IgxlIPZoOU/PD78PC6HTg==} - engines: {node: '>= 20.0.0'} - nlcst-to-string@4.0.0: resolution: {integrity: sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==} @@ -7826,40 +7765,6 @@ packages: perfect-debounce@2.1.0: resolution: {integrity: sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g==} - pg-cloudflare@1.4.0: - resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} - - pg-connection-string@2.14.0: - resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} - - pg-int8@1.0.1: - resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} - engines: {node: '>=4.0.0'} - - pg-pool@3.14.0: - resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} - peerDependencies: - pg: '>=8.0' - - pg-protocol@1.16.0: - resolution: {integrity: sha512-sILXutLVjCLjcDuOmvhX5e2Z4cS5qG/6Bu3VkpFwdf/633ElGLpEh9bgmuI5I4sqKqkifQiGyiCcx1HdtrK7tg==} - - pg-types@2.2.0: - resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} - engines: {node: '>=4'} - - pg@8.23.0: - resolution: {integrity: sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==} - engines: {node: '>= 16.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - - pgpass@1.0.5: - resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} - piccolore@0.1.3: resolution: {integrity: sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==} @@ -7934,22 +7839,6 @@ packages: resolution: {integrity: sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==} engines: {node: ^10 || ^12 || >=14} - postgres-array@2.0.0: - resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} - engines: {node: '>=4'} - - postgres-bytea@1.0.1: - resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} - engines: {node: '>=0.10.0'} - - postgres-date@1.0.7: - resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} - engines: {node: '>=0.10.0'} - - postgres-interval@1.2.0: - resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} - engines: {node: '>=0.10.0'} - posthog-js@1.402.3: resolution: {integrity: sha512-MCn4O9wJc1BGBbb820MlmYifDb6EhFBFhfR1K9294gfgFRcxin+NSlypswVBdt1ZO4XD/P7EPmwApyVX/EYlUw==} @@ -8059,10 +7948,6 @@ packages: resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} engines: {node: '>=12.0.0'} - protobufjs@8.8.0: - resolution: {integrity: sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==} - engines: {node: '>=12.0.0'} - proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -8504,10 +8389,6 @@ packages: resolution: {integrity: sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==} engines: {node: '>=0.10.0'} - split2@4.2.0: - resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} - engines: {node: '>= 10.x'} - sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} @@ -9692,10 +9573,6 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} - xtend@4.0.2: - resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} - engines: {node: '>=0.4'} - xxhash-wasm@1.1.0: resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} @@ -11060,11 +10937,6 @@ snapshots: '@formkit/auto-animate@0.9.0': {} - '@grpc/grpc-js@1.14.4': - dependencies: - '@grpc/proto-loader': 0.8.1 - '@js-sdsl/ordered-map': 4.4.2 - '@grpc/grpc-js@1.9.16': dependencies: '@grpc/proto-loader': 0.7.15 @@ -11077,13 +10949,6 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 - '@grpc/proto-loader@0.8.1': - dependencies: - lodash.camelcase: 4.3.0 - long: 5.3.2 - protobufjs: 7.6.5 - yargs: 17.7.3 - '@hapi/address@5.1.1': dependencies: '@hapi/hoek': 11.0.7 @@ -11390,8 +11255,6 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 - '@js-sdsl/ordered-map@4.4.2': {} - '@jsdevtools/ono@7.1.3': {} '@keyv/bigmap@1.3.1(keyv@5.6.0)': @@ -12503,29 +12366,6 @@ snapshots: '@tanstack/virtual-core': 3.17.4 vue: 3.5.42(typescript@6.0.3) - '@temporalio/client@1.23.0': - dependencies: - '@grpc/grpc-js': 1.14.4 - '@temporalio/common': 1.23.0 - '@temporalio/proto': 1.23.0 - abort-controller: 3.0.0 - long: 5.3.2 - nexus-rpc: 0.0.3 - uuid: 11.1.1 - - '@temporalio/common@1.23.0': - dependencies: - '@temporalio/proto': 1.23.0 - long: 5.3.2 - ms: 3.0.0-canary.1 - nexus-rpc: 0.0.3 - protobufjs: 8.8.0 - - '@temporalio/proto@1.23.0': - dependencies: - long: 5.3.2 - protobufjs: 8.8.0 - '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -12847,12 +12687,6 @@ snapshots: dependencies: undici-types: 7.24.6 - '@types/pg@8.23.1': - dependencies: - '@types/node': 25.9.5 - pg-protocol: 1.16.0 - pg-types: 2.2.0 - '@types/react-dom@19.3.0(@types/react@19.3.0)': dependencies: '@types/react': 19.3.0 @@ -13548,10 +13382,6 @@ snapshots: abbrev@2.0.0: {} - abort-controller@3.0.0: - dependencies: - event-target-shim: 5.0.1 - acorn-jsx@5.3.2(acorn@8.17.0): dependencies: acorn: 8.17.0 @@ -14706,8 +14536,6 @@ snapshots: esutils@2.0.3: {} - event-target-shim@5.0.1: {} - eventemitter3@5.0.4: {} eventsource-parser@4.1.0: {} @@ -16458,8 +16286,6 @@ snapshots: ms@2.1.3: {} - ms@3.0.0-canary.1: {} - muggle-string@0.4.1: {} multi-buffer-data-view@6.0.27: @@ -16479,8 +16305,6 @@ snapshots: dependencies: '@segment/isodate': 1.0.3 - nexus-rpc@0.0.3: {} - nlcst-to-string@4.0.0: dependencies: '@types/nlcst': 2.0.3 @@ -16860,41 +16684,6 @@ snapshots: perfect-debounce@2.1.0: {} - pg-cloudflare@1.4.0: - optional: true - - pg-connection-string@2.14.0: {} - - pg-int8@1.0.1: {} - - pg-pool@3.14.0(pg@8.23.0): - dependencies: - pg: 8.23.0 - - pg-protocol@1.16.0: {} - - pg-types@2.2.0: - dependencies: - pg-int8: 1.0.1 - postgres-array: 2.0.0 - postgres-bytea: 1.0.1 - postgres-date: 1.0.7 - postgres-interval: 1.2.0 - - pg@8.23.0: - dependencies: - pg-connection-string: 2.14.0 - pg-pool: 3.14.0(pg@8.23.0) - pg-protocol: 1.16.0 - pg-types: 2.2.0 - pgpass: 1.0.5 - optionalDependencies: - pg-cloudflare: 1.4.0 - - pgpass@1.0.5: - dependencies: - split2: 4.2.0 - piccolore@0.1.3: {} picocolors@1.1.1: {} @@ -16963,16 +16752,6 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postgres-array@2.0.0: {} - - postgres-bytea@1.0.1: {} - - postgres-date@1.0.7: {} - - postgres-interval@1.2.0: - dependencies: - xtend: 4.0.2 - posthog-js@1.402.3: dependencies: '@posthog/core': 1.42.1 @@ -17123,10 +16902,6 @@ snapshots: '@types/node': 25.9.5 long: 5.3.2 - protobufjs@8.8.0: - dependencies: - long: 5.3.2 - proxy-from-env@1.1.0: {} proxy-from-env@2.1.0: {} @@ -17717,8 +17492,6 @@ snapshots: speakingurl@14.0.1: {} - split2@4.2.0: {} - sprintf-js@1.0.3: optional: true @@ -18968,8 +18741,6 @@ snapshots: xmlchars@2.2.0: {} - xtend@4.0.2: {} - xxhash-wasm@1.1.0: {} y18n@5.0.8: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 833f2bbf9973..49b01eef5b83 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -47,7 +47,6 @@ catalog: '@stripe/stripe-js': ^9.12.0 '@tailwindcss/vite': ^4.3.2 '@tanstack/vue-virtual': ^3.13.12 - '@temporalio/client': ^1.23.0 '@testing-library/jest-dom': ^6.9.1 '@testing-library/user-event': ^14.6.1 '@testing-library/vue': ^8.1.0 @@ -60,7 +59,6 @@ catalog: '@types/jsdom': ^21.1.7 '@types/leaflet': ^1.9.20 '@types/node': ^25.9.5 - '@types/pg': ^8.23.1 '@types/semver': ^7.8.0 '@types/three': ^0.184.1 '@vee-validate/zod': ^4.15.1 @@ -121,7 +119,6 @@ catalog: oxlint: ^1.77.0 oxlint-tsgolint: ^7.0.2001 parse5: 8.0.1 - pg: ^8.23.0 picocolors: ^1.1.1 pinia: ^4.0.2 postcss-html: ^1.8.0 From 97f6b6cd61e2915e15ba1b160beeb32ed08fb3e2 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:27:16 -0700 Subject: [PATCH 08/23] test: use pricing entry for no-card checkout --- browser_tests/fixtures/liveCloudBillingFixture.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index cde5b115f3d2..498f88877b2d 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -132,9 +132,7 @@ export const liveCloudBillingFixture = base.extend<{ (response) => new URL(response.url()).pathname === '/api/billing/preview-subscribe' ), - page.goto( - `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/subscribe?tier=creator&cycle=monthly` - ) + page.goto(`${sandbox.PLAYWRIGHT_TEST_URL}/?pricing=creator&cycle=monthly`) ]) expect(previewResponse.status()).toBe(200) const preview = zPreviewSubscribeResponse.parse( From bac5882fc2a942af51febe2ee4c96f69cfc09f5b Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:33:19 -0700 Subject: [PATCH 09/23] test: skip survey in local Cloud billing checkout --- browser_tests/fixtures/liveCloudBillingFixture.ts | 4 ++++ docs/testing/cloud-billing-e2e.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 498f88877b2d..50e71cbcde02 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -12,6 +12,7 @@ import { comfyExpect as expect, comfyPageFixture as base } from '@e2e/fixtures/ComfyPage' +import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' interface BillingSandbox { @@ -61,6 +62,9 @@ export const liveCloudBillingFixture = base.extend<{ }, billingSandbox: async ({ page }, use, testInfo) => { const sandbox = loadLiveCloudBillingConfig() + await new FeatureFlagHelper(page).seedFlags({ + onboarding_survey_enabled: false + }) const documentResponse = await page.goto( `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/login` ) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index a2cfbc15ca3c..42f0f556a290 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -21,6 +21,10 @@ For a local frontend, start `pnpm dev:cloud` and use its Vite URL with `PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The local server must proxy to the selected sandbox. Production targets are rejected. +The fixture disables `onboarding_survey_enabled` through the existing dev-only +feature-flag helper before navigation. Deployed builds ignore this override, so +accounts used against a deployed frontend must have completed the survey. + The account must own a personal workspace with no active paid subscription or saved payment method. Use an account reserved for this test. Do not run concurrent billing tests against that account. From d7f6e9d0cbd3e73fad373caf0dac5f4c0ebb62bc Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:40:04 -0700 Subject: [PATCH 10/23] test: match live checkout preview and sandbox host --- browser_tests/fixtures/liveCloudBillingFixture.ts | 1 + browser_tests/tests/liveCloud/checkoutRecovery.spec.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 50e71cbcde02..492872fc7929 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -45,6 +45,7 @@ export const liveCloudBillingFixture = base.extend<{ 'https://securetoken.googleapis.com', 'https://dreamboothy-dev.firebaseapp.com', 'https://checkout.stripe.com', + 'https://checkout.comfy.org', 'https://api.stripe.com', 'https://js.stripe.com', 'https://m.stripe.network', diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts index 351ffe0aaad9..f1e94021ecc0 100644 --- a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -5,7 +5,8 @@ import { liveCloudBillingFixture as test } from '@e2e/fixtures/liveCloudBillingFixture' -const testCheckoutUrl = /^https:\/\/checkout\.stripe\.com\/c\/pay\/cs_test_/ +const testCheckoutUrl = + /^https:\/\/checkout\.(?:stripe\.com|comfy\.org)\/c\/pay\/cs_test_/ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { test('resumes an abandoned no-card checkout using the same billing operation', async ({ @@ -23,8 +24,12 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { Number(preview.amount_due_cents ?? preview.cost_today_cents) / 100 ) + const confirmation = comfyPage.page.getByRole('dialog') await expect( - comfyPage.page.getByText(`Total due today ${amountDue}`, { exact: true }) + confirmation.getByText('Total due today', { exact: true }) + ).toBeVisible() + await expect( + confirmation.getByText(amountDue, { exact: true }) ).toBeVisible() await testInfo.attach('preview.png', { From b1e56809c40ac8152cba2d8f95eb78ae237d8a95 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:49:56 -0700 Subject: [PATCH 11/23] test: address live billing safety and repeatability review --- browser_tests/README.md | 2 +- .../fixtures/helpers/LiveCloudBilling.ts | 185 +++++++++++++++ .../fixtures/liveCloudBillingFixture.ts | 216 ++++++++---------- .../utils/liveCloudBillingConfig.test.ts | 26 +++ .../fixtures/utils/liveCloudBillingConfig.ts | 13 +- browser_tests/playwrightSelection.test.ts | 58 +++++ .../tests/liveCloud/checkoutRecovery.spec.ts | 136 +---------- ...-BILLING-0031-real-cloud-checkout-tests.md | 5 + docs/testing/cloud-billing-e2e.md | 11 +- playwright.config.ts | 18 +- 10 files changed, 407 insertions(+), 263 deletions(-) create mode 100644 browser_tests/fixtures/helpers/LiveCloudBilling.ts create mode 100644 browser_tests/playwrightSelection.test.ts diff --git a/browser_tests/README.md b/browser_tests/README.md index 94f59d462645..27f67636f985 100644 --- a/browser_tests/README.md +++ b/browser_tests/README.md @@ -24,7 +24,7 @@ to a section below. For real Cloud billing checkout recovery, sandbox setup, and remaining release criteria, see [Cloud billing E2E coverage](../docs/testing/cloud-billing-e2e.md). -Its dedicated runner requires real authentication and a backend reset hook; +Its dedicated runner requires real authentication and a dedicated no-card account; the live suite is excluded from the ordinary browser test configurations. **CRITICAL**: Start the ComfyUI backend with `--multi-user`: diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts new file mode 100644 index 000000000000..704cd658165d --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -0,0 +1,185 @@ +import { + zBillingOpStatusResponse, + zBillingStatusResponse, + zCurrentWorkspaceResponse, + zListSavedPaymentMethodsResponse, + zPreviewSubscribeResponse, + zSubscribeResponse +} from '@comfyorg/ingest-types/zod' +import type { + APIRequestContext, + Page, + Response, + TestInfo +} from '@playwright/test' +import { expect } from '@playwright/test' +import type { z } from 'zod' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' + +export class LiveCloudBillingSession { + constructor( + private readonly request: APIRequestContext, + private readonly backend: string, + private readonly headers: Record + ) {} + + async read(path: string, schema: z.ZodType): Promise { + const response = await this.request.get(new URL(path, this.backend).href, { + headers: this.headers, + maxRedirects: 0 + }) + expect(response.status(), `GET ${path}`).toBe(200) + return schema.parse(await response.json()) + } + + async assertNoCardAccount(testInfo: TestInfo) { + const workspace = await this.read( + '/api/workspaces/current', + zCurrentWorkspaceResponse + ) + expect(workspace.type).toBe('personal') + expect(workspace.role).toBe('owner') + const status = await this.read( + '/api/billing/status', + zBillingStatusResponse + ) + const methods = await this.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + await testInfo.attach('billing-preflight.json', { + body: JSON.stringify({ + workspaceId: workspace.id, + isActive: status.is_active, + billingStatus: status.billing_status, + subscriptionTier: status.subscription_tier, + paymentMethodCount: methods.length, + backend: this.backend + }), + contentType: 'application/json' + }) + if (status.is_active) expect(status.subscription_tier).toBe('FREE') + else + expect(['inactive', 'awaiting_payment_method']).toContain( + status.billing_status + ) + expect(methods).toHaveLength(0) + } + + async expectPending(operationId: string) { + await expect + .poll(() => + this.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + ) + .toMatchObject({ + id: operationId, + status: 'pending', + phase: 'awaiting_payment_method' + }) + } +} + +export function matchesBillingResponse( + response: Response, + frontend: string, + path: string +) { + return response.url() === new URL(path, frontend).href +} + +export class LiveCloudCheckout { + private readonly page: Page + readonly confirmation + readonly resumePayment + private readonly subscribe + + constructor( + private readonly comfyPage: ComfyPage, + private readonly frontend: string + ) { + this.page = comfyPage.page + this.confirmation = this.page.getByRole('dialog').filter({ + has: this.page.getByRole('heading', { + name: 'Confirm your payment', + exact: true + }) + }) + this.resumePayment = this.confirmation.getByRole('button', { + name: 'Complete your payment', + exact: true + }) + this.subscribe = this.confirmation.getByRole('button', { + name: 'Subscribe to Creator', + exact: true + }) + } + + async open() { + const [response] = await Promise.all([ + this.page.waitForResponse((response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/preview-subscribe' + ) + ), + this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) + ]) + expect(response.status()).toBe(200) + const preview = zPreviewSubscribeResponse.parse(await response.json()) + expect(preview.allowed).toBe(true) + expect(preview.new_plan.slug).toBe('creator-monthly') + await expect( + this.confirmation.getByText('Total due today', { exact: true }) + ).toBeVisible() + await expect(this.subscribe.or(this.resumePayment)).toBeEnabled() + } + + async abandonCheckout(testInfo: TestInfo, resume = false) { + const action = resume + ? this.resumePayment + : this.subscribe.or(this.resumePayment) + await expect(action).toBeEnabled() + const [response, checkout] = await Promise.all([ + this.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/subscribe' + ) && response.request().method() === 'POST' + ), + this.page.waitForEvent('popup'), + action.click() + ]) + expect(response.status()).toBe(200) + const subscription = zSubscribeResponse.parse(await response.json()) + expect(subscription.status).toBe('needs_payment_method') + expect(subscription.billing_op_id).not.toBe('') + const testCheckoutUrl = + /^https:\/\/checkout\.(?:stripe\.com|comfy\.org)\/c\/pay\/cs_test_/ + expect(subscription.payment_method_url).toMatch(testCheckoutUrl) + await expect(checkout).toHaveURL(testCheckoutUrl) + await testInfo.attach('billing-operation.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + status: subscription.status + }), + contentType: 'application/json' + }) + await testInfo.attach('stripe-checkout.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await checkout.close() + return subscription.billing_op_id + } + + async attachScreenshot(name: string) { + await this.comfyPage.attachScreenshot(name, { runInCI: true }) + } +} diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 492872fc7929..14503efdfb21 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -1,46 +1,26 @@ -import { - zBillingOpStatusResponse, - zBillingStatusResponse, - zCurrentWorkspaceResponse, - zListSavedPaymentMethodsResponse, - zPreviewSubscribeResponse -} from '@comfyorg/ingest-types/zod' -import type { z } from 'zod' +import { expect } from '@playwright/test' -import { - ComfyPage, - comfyExpect as expect, - comfyPageFixture as base -} from '@e2e/fixtures/ComfyPage' +import { ComfyPage } from '@e2e/fixtures/ComfyPage' import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { + LiveCloudBillingSession, + LiveCloudCheckout, + matchesBillingResponse +} from '@e2e/fixtures/helpers/LiveCloudBilling' +import { networkIsolationFixture as base } from '@e2e/fixtures/networkIsolationFixture' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' -interface BillingSandbox { - preview: z.infer - readOperation: ( - operationId: string - ) => Promise> -} - export const liveCloudBillingFixture = base.extend<{ - billingSandbox: BillingSandbox + billingSession: LiveCloudBillingSession + comfyPage: ComfyPage + checkout: LiveCloudCheckout }>({ baseURL: process.env.PLAYWRIGHT_TEST_URL, - networkPolicy: async ({ baseURL }, use) => { - const sandbox = loadLiveCloudBillingConfig() + networkPolicy: async ({ baseURL }, use, testInfo) => { + const config = loadLiveCloudBillingConfig() const origins = new Set([ - new URL(baseURL ?? sandbox.PLAYWRIGHT_TEST_URL).origin, - sandbox.PLAYWRIGHT_SETUP_API_URL, - 'https://testapi.comfy.org', - 'https://t.comfy.org', - 'https://mp.comfy.org', - 'https://browser-intake-us5-datadoghq.com', - 'https://cdn.sy-d.io', - 'https://e2.sy-d.io', - 'https://consumer.cloud.gist.build', - 'https://realtime.cloud.gist.build', - 'https://cdp.customer.io', - 'https://o4507954455314432.ingest.us.sentry.io', + new URL(baseURL ?? config.PLAYWRIGHT_TEST_URL).origin, + config.PLAYWRIGHT_SETUP_API_URL, 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', 'https://dreamboothy-dev.firebaseapp.com', @@ -56,117 +36,105 @@ export const liveCloudBillingFixture = base.extend<{ ]) const unexpected = new Set() await use({ origins, unexpected }) + const blocked = [...unexpected] + await testInfo.attach('blocked-egress.json', { + body: JSON.stringify(blocked), + contentType: 'application/json' + }) expect( - [...unexpected], - 'Add the required sandbox dependency origin' + blocked.filter((entry) => /^(API|Navigation) /.test(entry)), + 'Unexpected API or navigation destination' ).toEqual([]) }, - billingSandbox: async ({ page }, use, testInfo) => { - const sandbox = loadLiveCloudBillingConfig() + context: async ({ context, networkPolicy }, use) => { + const config = loadLiveCloudBillingConfig() + await context.route('**/*', async (route) => { + const request = route.request() + const url = new URL(request.url()) + if ( + !networkPolicy.origins.has(url.origin) && + url.hostname.endsWith('.comfy.org') && + /^\/(api|customers)(\/|$)/.test(url.pathname) + ) { + networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + request.isNavigationRequest() && + !networkPolicy.origins.has(url.origin) + ) { + networkPolicy.unexpected.add(`Navigation ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + url.origin === config.PLAYWRIGHT_TEST_URL && + /^\/(api|internal)(\/|$)/.test(url.pathname) + ) { + const response = await route.fetch({ + url: new URL( + url.pathname + url.search, + config.PLAYWRIGHT_SETUP_API_URL + ).href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + return + } + await route.fallback() + }) + await use(context) + }, + billingSession: async ({ page }, use) => { + const config = loadLiveCloudBillingConfig() await new FeatureFlagHelper(page).seedFlags({ onboarding_survey_enabled: false }) - const documentResponse = await page.goto( - `${sandbox.PLAYWRIGHT_TEST_URL}/cloud/login` - ) + await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) await page .getByRole('button', { name: 'Use email instead', exact: true }) .click() await page .getByRole('textbox', { name: 'Email', exact: true }) - .fill(sandbox.CLOUD_ACCOUNT_EMAIL) + .fill(config.CLOUD_ACCOUNT_EMAIL) await page .getByLabel('Password', { exact: true }) - .fill(sandbox.CLOUD_ACCOUNT_PASSWORD) - const [billingResponse] = await Promise.all([ + .fill(config.CLOUD_ACCOUNT_PASSWORD) + const [response] = await Promise.all([ page.waitForResponse( (response) => - new URL(response.url()).pathname === '/api/billing/status' && - response.status() === 200 + matchesBillingResponse( + response, + config.PLAYWRIGHT_TEST_URL, + '/api/billing/status' + ) && response.status() === 200 ), page.getByRole('button', { name: 'Sign in', exact: true }).click() ]) - const headers = await billingResponse.request().allHeaders() - - async function read(path: string, schema: z.ZodType): Promise { - const response = await page.request.get( - new URL(path, sandbox.PLAYWRIGHT_TEST_URL).href, - { headers } + const authorization = await response.request().headerValue('authorization') + expect(authorization).toBeTruthy() + if (!authorization) throw new Error('Missing billing authorization') + await use( + new LiveCloudBillingSession( + page.request, + config.PLAYWRIGHT_SETUP_API_URL, + { authorization } ) - expect(response.status(), `GET ${path}`).toBe(200) - return schema.parse(await response.json()) - } - - const workspace = await read( - '/api/workspaces/current', - zCurrentWorkspaceResponse ) - expect(workspace.type).toBe('personal') - expect(workspace.role).toBe('owner') - - async function assertNoCardAccount() { - const [status, methods] = await Promise.all([ - read('/api/billing/status', zBillingStatusResponse), - read('/api/billing/payment-methods', zListSavedPaymentMethodsResponse) - ]) - await testInfo.attach('billing-preflight.json', { - body: JSON.stringify({ - workspaceId: workspace.id, - isActive: status.is_active, - billingStatus: status.billing_status, - subscriptionTier: status.subscription_tier, - planSlug: status.plan_slug, - paymentMethodCount: methods.length - }), - contentType: 'application/json' - }) - if (status.is_active) { - expect(status.subscription_tier).toBe('FREE') - } else { - expect(['inactive', 'awaiting_payment_method']).toContain( - status.billing_status - ) - } - expect(methods).toHaveLength(0) - } - - await assertNoCardAccount() - - const [previewResponse] = await Promise.all([ - page.waitForResponse( - (response) => - new URL(response.url()).pathname === '/api/billing/preview-subscribe' - ), - page.goto(`${sandbox.PLAYWRIGHT_TEST_URL}/?pricing=creator&cycle=monthly`) - ]) - expect(previewResponse.status()).toBe(200) - const preview = zPreviewSubscribeResponse.parse( - await previewResponse.json() - ) - expect(preview.allowed).toBe(true) - expect(preview.new_plan.slug).toBe('creator-monthly') - await testInfo.attach('sandbox.json', { - body: JSON.stringify({ - baseURL: sandbox.PLAYWRIGHT_TEST_URL, - workspaceId: workspace.id, - frontendVersion: - documentResponse?.headers()['x-frontend-version'] ?? null - }), - contentType: 'application/json' - }) - await use({ - preview, - readOperation: (operationId) => - read( - `/api/billing/ops/${encodeURIComponent(operationId)}`, - zBillingOpStatusResponse - ) - }) }, - comfyPage: async ({ page, request, billingSandbox }, use) => { - expect(billingSandbox.preview.allowed).toBe(true) + comfyPage: async ({ page, request }, use) => { await use(new ComfyPage(page, request)) + }, + checkout: async ({ comfyPage, billingSession }, use, testInfo) => { + await billingSession.assertNoCardAccount(testInfo) + const checkout = new LiveCloudCheckout( + comfyPage, + loadLiveCloudBillingConfig().PLAYWRIGHT_TEST_URL + ) + await checkout.open() + await use(checkout) } }) -export { comfyExpect } from '@e2e/fixtures/ComfyPage' +export { expect as comfyExpect } from '@playwright/test' diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts index 99c34841f172..f4f360dc08c1 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.test.ts @@ -61,3 +61,29 @@ describe('Live Cloud billing prerequisites', () => { expect(loadLiveCloudBillingConfig).not.toThrow('private-value') }) }) + +describe('Frontend origins', () => { + it.for([ + 'http://localhost:5173/app', + 'http://localhost:5173/?x=1', + 'http://localhost:5173/#x', + 'http://user:pass@localhost:5173' + ])('rejects non-origin frontend %s', (PLAYWRIGHT_TEST_URL) => { + expect( + liveCloudBillingConfigSchema.safeParse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL + }).success + ).toBe(false) + }) + + it('normalizes root trailing slashes on both origins', () => { + const config = liveCloudBillingConfigSchema.parse({ + ...sandboxConfig, + PLAYWRIGHT_TEST_URL: 'http://localhost:5173/', + PLAYWRIGHT_SETUP_API_URL: 'https://testcloud.comfy.org/' + }) + expect(config.PLAYWRIGHT_TEST_URL).toBe('http://localhost:5173') + expect(config.PLAYWRIGHT_SETUP_API_URL).toBe('https://testcloud.comfy.org') + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index 62cdb31a9ce4..cb382c4d1906 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -8,16 +8,25 @@ const sandboxURL = z const url = new URL(value) return ( url.protocol === 'https:' && - url.origin === value && + url.href === `${url.origin}/` && (url.hostname === 'testcloud.comfy.org' || url.hostname === 'stagingcloud.comfy.org' || /^pr-\d+\.testenvs\.comfy\.org$/.test(url.hostname)) ) }, 'Use a Cloud test, staging, or PR preview origin') + .transform((value) => new URL(value).origin) export const liveCloudBillingConfigSchema = z .object({ - PLAYWRIGHT_TEST_URL: z.string().url(), + PLAYWRIGHT_TEST_URL: z + .string() + .url() + .refine((value) => { + if (!URL.canParse(value)) return false + const url = new URL(value) + return url.href === `${url.origin}/` + }, 'Use an origin without a path, query, credentials, or fragment') + .transform((value) => new URL(value).origin), PLAYWRIGHT_SETUP_API_URL: sandboxURL, CLOUD_ACCOUNT_EMAIL: z.string().email(), CLOUD_ACCOUNT_PASSWORD: z.string().min(1) diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts new file mode 100644 index 000000000000..a8d88bf5dd63 --- /dev/null +++ b/browser_tests/playwrightSelection.test.ts @@ -0,0 +1,58 @@ +import { spawnSync } from 'node:child_process' + +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +describe('Live billing opt-in', () => { + it.for(['0', '1'])( + 'collects live billing only when explicitly enabled (%s)', + { timeout: 90_000 }, + (enabled) => { + const result = spawnSync( + 'pnpm', + [ + 'exec', + 'playwright', + 'test', + ...(enabled === '1' ? ['--project=cloud-live'] : ['tests/liveCloud']), + '--list', + '--reporter=json' + ], + { + cwd: process.cwd(), + env: { + ...process.env, + PLAYWRIGHT_CLOUD_LIVE: enabled, + DISTRIBUTION: 'cloud' + }, + encoding: 'utf8', + timeout: 60_000, + maxBuffer: 10_000_000 + } + ) + expect(result.error).toBeUndefined() + expect(result.status).toBe(enabled === '1' ? 0 : 1) + const report = z + .object({ + suites: z.array( + z.object({ + suites: z.array(z.object({ specs: z.array(z.unknown()) })) + }) + ), + config: z.object({ + globalSetup: z.unknown(), + globalTeardown: z.unknown() + }) + }) + .parse(JSON.parse(result.stdout)) + const specs = report.suites.flatMap((suite) => + suite.suites.flatMap((child) => child.specs) + ) + expect(specs).toHaveLength(enabled === '1' ? 1 : 0) + if (enabled === '1') { + expect(report.config.globalSetup).toBeNull() + expect(report.config.globalTeardown).toBeNull() + } + } + ) +}) diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts index f1e94021ecc0..24a5f1644c17 100644 --- a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -1,134 +1,22 @@ -import { zSubscribeResponse } from '@comfyorg/ingest-types/zod' - import { comfyExpect as expect, liveCloudBillingFixture as test } from '@e2e/fixtures/liveCloudBillingFixture' -const testCheckoutUrl = - /^https:\/\/checkout\.(?:stripe\.com|comfy\.org)\/c\/pay\/cs_test_/ - test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { test('resumes an abandoned no-card checkout using the same billing operation', async ({ - comfyPage, - billingSandbox + checkout, + billingSession }, testInfo) => { - const preview = billingSandbox.preview - const currency = - preview.amount_due_cents === undefined ? 'USD' : preview.currency - if (!currency) throw new Error('The live billing preview has no currency') - const amountDue = new Intl.NumberFormat('en-US', { - style: 'currency', - currency - }).format( - Number(preview.amount_due_cents ?? preview.cost_today_cents) / 100 - ) - - const confirmation = comfyPage.page.getByRole('dialog') - await expect( - confirmation.getByText('Total due today', { exact: true }) - ).toBeVisible() - await expect( - confirmation.getByText(amountDue, { exact: true }) - ).toBeVisible() - - await testInfo.attach('preview.png', { - body: await comfyPage.page.screenshot(), - contentType: 'image/png' - }) - - const [subscribeResponse, checkout] = await Promise.all([ - comfyPage.page.waitForResponse( - (response) => - new URL(response.url()).pathname === '/api/billing/subscribe' && - response.request().method() === 'POST' - ), - comfyPage.page.waitForEvent('popup'), - comfyPage.page - .getByRole('button', { name: 'Subscribe to Creator', exact: true }) - .click() - ]) - expect(subscribeResponse.status()).toBe(200) - const subscription = zSubscribeResponse.parse( - await subscribeResponse.json() - ) - await testInfo.attach('billing-operation.json', { - body: JSON.stringify({ - operationId: subscription.billing_op_id, - status: subscription.status - }), - contentType: 'application/json' - }) - expect(subscription.status).toBe('needs_payment_method') - expect(subscription.billing_op_id).not.toBe('') - expect(subscription.payment_method_url).toMatch(testCheckoutUrl) - await expect(checkout).toHaveURL(testCheckoutUrl) - await testInfo.attach('stripe-checkout.png', { - body: await checkout.screenshot(), - contentType: 'image/png' - }) - await checkout.close() - - await expect - .poll(async () => { - const operation = await billingSandbox.readOperation( - subscription.billing_op_id - ) - return { - id: operation.id, - status: operation.status, - phase: operation.phase - } - }) - .toEqual({ - id: subscription.billing_op_id, - status: 'pending', - phase: 'awaiting_payment_method' - }) - - const resumePayment = comfyPage.page.getByRole('button', { - name: 'Complete your payment', - exact: true - }) - await expect(resumePayment).toBeEnabled() - await testInfo.attach('resume-payment.png', { - body: await comfyPage.page.screenshot(), - contentType: 'image/png' - }) - const [resumeResponse, resumedCheckout] = await Promise.all([ - comfyPage.page.waitForResponse( - (response) => - new URL(response.url()).pathname === '/api/billing/subscribe' && - response.request().method() === 'POST' - ), - comfyPage.page.waitForEvent('popup'), - resumePayment.click() - ]) - expect(resumeResponse.status()).toBe(200) - const resumedSubscription = zSubscribeResponse.parse( - await resumeResponse.json() - ) - expect(resumedSubscription.status).toBe('needs_payment_method') - expect(resumedSubscription.billing_op_id).toBe(subscription.billing_op_id) - expect(resumedSubscription.payment_method_url).toMatch(testCheckoutUrl) - await expect(resumedCheckout).toHaveURL(testCheckoutUrl) - await resumedCheckout.close() - await expect(resumePayment).toBeEnabled() - await expect - .poll(async () => { - const operation = await billingSandbox.readOperation( - subscription.billing_op_id - ) - return { - id: operation.id, - status: operation.status, - phase: operation.phase - } - }) - .toEqual({ - id: subscription.billing_op_id, - status: 'pending', - phase: 'awaiting_payment_method' - }) + await checkout.attachScreenshot('preview.png') + const operationId = await checkout.abandonCheckout(testInfo) + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await checkout.attachScreenshot('resume-payment.png') + + const resumedId = await checkout.abandonCheckout(testInfo, true) + expect(resumedId).toBe(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await billingSession.expectPending(operationId) }) }) diff --git a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md index 801ecbf39cc9..0106c83b8132 100644 --- a/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md +++ b/docs/adr/TESTING-BILLING-0031-real-cloud-checkout-tests.md @@ -31,8 +31,13 @@ can be added separately when required for repeatable CI. ## Consequences +### Positive + - The browser test requires only sandbox URLs and account credentials. - No payment is submitted and no backend billing state is reset. + +### Negative + - Unpaid operations can remain pending after a run. Account reuse depends on the backend's ability to resume that state, and concurrent runs must be avoided. - Cleanup is not a substitute for an assertion about product recovery. diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 42f0f556a290..778711922ad4 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -18,8 +18,8 @@ Set these variables using the existing `.env` convention or inject them with | `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | For a local frontend, start `pnpm dev:cloud` and use its Vite URL with -`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The local server must -proxy to the selected sandbox. Production targets are rejected. +`PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The test routes browser API requests and its API checks directly to the selected +sandbox, independently of the local Vite proxy. Production targets are rejected. The fixture disables `onboarding_survey_enabled` through the existing dev-only feature-flag helper before navigation. Deployed builds ignore this override, so @@ -29,16 +29,15 @@ The account must own a personal workspace with no active paid subscription or sa payment method. Use an account reserved for this test. Do not run concurrent billing tests against that account. -The test signs in, checks the displayed amount against the real preview, opens +The test signs in, checks the real preview and payment dialog are ready, opens Stripe test checkout, closes it without entering a card, and resumes the same pending billing operation. The report attaches screenshots and the operation ID. Authentication traces and saved credentials are not retained. No database, Stripe secret key, or Temporal access is required. Closing the browser leaves the unpaid checkout pending in the backend. The test does not -reset billing state or claim that the operation expired. A subsequent run may -resume the pending operation if the backend permits it; otherwise use a fresh -no-card test account. Backend expiry and fixture reset remain separate work. +reset billing state or claim that the operation expired. The first action accepts either a fresh Subscribe button or the pending +Complete your payment button, so sequential runs can resume an existing checkout. Backend expiry and fixture reset remain separate work. The first test covers checkout handoff and retry. Payment completion, page-reload recovery, the 24-hour timeout, and CI integration remain follow-up work. A diff --git a/playwright.config.ts b/playwright.config.ts index 50aa8b30e5b5..b8c3c6384ffb 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -57,8 +57,14 @@ export default defineConfig({ reporter: process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR ? 'blob' : 'html', ...maybeLocalOptions, - globalSetup: './browser_tests/globalSetup.ts', - globalTeardown: './browser_tests/globalTeardown.ts', + globalSetup: + process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? undefined + : './browser_tests/globalSetup.ts', + globalTeardown: + process.env.PLAYWRIGHT_CLOUD_LIVE === '1' + ? undefined + : './browser_tests/globalTeardown.ts', projects: [ ...(process.env.PLAYWRIGHT_CLOUD_LIVE === '1' @@ -74,11 +80,11 @@ export default defineConfig({ use: { ...devices['Desktop Chrome'], locale: 'en-US', - trace: 'off' as const, - video: 'off' as const, - screenshot: 'off' as const + trace: 'off', + video: 'off', + screenshot: 'off' } - } + } satisfies NonNullable[number] ] : []), { From 71e2cdd9624bf7cc59161e175120567e14fda554 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:17:47 -0700 Subject: [PATCH 12/23] test: handle legacy account reads and proxy teardown --- .../fixtures/liveCloudBillingFixture.ts | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 14503efdfb21..2156391a3a2f 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -21,6 +21,9 @@ export const liveCloudBillingFixture = base.extend<{ const origins = new Set([ new URL(baseURL ?? config.PLAYWRIGHT_TEST_URL).origin, config.PLAYWRIGHT_SETUP_API_URL, + ...(config.PLAYWRIGHT_SETUP_API_URL === 'https://testcloud.comfy.org' + ? ['https://testapi.comfy.org'] + : []), 'https://identitytoolkit.googleapis.com', 'https://securetoken.googleapis.com', 'https://dreamboothy-dev.firebaseapp.com', @@ -51,6 +54,20 @@ export const liveCloudBillingFixture = base.extend<{ await context.route('**/*', async (route) => { const request = route.request() const url = new URL(request.url()) + if ( + config.PLAYWRIGHT_SETUP_API_URL === 'https://testcloud.comfy.org' && + url.origin === 'https://testapi.comfy.org' && + request.method() === 'GET' && + /^\/customers(?:\/balance)?$/.test(url.pathname) + ) { + await route.fallback() + return + } + if (url.origin === 'https://testapi.comfy.org') { + networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } if ( !networkPolicy.origins.has(url.origin) && url.hostname.endsWith('.comfy.org') && @@ -72,19 +89,24 @@ export const liveCloudBillingFixture = base.extend<{ url.origin === config.PLAYWRIGHT_TEST_URL && /^\/(api|internal)(\/|$)/.test(url.pathname) ) { - const response = await route.fetch({ - url: new URL( - url.pathname + url.search, - config.PLAYWRIGHT_SETUP_API_URL - ).href, - maxRedirects: 0 - }) - await route.fulfill({ response }) + try { + const response = await route.fetch({ + url: new URL( + url.pathname + url.search, + config.PLAYWRIGHT_SETUP_API_URL + ).href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + } catch { + throw new Error(`Cloud proxy request failed: ${url.pathname}`) + } return } await route.fallback() }) await use(context) + await context.unrouteAll({ behavior: 'ignoreErrors' }) }, billingSession: async ({ page }, use) => { const config = loadLiveCloudBillingConfig() From c0f6c8c6737ea5f0aa604361b9581f62c0565eee Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:19:18 -0700 Subject: [PATCH 13/23] test: allow the selected test Cloud account API --- browser_tests/fixtures/liveCloudBillingFixture.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 2156391a3a2f..24e5939c9434 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -54,20 +54,6 @@ export const liveCloudBillingFixture = base.extend<{ await context.route('**/*', async (route) => { const request = route.request() const url = new URL(request.url()) - if ( - config.PLAYWRIGHT_SETUP_API_URL === 'https://testcloud.comfy.org' && - url.origin === 'https://testapi.comfy.org' && - request.method() === 'GET' && - /^\/customers(?:\/balance)?$/.test(url.pathname) - ) { - await route.fallback() - return - } - if (url.origin === 'https://testapi.comfy.org') { - networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) - await route.abort('blockedbyclient') - return - } if ( !networkPolicy.origins.has(url.origin) && url.hostname.endsWith('.comfy.org') && From a1fd14b62ac0ad84df482f3379b2f57d7732b85e Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:19:22 -0700 Subject: [PATCH 14/23] test: cover live checkout recovery after reload --- .../fixtures/helpers/LiveCloudBilling.ts | 12 +- .../fixtures/liveCloudBillingFixture.ts | 136 +++++++----------- .../fixtures/networkIsolationFixture.ts | 128 ++++++++++------- .../fixtures/utils/liveCloudBillingContext.ts | 93 ++++++++++++ browser_tests/playwrightSelection.test.ts | 2 +- .../liveCloud/checkoutPersistence.spec.ts | 70 +++++++++ .../tests/liveCloud/checkoutRecovery.spec.ts | 26 ++-- docs/testing/cloud-billing-e2e.md | 21 ++- package.json | 1 - pnpm-lock.yaml | 130 +++++++++++++---- 10 files changed, 438 insertions(+), 181 deletions(-) create mode 100644 browser_tests/fixtures/utils/liveCloudBillingContext.ts create mode 100644 browser_tests/tests/liveCloud/checkoutPersistence.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index 704cd658165d..3421a27d596d 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -25,10 +25,14 @@ export class LiveCloudBillingSession { ) {} async read(path: string, schema: z.ZodType): Promise { - const response = await this.request.get(new URL(path, this.backend).href, { - headers: this.headers, - maxRedirects: 0 - }) + const response = await this.request + .get(new URL(path, this.backend).href, { + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) expect(response.status(), `GET ${path}`).toBe(200) return schema.parse(await response.json()) } diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index 24e5939c9434..e96590dd8b7a 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -1,19 +1,29 @@ +import type { BrowserContext } from '@playwright/test' import { expect } from '@playwright/test' import { ComfyPage } from '@e2e/fixtures/ComfyPage' -import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudBilling' import { - LiveCloudBillingSession, - LiveCloudCheckout, - matchesBillingResponse -} from '@e2e/fixtures/helpers/LiveCloudBilling' -import { networkIsolationFixture as base } from '@e2e/fixtures/networkIsolationFixture' + installContextNetworkIsolation, + networkIsolationFixture as base +} from '@e2e/fixtures/networkIsolationFixture' +import { + installLiveCloudBillingRouting, + signInToLiveCloud +} from '@e2e/fixtures/utils/liveCloudBillingContext' import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' +interface FreshBillingSession { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout +} + export const liveCloudBillingFixture = base.extend<{ billingSession: LiveCloudBillingSession comfyPage: ComfyPage checkout: LiveCloudCheckout + freshBillingSession: () => Promise }>({ baseURL: process.env.PLAYWRIGHT_TEST_URL, networkPolicy: async ({ baseURL }, use, testInfo) => { @@ -50,86 +60,50 @@ export const liveCloudBillingFixture = base.extend<{ ).toEqual([]) }, context: async ({ context, networkPolicy }, use) => { - const config = loadLiveCloudBillingConfig() - await context.route('**/*', async (route) => { - const request = route.request() - const url = new URL(request.url()) - if ( - !networkPolicy.origins.has(url.origin) && - url.hostname.endsWith('.comfy.org') && - /^\/(api|customers)(\/|$)/.test(url.pathname) - ) { - networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) - await route.abort('blockedbyclient') - return - } - if ( - request.isNavigationRequest() && - !networkPolicy.origins.has(url.origin) - ) { - networkPolicy.unexpected.add(`Navigation ${url.origin}${url.pathname}`) - await route.abort('blockedbyclient') - return - } - if ( - url.origin === config.PLAYWRIGHT_TEST_URL && - /^\/(api|internal)(\/|$)/.test(url.pathname) - ) { - try { - const response = await route.fetch({ - url: new URL( - url.pathname + url.search, - config.PLAYWRIGHT_SETUP_API_URL - ).href, - maxRedirects: 0 - }) - await route.fulfill({ response }) - } catch { - throw new Error(`Cloud proxy request failed: ${url.pathname}`) - } - return - } - await route.fallback() - }) + await installLiveCloudBillingRouting(context, networkPolicy) await use(context) await context.unrouteAll({ behavior: 'ignoreErrors' }) }, billingSession: async ({ page }, use) => { + await use(await signInToLiveCloud(page)) + }, + freshBillingSession: async ( + { browser, contextOptions, networkPolicy }, + use + ) => { const config = loadLiveCloudBillingConfig() - await new FeatureFlagHelper(page).seedFlags({ - onboarding_survey_enabled: false - }) - await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) - await page - .getByRole('button', { name: 'Use email instead', exact: true }) - .click() - await page - .getByRole('textbox', { name: 'Email', exact: true }) - .fill(config.CLOUD_ACCOUNT_EMAIL) - await page - .getByLabel('Password', { exact: true }) - .fill(config.CLOUD_ACCOUNT_PASSWORD) - const [response] = await Promise.all([ - page.waitForResponse( - (response) => - matchesBillingResponse( - response, - config.PLAYWRIGHT_TEST_URL, - '/api/billing/status' - ) && response.status() === 200 - ), - page.getByRole('button', { name: 'Sign in', exact: true }).click() - ]) - const authorization = await response.request().headerValue('authorization') - expect(authorization).toBeTruthy() - if (!authorization) throw new Error('Missing billing authorization') - await use( - new LiveCloudBillingSession( - page.request, - config.PLAYWRIGHT_SETUP_API_URL, - { authorization } - ) - ) + const contexts: BrowserContext[] = [] + try { + await use(async () => { + const context = await browser.newContext({ + ...contextOptions, + baseURL: config.PLAYWRIGHT_TEST_URL, + storageState: { cookies: [], origins: [] }, + serviceWorkers: 'block', + recordVideo: undefined, + recordHar: undefined + }) + contexts.push(context) + await installContextNetworkIsolation( + context, + networkPolicy, + config.PLAYWRIGHT_TEST_URL + ) + await installLiveCloudBillingRouting(context, networkPolicy) + const page = await context.newPage() + const billingSession = await signInToLiveCloud(page) + const comfyPage = new ComfyPage(page, context.request) + return { + billingSession, + checkout: new LiveCloudCheckout(comfyPage, config.PLAYWRIGHT_TEST_URL) + } + }) + } finally { + for (const context of contexts) { + await context.unrouteAll({ behavior: 'ignoreErrors' }) + await context.close() + } + } }, comfyPage: async ({ page, request }, use) => { await use(new ComfyPage(page, request)) diff --git a/browser_tests/fixtures/networkIsolationFixture.ts b/browser_tests/fixtures/networkIsolationFixture.ts index 36635ba123b3..bee03306eef3 100644 --- a/browser_tests/fixtures/networkIsolationFixture.ts +++ b/browser_tests/fixtures/networkIsolationFixture.ts @@ -1,4 +1,4 @@ -import type { APIRequestContext } from '@playwright/test' +import type { APIRequestContext, BrowserContext } from '@playwright/test' import { expect, test as base } from '@playwright/test' import { config as dotenvConfig } from 'dotenv' @@ -31,8 +31,76 @@ function guardApiRequests( } } +export interface NetworkPolicy { + origins: Set + unexpected: Set +} + +export async function installContextNetworkIsolation( + context: BrowserContext, + { origins, unexpected }: NetworkPolicy, + baseURL?: string +) { + const restore = guardApiRequests( + context.request, + origins, + unexpected, + baseURL + ) + await context.route('**/*', async (route) => { + const url = new URL(route.request().url()) + if (origins.has(url.origin)) { + await route.continue() + return + } + unexpected.add(`${route.request().method()} ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + }) + await context.routeWebSocket( + (url) => !origins.has(url.origin.replace(/^ws/, 'http')), + async (socket) => { + const url = new URL(socket.url()) + unexpected.add(`WebSocket ${url.origin}${url.pathname}`) + await socket.close() + } + ) + await context.route('https://huggingface.co/**/resolve/**', (route) => + route.request().method() === 'HEAD' + ? route.fulfill({ status: 200, body: '' }) + : route.fallback() + ) + await context.route( + 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', + (route) => + route.fulfill({ contentType: 'application/javascript', body: '' }) + ) + await context.route('https://apis.google.com/js/api.js**', (route) => + route.fulfill({ status: 503, body: '' }) + ) + await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => + route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) + ) + for (const slide of HERO_SLIDES) { + await context.route(slide.poster, (route) => + route.fulfill({ path: assetPath('image32x32.webp') }) + ) + await context.route(slide.src, (route) => + route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) + ) + } + await context.route( + 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', + (route) => route.fulfill({ status: 404, body: '' }) + ) + await context.route( + 'https://{api,stagingapi}.comfy.org/nodes{,?*}', + (route) => route.fulfill({ status: 404, body: '' }) + ) + return restore +} + export const networkIsolationFixture = base.extend<{ - networkPolicy: { origins: Set; unexpected: Set } + networkPolicy: NetworkPolicy }>({ serviceWorkers: 'block', networkPolicy: [ @@ -68,61 +136,11 @@ export const networkIsolationFixture = base.extend<{ { context, baseURL, networkPolicy: { origins, unexpected } }, use ) => { - const restore = guardApiRequests( - context.request, - origins, - unexpected, + const restore = await installContextNetworkIsolation( + context, + { origins, unexpected }, baseURL ) - await context.route('**/*', async (route) => { - const url = new URL(route.request().url()) - if (origins.has(url.origin)) { - await route.continue() - return - } - unexpected.add(`${route.request().method()} ${url.origin}${url.pathname}`) - await route.abort('blockedbyclient') - }) - await context.routeWebSocket( - (url) => !origins.has(url.origin.replace(/^ws/, 'http')), - async (socket) => { - const url = new URL(socket.url()) - unexpected.add(`WebSocket ${url.origin}${url.pathname}`) - await socket.close() - } - ) - await context.route('https://huggingface.co/**/resolve/**', (route) => - route.request().method() === 'HEAD' - ? route.fulfill({ status: 200, body: '' }) - : route.fallback() - ) - await context.route( - 'https://utt.impactcdn.com/A6951770-3747-434a-9ac7-4e582e67d91f1.js', - (route) => - route.fulfill({ contentType: 'application/javascript', body: '' }) - ) - await context.route('https://apis.google.com/js/api.js**', (route) => - route.fulfill({ status: 503, body: '' }) - ) - await context.route('https://cloud.comfy.org/cdn-cgi/trace', (route) => - route.fulfill({ contentType: 'text/plain', body: 'loc=US\n' }) - ) - for (const slide of HERO_SLIDES) { - await context.route(slide.poster, (route) => - route.fulfill({ path: assetPath('image32x32.webp') }) - ) - await context.route(slide.src, (route) => - route.fulfill({ path: assetPath('video/video-preview-portrait.webm') }) - ) - } - await context.route( - 'https://{api,stagingapi}.comfy.org/comfy-nodes/*/node', - (route) => route.fulfill({ status: 404, body: '' }) - ) - await context.route( - 'https://{api,stagingapi}.comfy.org/nodes{,?*}', - (route) => route.fulfill({ status: 404, body: '' }) - ) await use(context) restore() diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts new file mode 100644 index 000000000000..f09aeac9c891 --- /dev/null +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -0,0 +1,93 @@ +import type { BrowserContext, Page } from '@playwright/test' +import { expect } from '@playwright/test' + +import { FeatureFlagHelper } from '@e2e/fixtures/helpers/FeatureFlagHelper' +import { + LiveCloudBillingSession, + matchesBillingResponse +} from '@e2e/fixtures/helpers/LiveCloudBilling' +import type { NetworkPolicy } from '@e2e/fixtures/networkIsolationFixture' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +export async function installLiveCloudBillingRouting( + context: BrowserContext, + networkPolicy: NetworkPolicy +) { + const config = loadLiveCloudBillingConfig() + await context.route('**/*', async (route) => { + const request = route.request() + const url = new URL(request.url()) + if ( + !networkPolicy.origins.has(url.origin) && + url.hostname.endsWith('.comfy.org') && + /^\/(api|customers)(\/|$)/.test(url.pathname) + ) { + networkPolicy.unexpected.add(`API ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + request.isNavigationRequest() && + !networkPolicy.origins.has(url.origin) + ) { + networkPolicy.unexpected.add(`Navigation ${url.origin}${url.pathname}`) + await route.abort('blockedbyclient') + return + } + if ( + url.origin === config.PLAYWRIGHT_TEST_URL && + /^\/(api|internal)(\/|$)/.test(url.pathname) + ) { + try { + const response = await route.fetch({ + url: new URL( + url.pathname + url.search, + config.PLAYWRIGHT_SETUP_API_URL + ).href, + maxRedirects: 0 + }) + await route.fulfill({ response }) + } catch { + throw new Error(`Cloud proxy request failed: ${url.pathname}`) + } + return + } + await route.fallback() + }) +} + +export async function signInToLiveCloud(page: Page) { + const config = loadLiveCloudBillingConfig() + await new FeatureFlagHelper(page).seedFlags({ + onboarding_survey_enabled: false + }) + await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) + await page + .getByRole('button', { name: 'Use email instead', exact: true }) + .click() + await page + .getByRole('textbox', { name: 'Email', exact: true }) + .fill(config.CLOUD_ACCOUNT_EMAIL) + await page + .getByLabel('Password', { exact: true }) + .fill(config.CLOUD_ACCOUNT_PASSWORD) + const [response] = await Promise.all([ + page.waitForResponse( + (response) => + matchesBillingResponse( + response, + config.PLAYWRIGHT_TEST_URL, + '/api/billing/status' + ) && response.status() === 200 + ), + page.getByRole('button', { name: 'Sign in', exact: true }).click() + ]) + const authorization = await response.request().headerValue('authorization') + expect(authorization).toBeTruthy() + if (!authorization) throw new Error('Missing billing authorization') + return new LiveCloudBillingSession( + page.request, + config.PLAYWRIGHT_SETUP_API_URL, + { authorization } + ) +} diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index a8d88bf5dd63..8aca5f0e22c1 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -48,7 +48,7 @@ describe('Live billing opt-in', () => { const specs = report.suites.flatMap((suite) => suite.suites.flatMap((child) => child.specs) ) - expect(specs).toHaveLength(enabled === '1' ? 1 : 0) + expect(specs).toHaveLength(enabled === '1' ? 3 : 0) if (enabled === '1') { expect(report.config.globalSetup).toBeNull() expect(report.config.globalTeardown).toBeNull() diff --git a/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts b/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts new file mode 100644 index 000000000000..36aca0c452c1 --- /dev/null +++ b/browser_tests/tests/liveCloud/checkoutPersistence.spec.ts @@ -0,0 +1,70 @@ +import { + comfyExpect as expect, + liveCloudBillingFixture as test +} from '@e2e/fixtures/liveCloudBillingFixture' + +test.describe( + 'Real Cloud checkout persistence', + { tag: ['@cloud-live'] }, + () => { + test('resumes the same pending checkout after reloading the page', async ({ + comfyPage, + checkout, + billingSession + }, testInfo) => { + const operationId = + await test.step('Leave the checkout pending', async () => { + const operationId = await checkout.abandonCheckout(testInfo) + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + return operationId + }) + + await test.step('Reload and reopen the checkout', async () => { + await comfyPage.page.reload() + await checkout.open() + await expect(checkout.resumePayment).toBeEnabled() + await checkout.attachScreenshot('checkout-after-reload.png') + }) + + await test.step('Resume the same checkout after reload', async () => { + const resumedId = await checkout.abandonCheckout(testInfo, true) + expect(resumedId).toBe(operationId) + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + }) + }) + + test('reuses the pending operation after signing in from a fresh browser context', async ({ + comfyPage, + checkout, + billingSession, + freshBillingSession + }, testInfo) => { + const operationId = + await test.step('Leave the checkout pending', async () => { + const operationId = await checkout.abandonCheckout(testInfo) + await billingSession.expectPending(operationId) + return operationId + }) + + const fresh = + await test.step('Sign in from a fresh browser context', async () => { + await comfyPage.page.close() + const fresh = await freshBillingSession() + await fresh.billingSession.assertNoCardAccount(testInfo) + await fresh.billingSession.expectPending(operationId) + await fresh.checkout.open() + return fresh + }) + + await test.step('Resume the same checkout in the fresh context', async () => { + const resumedId = await fresh.checkout.abandonCheckout(testInfo) + expect(resumedId).toBe(operationId) + await fresh.billingSession.expectPending(operationId) + await expect(fresh.checkout.resumePayment).toBeEnabled() + await fresh.checkout.attachScreenshot('checkout-in-fresh-context.png') + }) + }) + } +) diff --git a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts index 24a5f1644c17..4d0eaa5d8b38 100644 --- a/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts +++ b/browser_tests/tests/liveCloud/checkoutRecovery.spec.ts @@ -8,15 +8,23 @@ test.describe('Real Cloud checkout recovery', { tag: ['@cloud-live'] }, () => { checkout, billingSession }, testInfo) => { - await checkout.attachScreenshot('preview.png') - const operationId = await checkout.abandonCheckout(testInfo) - await billingSession.expectPending(operationId) - await expect(checkout.resumePayment).toBeEnabled() - await checkout.attachScreenshot('resume-payment.png') + const operationId = + await test.step('Leave the checkout pending', async () => { + await checkout.attachScreenshot('preview.png') + return await checkout.abandonCheckout(testInfo) + }) - const resumedId = await checkout.abandonCheckout(testInfo, true) - expect(resumedId).toBe(operationId) - await expect(checkout.resumePayment).toBeEnabled() - await billingSession.expectPending(operationId) + await test.step('See the pending checkout recovery action', async () => { + await billingSession.expectPending(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await checkout.attachScreenshot('resume-payment.png') + }) + + await test.step('Retry the same checkout', async () => { + const resumedId = await checkout.abandonCheckout(testInfo, true) + expect(resumedId).toBe(operationId) + await expect(checkout.resumePayment).toBeEnabled() + await billingSession.expectPending(operationId) + }) }) }) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 778711922ad4..dd13bcdff8a6 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -3,12 +3,12 @@ Run the opt-in `cloud-live` project through the existing browser-test runner: ```sh -pnpm test:browser:cloud-billing --headed +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 --headed pnpm exec playwright show-report ``` Set these variables using the existing `.env` convention or inject them with -`op run --env-file -- pnpm test:browser:cloud-billing`: +`op run --env-file -- zsh -f`: | Variable | Value | | -------------------------- | ---------------------------------------------------- | @@ -17,6 +17,18 @@ Set these variables using the existing `.env` convention or inject them with | `CLOUD_ACCOUNT_EMAIL` | Sandbox account email | | `CLOUD_ACCOUNT_PASSWORD` | Sandbox account password | +For repeated local runs, open one credential shell and run all test commands +inside it. This reuses credentials already loaded by 1Password instead of +requesting them again for each command: + +```sh +op run --env-file -- zsh -f +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1 +``` + +Keep the shell open for the session. Exiting it ends credential reuse. The env +file contains 1Password references, not passwords. + For a local frontend, start `pnpm dev:cloud` and use its Vite URL with `PLAYWRIGHT_SETUP_API_URL=https://testcloud.comfy.org`. The test routes browser API requests and its API checks directly to the selected sandbox, independently of the local Vite proxy. Production targets are rejected. @@ -39,6 +51,7 @@ browser leaves the unpaid checkout pending in the backend. The test does not reset billing state or claim that the operation expired. The first action accepts either a fresh Subscribe button or the pending Complete your payment button, so sequential runs can resume an existing checkout. Backend expiry and fixture reset remain separate work. -The first test covers checkout handoff and retry. Payment completion, page-reload -recovery, the 24-hour timeout, and CI integration remain follow-up work. A +The no-card tests cover checkout handoff, retry, page reload, and signing in +from an empty browser context. Payment completion, the 24-hour timeout, and +CI integration remain follow-up work. A successful sign-in alone does not establish billing coverage. diff --git a/package.json b/package.json index c4c746866064..aa24d109d6c3 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "comfy-test": "tsx tools/test-recorder/src/index.ts", "test:assets-flag-artifact": "tsx scripts/checkAssetsFlagArtifact.ts", "test:browser": "pnpm exec playwright test", - "test:browser:cloud-billing": "cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live --workers=1", "test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser", "test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser", "custom-node-quarantine": "tsx scripts/customNodeQuarantine.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2c411b7b94d..9be46e513ee9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -579,7 +579,7 @@ importers: version: 14.4.0(vue@3.5.42(typescript@6.0.3)) '@vueuse/integrations': specifier: 'catalog:' - version: 14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3)) + version: 14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3)) '@vueuse/router': specifier: ^14.4.0 version: 14.4.0(vue-router@5.2.0(@vue/compiler-sfc@3.5.42)(esbuild@0.28.1)(pinia@4.0.2(@vue/devtools-api@8.1.5)(typescript@6.0.3)(vue@3.5.42(typescript@6.0.3)))(rolldown@1.2.6)(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)))(vue@3.5.42(typescript@6.0.3)) @@ -739,7 +739,7 @@ importers: version: 14.6.1(@testing-library/dom@10.4.1) '@testing-library/vue': specifier: 'catalog:' - version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) + version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) '@total-typescript/shoehorn': specifier: 'catalog:' version: 0.1.2 @@ -1055,7 +1055,7 @@ importers: version: 14.6.1(@testing-library/dom@10.4.1) '@testing-library/vue': specifier: 'catalog:' - version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) + version: 8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) '@total-typescript/shoehorn': specifier: 'catalog:' version: 0.1.2 @@ -2653,6 +2653,9 @@ packages: '@jridgewell/source-map@0.3.11': resolution: {integrity: sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==} + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/sourcemap-codec@1.6.0': resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} @@ -4902,15 +4905,27 @@ packages: peerDependencies: '@babel/core': ^7.0.0-0 + '@vue/compiler-core@3.5.41': + resolution: {integrity: sha512-q0Xtv/F9w2YO/7htQhtiL+Ev2WCJbe5N2hc+XfgyKkEKqWpSxknmT8QOuGdEKNdjPq0c3F7rNpFkTo3Kfrm7pg==} + '@vue/compiler-core@3.5.42': resolution: {integrity: sha512-2Ye1ilMtKXxl8qZUrQ5j0CdgenFp/HFQmta6rfRyfEsTG69L6Wk+tWuNoHYHMx9E8tF2Slvdg1FuwDvAXdy1LQ==} + '@vue/compiler-dom@3.5.41': + resolution: {integrity: sha512-oKacVfNglLvGjnS6BXOlGL7EyG2h8X03pqXCjzotRZUaXGjbrTJUnVAQjrCqUnS+lyu31nwQjZY/d817GmCnfw==} + '@vue/compiler-dom@3.5.42': resolution: {integrity: sha512-qbhQZEFmycr+ni/qyuccS4sucNN7VAbDfbkvNxWOX2VfgFm90MNs3/UhRNKoPMEIVn0F8gdlYjLPvqxHwHeQOA==} + '@vue/compiler-sfc@3.5.41': + resolution: {integrity: sha512-XJhip7R2wy6vX3knCxdZN4KracFaZUef58s1KYewqluedHIJaPIVfXoYT7MF1F8nCvv6k8bWWxDC8opMkg1VTQ==} + '@vue/compiler-sfc@3.5.42': resolution: {integrity: sha512-fkCAFB4okcAANGMThboWnScp/gzWjU0ZSkVnjTIiplmMDq2uq0tIB3j+xVu4rhv5rvOgBySCysudmbMd6xRRqw==} + '@vue/compiler-ssr@3.5.41': + resolution: {integrity: sha512-U3v5OejKEGqOI0Wy0+Sz7hGuIFZHA4LSXzrNM3IMIeDyJEBBfTpX26n3SDgToRpP2bLc9FfI2j/kSgcJ8Emq5A==} + '@vue/compiler-ssr@3.5.42': resolution: {integrity: sha512-xmLk3wLkbizPAiLyomjgFFosf2ys9b5Ghb+oh/k2tnvipNz8OFrQOiTcWCzyK7MpBp9KkyGtfvgfLUivbmuGYA==} @@ -4958,6 +4973,9 @@ packages: '@vue/server-renderer@3.5.42': resolution: {integrity: sha512-2++5dUyYS4gvo7xQXSECUDhB7TS0aOl5SeVfC5qSq1Jgfhjvegw1zqhwTIR3imZ+QYPJQw9gfcFvXGAjGZ7ajQ==} + '@vue/shared@3.5.41': + resolution: {integrity: sha512-IOnwSCma8j+9xJT6b8H0dEYidC80NsYmNMlZxRsukYcSoGaDBohog5hDxzeUXdFeGWFA++vWvxqOmrr96VlqMA==} + '@vue/shared@3.5.42': resolution: {integrity: sha512-2rPxex1jQf4jvl9MOHl6YaXCPcrNqz/FstMOEh3QWY+/OME9nQTvl9WYeCwhW7AFjaR0SnngZGlp/wkR6rkI6g==} @@ -7153,6 +7171,9 @@ packages: magic-string@1.1.1: resolution: {integrity: sha512-qFemKPzc3ttrYVaMmnSkGtGc5nE6Ncl4bj7c9IE6C9OUIRXjf6PzJ+UZ1xhVIjYc7dolHq3qKzpAJPZUbvNj+A==} + magicast@0.5.3: + resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} + magicast@0.5.4: resolution: {integrity: sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==} @@ -9315,6 +9336,9 @@ packages: vscode-nls@5.2.0: resolution: {integrity: sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==} + vscode-uri@3.1.0: + resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} + vscode-uri@3.2.0: resolution: {integrity: sha512-m2gXo3bn0G1kT9InzMf07fTbqMbGtyckj3bH5ktLO+1Ssv+yiATZ4dhwaQv9UZWxJh6E9IFGnQyjgWVDWVBDrg==} @@ -9844,7 +9868,7 @@ snapshots: dependencies: '@astrojs/compiler': 2.13.1 '@astrojs/yaml2ts': 0.2.4 - '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/sourcemap-codec': 1.5.5 '@volar/kit': 2.4.28(typescript@6.0.3) '@volar/language-core': 2.4.28 '@volar/language-server': 2.4.28 @@ -9859,7 +9883,7 @@ snapshots: volar-service-typescript-twoslash-queries: 0.0.71(@volar/language-service@2.4.28) volar-service-yaml: 0.0.71(@volar/language-service@2.4.28) vscode-html-languageservice: 5.6.2 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: prettier: 3.9.5 transitivePeerDependencies: @@ -9938,7 +9962,7 @@ snapshots: dependencies: '@vitejs/plugin-vue': 6.0.8(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) '@vitejs/plugin-vue-jsx': 5.1.6(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) - '@vue/compiler-sfc': 3.5.42 + '@vue/compiler-sfc': 3.5.41 astro: 7.2.1(@astrojs/markdown-remark@7.2.2)(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@25.9.5)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vite: 8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0) vite-plugin-vue-devtools: 8.2.1(vite@8.2.2(@types/node@25.9.5)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue@3.5.42(typescript@6.0.3)) @@ -11248,6 +11272,8 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 + '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/sourcemap-codec@1.6.0': {} '@jridgewell/trace-mapping@0.3.31': @@ -12401,6 +12427,18 @@ snapshots: dependencies: '@testing-library/dom': 10.4.1 + '@testing-library/vue@8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.41)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': + dependencies: + '@babel/runtime': 7.29.7 + '@testing-library/dom': 9.3.4 + '@vue/test-utils': 2.4.11(@vue/compiler-dom@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3)) + vue: 3.5.42(typescript@6.0.3) + optionalDependencies: + '@vue/compiler-sfc': 3.5.41 + transitivePeerDependencies: + - '@vue/compiler-dom' + - '@vue/server-renderer' + '@testing-library/vue@8.1.0(@vue/compiler-dom@3.5.42)(@vue/compiler-sfc@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': dependencies: '@babel/runtime': 7.29.7 @@ -13090,7 +13128,7 @@ snapshots: typesafe-path: 0.2.2 typescript: 6.0.3 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 '@volar/language-core@2.4.28': dependencies: @@ -13106,14 +13144,14 @@ snapshots: vscode-languageserver: 9.0.1 vscode-languageserver-protocol: 3.18.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 '@volar/language-service@2.4.28': dependencies: '@volar/language-core': 2.4.28 vscode-languageserver-protocol: 3.18.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 '@volar/source-map@2.4.28': {} @@ -13129,7 +13167,7 @@ snapshots: jsonc-parser: 2.3.1 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 '@vscode/l10n@0.0.18': {} @@ -13201,6 +13239,14 @@ snapshots: transitivePeerDependencies: - supports-color + '@vue/compiler-core@3.5.41': + dependencies: + '@babel/parser': 7.29.8 + '@vue/shared': 3.5.41 + entities: 7.0.1 + estree-walker: 2.0.2 + source-map-js: 1.2.1 + '@vue/compiler-core@3.5.42': dependencies: '@babel/parser': 7.29.8 @@ -13209,11 +13255,28 @@ snapshots: estree-walker: 2.0.2 source-map-js: 1.2.1 + '@vue/compiler-dom@3.5.41': + dependencies: + '@vue/compiler-core': 3.5.41 + '@vue/shared': 3.5.41 + '@vue/compiler-dom@3.5.42': dependencies: '@vue/compiler-core': 3.5.42 '@vue/shared': 3.5.42 + '@vue/compiler-sfc@3.5.41': + dependencies: + '@babel/parser': 7.29.8 + '@vue/compiler-core': 3.5.41 + '@vue/compiler-dom': 3.5.41 + '@vue/compiler-ssr': 3.5.41 + '@vue/shared': 3.5.41 + estree-walker: 2.0.2 + magic-string: 0.30.21 + postcss: 8.5.26 + source-map-js: 1.2.1 + '@vue/compiler-sfc@3.5.42': dependencies: '@babel/parser': 7.29.8 @@ -13226,6 +13289,11 @@ snapshots: postcss: 8.5.26 source-map-js: 1.2.1 + '@vue/compiler-ssr@3.5.41': + dependencies: + '@vue/compiler-dom': 3.5.41 + '@vue/shared': 3.5.41 + '@vue/compiler-ssr@3.5.42': dependencies: '@vue/compiler-dom': 3.5.42 @@ -13312,6 +13380,8 @@ snapshots: '@vue/runtime-dom': 3.5.42 '@vue/shared': 3.5.42 + '@vue/shared@3.5.41': {} + '@vue/shared@3.5.42': {} '@vue/test-utils@2.4.11(@vue/compiler-dom@3.5.42)(@vue/server-renderer@3.5.42)(vue@3.5.42(typescript@6.0.3))': @@ -13339,7 +13409,7 @@ snapshots: '@vueuse/shared': 14.4.0(vue@3.5.42(typescript@6.0.3)) vue: 3.5.42(typescript@6.0.3) - '@vueuse/integrations@14.4.0(axios@1.18.1)(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3))': + '@vueuse/integrations@14.4.0(axios@1.18.1(debug@4.4.3))(fuse.js@7.0.0)(vue@3.5.42(typescript@6.0.3))': dependencies: '@vueuse/core': 14.4.0(vue@3.5.42(typescript@6.0.3)) '@vueuse/shared': 14.4.0(vue@3.5.42(typescript@6.0.3)) @@ -13574,7 +13644,7 @@ snapshots: js-yaml: 4.3.1 jsonc-parser: 3.3.1 magic-string: 1.1.1 - magicast: 0.5.4 + magicast: 0.5.3 mrmime: 2.0.1 neotraverse: 1.0.1 obug: 2.1.4 @@ -15684,7 +15754,7 @@ snapshots: magic-string@0.30.21: dependencies: - '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/sourcemap-codec': 1.5.5 magic-string@0.30.8: dependencies: @@ -15692,7 +15762,13 @@ snapshots: magic-string@1.1.1: dependencies: - '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/sourcemap-codec': 1.5.5 + + magicast@0.5.3: + dependencies: + '@babel/parser': 7.29.8 + '@babel/types': 7.29.8 + source-map-js: 1.2.1 magicast@0.5.4: dependencies: @@ -18357,7 +18433,7 @@ snapshots: dependencies: vscode-css-languageservice: 6.3.10 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18366,7 +18442,7 @@ snapshots: '@emmetio/css-parser': 0.4.1 '@emmetio/html-matcher': 1.3.0 '@vscode/emmet-helper': 2.11.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18374,20 +18450,20 @@ snapshots: dependencies: vscode-html-languageservice: 5.6.2 vscode-languageserver-textdocument: 1.0.12 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 volar-service-prettier@0.0.71(@volar/language-service@2.4.28)(prettier@3.9.5): dependencies: - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 prettier: 3.9.5 volar-service-typescript-twoslash-queries@0.0.71(@volar/language-service@2.4.28): dependencies: - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18398,13 +18474,13 @@ snapshots: typescript-auto-import-cache: 0.3.6 vscode-languageserver-textdocument: 1.0.12 vscode-nls: 5.2.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 optionalDependencies: '@volar/language-service': 2.4.28 volar-service-yaml@0.0.71(@volar/language-service@2.4.28): dependencies: - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 yaml-language-server: 1.23.0 optionalDependencies: '@volar/language-service': 2.4.28 @@ -18414,14 +18490,14 @@ snapshots: '@vscode/l10n': 0.0.18 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.17.5 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 vscode-html-languageservice@5.6.2: dependencies: '@vscode/l10n': 0.0.18 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 vscode-json-languageservice@4.1.8: dependencies: @@ -18429,7 +18505,7 @@ snapshots: vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 vscode-nls: 5.2.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 vscode-jsonrpc@8.2.0: {} @@ -18457,6 +18533,8 @@ snapshots: vscode-nls@5.2.0: {} + vscode-uri@3.1.0: {} + vscode-uri@3.2.0: {} vue-component-meta@3.3.7(typescript@5.9.3): @@ -18478,7 +18556,7 @@ snapshots: '@babel/parser': 7.29.8 '@babel/types': 7.29.8 '@vue/compiler-dom': 3.5.42 - '@vue/compiler-sfc': 3.5.42 + '@vue/compiler-sfc': 3.5.41 ast-types: 0.16.1 esm-resolve: 1.0.11 hash-sum: 2.0.0 @@ -18764,7 +18842,7 @@ snapshots: vscode-languageserver: 9.0.1 vscode-languageserver-textdocument: 1.0.12 vscode-languageserver-types: 3.18.0 - vscode-uri: 3.2.0 + vscode-uri: 3.1.0 yaml: 2.9.0 yaml@2.9.0: {} From 1514152f594028dec72ddd865f14e2eeedcaece0 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:24:48 -0700 Subject: [PATCH 15/23] fix: identify each top-up purchase attempt --- src/composables/billing/types.ts | 5 +- .../billing/useBillingContext.test.ts | 7 ++- src/composables/billing/useBillingContext.ts | 4 +- ...TopUpCreditsDialogContentWorkspace.test.ts | 51 ++++++++++++++++++- .../TopUpCreditsDialogContentWorkspace.vue | 10 +++- .../composables/useWorkspaceBilling.test.ts | 14 +++-- .../composables/useWorkspaceBilling.ts | 7 ++- 7 files changed, 84 insertions(+), 14 deletions(-) diff --git a/src/composables/billing/types.ts b/src/composables/billing/types.ts index 84719722b10b..31a7da5cce51 100644 --- a/src/composables/billing/types.ts +++ b/src/composables/billing/types.ts @@ -77,7 +77,10 @@ export interface BillingActions { * Pass-through by design: the caller owns the completed/pending follow-up * (balance refresh or billing-op polling), so this does not refresh. */ - topup: (amountCents: number) => Promise + topup: ( + amountCents: number, + idempotencyKey?: string + ) => Promise fetchPlans: () => Promise /** * Ensures billing is initialized and subscription is active. diff --git a/src/composables/billing/useBillingContext.test.ts b/src/composables/billing/useBillingContext.test.ts index eff32f5f06bd..91426ed1b550 100644 --- a/src/composables/billing/useBillingContext.test.ts +++ b/src/composables/billing/useBillingContext.test.ts @@ -352,9 +352,12 @@ describe('useBillingContext', () => { vi.clearAllMocks() expect(context.type.value).toBe('workspace') - await context.topup(500) + await context.topup(500, 'topup-attempt-1') - expect(workspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(workspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(mockPurchaseCredits).not.toHaveBeenCalled() }) diff --git a/src/composables/billing/useBillingContext.ts b/src/composables/billing/useBillingContext.ts index 4d25ba306ffb..ccb9b29b0ea3 100644 --- a/src/composables/billing/useBillingContext.ts +++ b/src/composables/billing/useBillingContext.ts @@ -315,7 +315,7 @@ function useBillingContextInternal(): BillingContext { return activeContext.value.resubscribe(options) } - async function topup(amountCents: number) { + async function topup(amountCents: number, idempotencyKey?: string) { if ( !Number.isInteger(amountCents) || amountCents <= 0 || @@ -325,7 +325,7 @@ function useBillingContextInternal(): BillingContext { 'Top-up amount must be a positive whole-dollar cent value' ) } - return activeContext.value.topup(amountCents) + return activeContext.value.topup(amountCents, idempotencyKey) } async function fetchPlans() { diff --git a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts index 96d4058fdd6d..67f1e2c02414 100644 --- a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts +++ b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.test.ts @@ -19,12 +19,20 @@ const mockFetchBalance = vi.fn() const mockFetchStatus = vi.fn() const mockManageSubscription = vi.fn<() => Promise>() const mockReportError = vi.hoisted(() => vi.fn()) +const mockCreateUuidv4 = vi.hoisted(() => vi.fn()) + +vi.mock(import('@/utils/uuid'), () => ({ createUuidv4: mockCreateUuidv4 })) vi.mock(import('@/platform/telemetry/reportError'), () => ({ reportError: mockReportError })) const mockTopup = - vi.fn<(amountCents: number) => Promise>() + vi.fn< + ( + amountCents: number, + idempotencyKey: string + ) => Promise + >() const mockShowSettings = vi.fn() const mockToastAdd = vi.fn() @@ -60,7 +68,8 @@ vi.mock(import('@/composables/billing/useBillingContext'), () => ({ fetchBalance: mockFetchBalance, fetchStatus: mockFetchStatus, manageSubscription: mockManageSubscription, - topup: (amountCents: number) => mockTopup(amountCents) + topup: (amountCents: number, idempotencyKey: string) => + mockTopup(amountCents, idempotencyKey) }) })) @@ -177,6 +186,7 @@ async function clickAddCredits() { } beforeEach(() => { + mockCreateUuidv4.mockReturnValue('topup-attempt-1') vi.mocked(useDialogStore().closeDialog).mockImplementation(() => {}) }) @@ -747,6 +757,43 @@ describe('TopUpCreditsDialogContentWorkspace', () => { ) }) + it('reuses the idempotency key after an uncertain request failure', async () => { + mockTopup + .mockRejectedValueOnce(new Error('connection lost')) + .mockResolvedValueOnce(topupResponse('pending')) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + + renderDialog() + await clickAddCredits() + const payButton = screen.getByRole('button', { name: 'Pay $50.00' }) + await userEvent.click(payButton) + await waitFor(() => expect(payButton).toBeEnabled()) + await userEvent.click(payButton) + + expect(mockTopup).toHaveBeenNthCalledWith(1, 5000, 'topup-attempt-1') + expect(mockTopup).toHaveBeenNthCalledWith(2, 5000, 'topup-attempt-1') + consoleError.mockRestore() + }) + + it('uses a new idempotency key after a definitive failure', async () => { + mockCreateUuidv4 + .mockReturnValueOnce('topup-attempt-1') + .mockReturnValueOnce('topup-attempt-2') + mockTopup + .mockResolvedValueOnce(topupResponse('failed')) + .mockResolvedValueOnce(topupResponse('pending')) + + renderDialog() + await clickAddCredits() + const payButton = screen.getByRole('button', { name: 'Pay $50.00' }) + await userEvent.click(payButton) + await waitFor(() => expect(payButton).toBeEnabled()) + await userEvent.click(payButton) + + expect(mockTopup).toHaveBeenNthCalledWith(1, 5000, 'topup-attempt-1') + expect(mockTopup).toHaveBeenNthCalledWith(2, 5000, 'topup-attempt-2') + }) + it('does not top up after the server capability is revoked', async () => { renderDialog() await clickAddCredits() diff --git a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue index ef3a8aa83fd0..f53ba22a6e10 100644 --- a/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue +++ b/src/platform/workspace/components/TopUpCreditsDialogContentWorkspace.vue @@ -305,6 +305,7 @@ import { useBillingCapabilities } from '@/platform/workspace/composables/useBill import { useHasSavedPaymentMethod } from '@/platform/workspace/composables/useHasSavedPaymentMethod' import { useBillingOperationStore } from '@/platform/workspace/stores/billingOperationStore' import { useDialogStore } from '@/stores/dialogStore' +import { createUuidv4 } from '@/utils/uuid' import { cn } from '@comfyorg/tailwind-utils' const { isInsufficientCredits = false } = defineProps<{ @@ -356,6 +357,7 @@ const payAmount = ref(50) const showCeilingWarning = ref(false) const loading = ref(false) const paymentSubmitted = ref(false) +let topupIdempotencyKey: string | undefined const step = ref<'amount' | 'confirm' | 'verifying'>( topupOperation.value && canTopUp.value ? 'verifying' : 'amount' ) @@ -378,6 +380,7 @@ const creditsModel = computed({ set: (newCredits: number) => { payAmount.value = Math.round(creditsToUsd(newCredits)) selectedPreset.value = null + topupIdempotencyKey = undefined } }) @@ -439,12 +442,14 @@ function handlePayAmountChange(value: number) { payAmount.value = value selectedPreset.value = null showCeilingWarning.value = false + topupIdempotencyKey = undefined } function handlePresetClick(amount: number) { showCeilingWarning.value = false payAmount.value = amount selectedPreset.value = amount + topupIdempotencyKey = undefined } function handlePrimaryAction() { @@ -481,6 +486,7 @@ function startOverTopup() { const operation = topupOperation.value if (operation) billingOperationStore.dismissOperation(operation.opId) paymentSubmitted.value = false + topupIdempotencyKey = undefined step.value = 'amount' } @@ -514,7 +520,8 @@ async function handleBuy() { }) const amountCents = payAmount.value * 100 - const response = await topup(amountCents) + topupIdempotencyKey ??= createUuidv4() + const response = await topup(amountCents, topupIdempotencyKey) if (!response) { paymentSubmitted.value = false telemetry?.trackBillingEvent({ @@ -574,6 +581,7 @@ async function handleBuy() { } else { // Synchronous 'failed' here means the charge was declined, not rejected pre-attempt. paymentSubmitted.value = false + topupIdempotencyKey = undefined telemetry?.trackBillingEvent({ operation: 'topup', stage: 'failed', diff --git a/src/platform/workspace/composables/useWorkspaceBilling.test.ts b/src/platform/workspace/composables/useWorkspaceBilling.test.ts index a07898027468..22c64bae06c1 100644 --- a/src/platform/workspace/composables/useWorkspaceBilling.test.ts +++ b/src/platform/workspace/composables/useWorkspaceBilling.test.ts @@ -1547,9 +1547,12 @@ describe('useWorkspaceBilling', () => { mockWorkspaceApi.createTopup.mockResolvedValue(topupResponse) const billing = setupBilling() - const result = await billing.topup(500) + const result = await billing.topup(500, 'topup-attempt-1') - expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(result).toBe(topupResponse) expect(mockWorkspaceApi.getBillingStatus).not.toHaveBeenCalled() expect(mockWorkspaceApi.getBillingBalance).not.toHaveBeenCalled() @@ -1634,9 +1637,12 @@ describe('useWorkspaceBilling', () => { mockWorkspaceApi.createTopup.mockResolvedValue(topupResponse) const billing = setupBilling() - const result = await billing.topup(500) + const result = await billing.topup(500, 'topup-attempt-1') - expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith(500) + expect(mockWorkspaceApi.createTopup).toHaveBeenCalledWith( + 500, + 'topup-attempt-1' + ) expect(result).toBe(topupResponse) expect(mockWorkspaceApi.getBillingStatus).not.toHaveBeenCalled() expect(mockWorkspaceApi.getBillingBalance).not.toHaveBeenCalled() diff --git a/src/platform/workspace/composables/useWorkspaceBilling.ts b/src/platform/workspace/composables/useWorkspaceBilling.ts index 1f4bdf07235c..4c07ede821ca 100644 --- a/src/platform/workspace/composables/useWorkspaceBilling.ts +++ b/src/platform/workspace/composables/useWorkspaceBilling.ts @@ -535,11 +535,14 @@ export function useWorkspaceBilling(): BillingState & BillingActions { } } - async function topup(amountCents: number): Promise { + async function topup( + amountCents: number, + idempotencyKey?: string + ): Promise { isLoading.value = true error.value = null try { - return await workspaceApi.createTopup(amountCents) + return await workspaceApi.createTopup(amountCents, idempotencyKey) } catch (err) { error.value = err instanceof Error ? err.message : 'Failed to top up credits' From e0e7a94ed550a03f498fa66cfd75cea76cf628aa Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:29:56 -0700 Subject: [PATCH 16/23] test: cover real Cloud saved-card top-ups --- .../fixtures/helpers/LiveCloudTopup.ts | 111 ++++++++++++++++++ .../fixtures/liveCloudPaidBillingFixture.ts | 17 +++ browser_tests/playwrightSelection.test.ts | 14 ++- .../liveCloud/paid/savedCardTopup.spec.ts | 9 ++ docs/testing/cloud-billing-e2e.md | 15 +++ playwright.config.ts | 38 +++--- 6 files changed, 183 insertions(+), 21 deletions(-) create mode 100644 browser_tests/fixtures/helpers/LiveCloudTopup.ts create mode 100644 browser_tests/fixtures/liveCloudPaidBillingFixture.ts create mode 100644 browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudTopup.ts b/browser_tests/fixtures/helpers/LiveCloudTopup.ts new file mode 100644 index 000000000000..156e1a351991 --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudTopup.ts @@ -0,0 +1,111 @@ +import { + zBillingBalanceResponse, + zBillingOpStatusResponse, + zBillingStatusResponse, + zCreateTopupResponse, + zListSavedPaymentMethodsResponse +} from '@comfyorg/ingest-types/zod' +import { expect } from '@playwright/test' +import type { TestInfo } from '@playwright/test' + +import type { ComfyPage } from '@e2e/fixtures/ComfyPage' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { matchesBillingResponse } from '@e2e/fixtures/helpers/LiveCloudBilling' + +export class LiveCloudTopup { + private readonly dialog + + constructor( + private readonly comfyPage: ComfyPage, + private readonly session: LiveCloudBillingSession, + private readonly frontend: string + ) { + this.dialog = comfyPage.page.getByRole('dialog') + } + + async open() { + const status = await this.session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.is_active).toBe(true) + expect(status.subscription_tier).toBe('CREATOR') + const methods = await this.session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).not.toHaveLength(0) + await this.comfyPage.page.goto(`${this.frontend}/?topup=1`) + await this.dialog.getByRole('button', { name: '$10', exact: true }).click() + await this.dialog + .getByRole('button', { name: 'Add credits', exact: true }) + .click() + await expect( + this.dialog.getByText( + 'Your saved payment method is charged immediately.', + { exact: true } + ) + ).toBeVisible() + } + + async verifyPurchase(testInfo: TestInfo) { + const before = await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const [response] = await Promise.all([ + this.comfyPage.page.waitForResponse( + (response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/topup' + ) && response.request().method() === 'POST' + ), + this.dialog + .getByRole('button', { name: 'Pay $10.00', exact: true }) + .click() + ]) + expect(response.status()).toBe(200) + const topup = zCreateTopupResponse.parse(await response.json()) + expect(topup.amount_cents).toBe(1000n) + expect(topup.billing_op_id).not.toBe('') + await testInfo.attach('topup.json', { + body: JSON.stringify({ + operationId: topup.billing_op_id, + topupId: topup.topup_id, + amountCents: Number(topup.amount_cents), + balanceBeforeCents: before.amount_micros + }), + contentType: 'application/json' + }) + await expect( + this.comfyPage.page.getByText('Credits added successfully', { + exact: true + }) + ).toBeVisible() + await expect + .poll(async () => { + const operation = await this.session.read( + `/api/billing/ops/${encodeURIComponent(topup.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { id: operation.id, status: operation.status } + }) + .toMatchObject({ id: topup.billing_op_id, status: 'succeeded' }) + await expect + .poll( + async () => + ( + await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + ).amount_micros + ) + .toBe(before.amount_micros + Number(topup.amount_cents)) + await this.comfyPage.attachScreenshot('saved-card-topup.png', { + runInCI: true + }) + } +} diff --git a/browser_tests/fixtures/liveCloudPaidBillingFixture.ts b/browser_tests/fixtures/liveCloudPaidBillingFixture.ts new file mode 100644 index 000000000000..aabdb618d05d --- /dev/null +++ b/browser_tests/fixtures/liveCloudPaidBillingFixture.ts @@ -0,0 +1,17 @@ +import { LiveCloudTopup } from '@e2e/fixtures/helpers/LiveCloudTopup' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +export const liveCloudPaidBillingFixture = liveCloudBillingFixture.extend<{ + topup: LiveCloudTopup +}>({ + topup: async ({ comfyPage, billingSession }, use) => { + const topup = new LiveCloudTopup( + comfyPage, + billingSession, + loadLiveCloudBillingConfig().PLAYWRIGHT_TEST_URL + ) + await topup.open() + await use(topup) + } +}) diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 8aca5f0e22c1..a85eb5879ea3 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -4,17 +4,21 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' describe('Live billing opt-in', () => { - it.for(['0', '1'])( - 'collects live billing only when explicitly enabled (%s)', + it.for([ + { enabled: '0', project: 'cloud-live', count: 0 }, + { enabled: '1', project: 'cloud-live', count: 3 }, + { enabled: '1', project: 'cloud-live-paid', count: 1 } + ])( + 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, - (enabled) => { + ({ enabled, project, count }) => { const result = spawnSync( 'pnpm', [ 'exec', 'playwright', 'test', - ...(enabled === '1' ? ['--project=cloud-live'] : ['tests/liveCloud']), + ...(enabled === '1' ? [`--project=${project}`] : ['tests/liveCloud']), '--list', '--reporter=json' ], @@ -48,7 +52,7 @@ describe('Live billing opt-in', () => { const specs = report.suites.flatMap((suite) => suite.suites.flatMap((child) => child.specs) ) - expect(specs).toHaveLength(enabled === '1' ? 3 : 0) + expect(specs).toHaveLength(count) if (enabled === '1') { expect(report.config.globalSetup).toBeNull() expect(report.config.globalTeardown).toBeNull() diff --git a/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts b/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts new file mode 100644 index 000000000000..e582df49d079 --- /dev/null +++ b/browser_tests/tests/liveCloud/paid/savedCardTopup.spec.ts @@ -0,0 +1,9 @@ +import { liveCloudPaidBillingFixture as test } from '@e2e/fixtures/liveCloudPaidBillingFixture' + +test.describe('Real Cloud saved-card billing', { tag: ['@cloud-live'] }, () => { + test('purchases credits with the saved test card', async ({ + topup + }, testInfo) => { + await topup.verifyPurchase(testInfo) + }) +}) diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index dd13bcdff8a6..0e2cccf1ed2c 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -55,3 +55,18 @@ The no-card tests cover checkout handoff, retry, page reload, and signing in from an empty browser context. Payment completion, the 24-hour timeout, and CI integration remain follow-up work. A successful sign-in alone does not establish billing coverage. + +## Saved-card billing + +Run the separate `cloud-live-paid` project with: + +```sh +pnpm exec cross-env DISTRIBUTION=cloud PLAYWRIGHT_CLOUD_LIVE=1 pnpm exec playwright test --project=cloud-live-paid --workers=1 +``` + +Inject the saved-card account credentials into `CLOUD_ACCOUNT_EMAIL` +and `CLOUD_ACCOUNT_PASSWORD`. This project requires an active Creator plan and +at least one saved sandbox payment method. It purchases $10 of test credits and +checks the completed billing operation, exact balance increase, and success UI. +The balance API returns cents despite its `amount_micros` field name. Never +point this command at the no-card account. diff --git a/playwright.config.ts b/playwright.config.ts index b8c3c6384ffb..4d0efc281cce 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -69,23 +69,29 @@ export default defineConfig({ projects: [ ...(process.env.PLAYWRIGHT_CLOUD_LIVE === '1' ? [ + { name: 'cloud-live', testMatch: '**/tests/liveCloud/*.spec.ts' }, { - name: 'cloud-live', - testMatch: '**/tests/liveCloud/**/*.spec.ts', - testIgnore: [], - fullyParallel: false, - retries: 0, - timeout: 120_000, - expect: { timeout: 30_000 }, - use: { - ...devices['Desktop Chrome'], - locale: 'en-US', - trace: 'off', - video: 'off', - screenshot: 'off' - } - } satisfies NonNullable[number] - ] + name: 'cloud-live-paid', + testMatch: '**/tests/liveCloud/paid/*.spec.ts' + } + ].map( + (project) => + ({ + ...project, + testIgnore: [], + fullyParallel: false, + retries: 0, + timeout: 120_000, + expect: { timeout: 30_000 }, + use: { + ...devices['Desktop Chrome'], + locale: 'en-US', + trace: 'off', + video: 'off', + screenshot: 'off' + } + }) satisfies NonNullable[number] + ) : []), { name: 'chromium', From 3f5f3cff31546c9371530c8b27f15b39b439481f Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:34:34 -0700 Subject: [PATCH 17/23] test: cover idempotent top-up retries --- .../fixtures/helpers/LiveCloudBilling.ts | 18 +++++ .../fixtures/helpers/LiveCloudTopup.ts | 81 ++++++++++++++++--- browser_tests/playwrightSelection.test.ts | 2 +- .../liveCloud/paid/topupIdempotency.spec.ts | 9 +++ 4 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index 3421a27d596d..c41f508f1eb4 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -37,6 +37,24 @@ export class LiveCloudBillingSession { return schema.parse(await response.json()) } + async post( + path: string, + data: Record, + schema: z.ZodType + ): Promise { + const response = await this.request + .post(new URL(path, this.backend).href, { + data, + headers: this.headers, + maxRedirects: 0 + }) + .catch(() => { + throw new Error(`Cloud billing request failed: ${path}`) + }) + expect(response.status(), `POST ${path}`).toBe(200) + return schema.parse(await response.json()) + } + async assertNoCardAccount(testInfo: TestInfo) { const workspace = await this.read( '/api/workspaces/current', diff --git a/browser_tests/fixtures/helpers/LiveCloudTopup.ts b/browser_tests/fixtures/helpers/LiveCloudTopup.ts index 156e1a351991..fbc0164eb4bd 100644 --- a/browser_tests/fixtures/helpers/LiveCloudTopup.ts +++ b/browser_tests/fixtures/helpers/LiveCloudTopup.ts @@ -2,6 +2,7 @@ import { zBillingBalanceResponse, zBillingOpStatusResponse, zBillingStatusResponse, + zCreateTopupRequest, zCreateTopupResponse, zListSavedPaymentMethodsResponse } from '@comfyorg/ingest-types/zod' @@ -53,6 +54,53 @@ export class LiveCloudTopup { '/api/billing/balance', zBillingBalanceResponse ) + const { topup } = await this.submitPurchase() + await this.attachPurchase(testInfo, topup, before.amount_micros) + await expect( + this.comfyPage.page.getByText('Credits added successfully', { + exact: true + }) + ).toBeVisible() + await this.expectPurchaseComplete( + topup.billing_op_id, + before.amount_micros + Number(topup.amount_cents) + ) + await this.comfyPage.attachScreenshot('saved-card-topup.png', { + runInCI: true + }) + } + + async verifyIdempotentRetry(testInfo: TestInfo) { + const before = await this.session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const { request, topup } = await this.submitPurchase() + expect(request.idempotency_key).toBeTruthy() + if (!request.idempotency_key) { + throw new Error('Missing top-up idempotency key') + } + const retry = await this.session.post( + '/api/billing/topup', + { + amount_cents: Number(request.amount_cents), + idempotency_key: request.idempotency_key + }, + zCreateTopupResponse + ) + expect(retry.billing_op_id).toBe(topup.billing_op_id) + expect(retry.topup_id).toBe(topup.topup_id) + await this.attachPurchase(testInfo, topup, before.amount_micros, { + retryOperationId: retry.billing_op_id, + retryTopupId: retry.topup_id + }) + await this.expectPurchaseComplete( + topup.billing_op_id, + before.amount_micros + Number(topup.amount_cents) + ) + } + + private async submitPurchase() { const [response] = await Promise.all([ this.comfyPage.page.waitForResponse( (response) => @@ -67,32 +115,44 @@ export class LiveCloudTopup { .click() ]) expect(response.status()).toBe(200) + const request = zCreateTopupRequest.parse(response.request().postDataJSON()) const topup = zCreateTopupResponse.parse(await response.json()) expect(topup.amount_cents).toBe(1000n) expect(topup.billing_op_id).not.toBe('') + return { request, topup } + } + + private async attachPurchase( + testInfo: TestInfo, + topup: ReturnType, + balanceBeforeCents: number, + extra: Record = {} + ) { await testInfo.attach('topup.json', { body: JSON.stringify({ operationId: topup.billing_op_id, topupId: topup.topup_id, amountCents: Number(topup.amount_cents), - balanceBeforeCents: before.amount_micros + balanceBeforeCents, + ...extra }), contentType: 'application/json' }) - await expect( - this.comfyPage.page.getByText('Credits added successfully', { - exact: true - }) - ).toBeVisible() + } + + private async expectPurchaseComplete( + operationId: string, + expectedBalanceCents: number + ) { await expect .poll(async () => { const operation = await this.session.read( - `/api/billing/ops/${encodeURIComponent(topup.billing_op_id)}`, + `/api/billing/ops/${encodeURIComponent(operationId)}`, zBillingOpStatusResponse ) return { id: operation.id, status: operation.status } }) - .toMatchObject({ id: topup.billing_op_id, status: 'succeeded' }) + .toMatchObject({ id: operationId, status: 'succeeded' }) await expect .poll( async () => @@ -103,9 +163,6 @@ export class LiveCloudTopup { ) ).amount_micros ) - .toBe(before.amount_micros + Number(topup.amount_cents)) - await this.comfyPage.attachScreenshot('saved-card-topup.png', { - runInCI: true - }) + .toBe(expectedBalanceCents) } } diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index a85eb5879ea3..92e255e04f3c 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -7,7 +7,7 @@ describe('Live billing opt-in', () => { it.for([ { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, - { enabled: '1', project: 'cloud-live-paid', count: 1 } + { enabled: '1', project: 'cloud-live-paid', count: 2 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts b/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts new file mode 100644 index 000000000000..ae09f759943a --- /dev/null +++ b/browser_tests/tests/liveCloud/paid/topupIdempotency.spec.ts @@ -0,0 +1,9 @@ +import { liveCloudPaidBillingFixture as test } from '@e2e/fixtures/liveCloudPaidBillingFixture' + +test.describe('Real Cloud saved-card billing', { tag: ['@cloud-live'] }, () => { + test('retries one top-up without granting credits twice', async ({ + topup + }, testInfo) => { + await topup.verifyIdempotentRetry(testInfo) + }) +}) From c4e15170eaa67b8ba438693627f9231ac16f2abc Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:47:03 -0700 Subject: [PATCH 18/23] test: complete real Cloud checkout --- .../fixtures/helpers/LiveCloudBilling.ts | 159 +++++++++++++++--- .../helpers/LiveCloudDisposableAccount.ts | 33 ++++ .../fixtures/liveCloudBillingFixture.ts | 7 +- .../liveCloudDisposableBillingFixture.ts | 34 ++++ .../fixtures/utils/liveCloudBillingConfig.ts | 25 ++- .../fixtures/utils/liveCloudBillingContext.ts | 40 +++-- browser_tests/playwrightSelection.test.ts | 3 +- .../disposable/checkoutCompletion.spec.ts | 23 +++ playwright.config.ts | 4 + 9 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts create mode 100644 browser_tests/fixtures/liveCloudDisposableBillingFixture.ts create mode 100644 browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index c41f508f1eb4..804d345acd01 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -1,8 +1,10 @@ import { + zBillingBalanceResponse, zBillingOpStatusResponse, zBillingStatusResponse, zCurrentWorkspaceResponse, zListSavedPaymentMethodsResponse, + zPaymentPortalResponse, zPreviewSubscribeResponse, zSubscribeResponse } from '@comfyorg/ingest-types/zod' @@ -103,6 +105,14 @@ export class LiveCloudBillingSession { phase: 'awaiting_payment_method' }) } + + async ensureProvisioned(returnUrl: string) { + await this.post( + '/api/billing/payment-portal', + { return_url: returnUrl }, + zPaymentPortalResponse + ) + } } export function matchesBillingResponse( @@ -140,31 +150,140 @@ export class LiveCloudCheckout { }) } - async open() { - const [response] = await Promise.all([ - this.page.waitForResponse((response) => - matchesBillingResponse( - response, - this.frontend, - '/api/billing/preview-subscribe' - ) - ), - this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) - ]) - expect(response.status()).toBe(200) - const preview = zPreviewSubscribeResponse.parse(await response.json()) + async open(retryTransientFailures = false) { + let preview: ReturnType | undefined + const loadPreview = async () => { + const [response] = await Promise.all([ + this.page.waitForResponse((response) => + matchesBillingResponse( + response, + this.frontend, + '/api/billing/preview-subscribe' + ) + ), + this.page.goto(`${this.frontend}/?pricing=creator&cycle=monthly`) + ]) + expect(response.status()).toBe(200) + preview = zPreviewSubscribeResponse.parse(await response.json()) + } + if (retryTransientFailures) { + await expect(loadPreview).toPass({ timeout: 90_000 }) + } else { + await loadPreview() + } + if (!preview) throw new Error('Cloud billing preview was not loaded') expect(preview.allowed).toBe(true) expect(preview.new_plan.slug).toBe('creator-monthly') await expect( this.confirmation.getByText('Total due today', { exact: true }) ).toBeVisible() await expect(this.subscribe.or(this.resumePayment)).toBeEnabled() + return preview } async abandonCheckout(testInfo: TestInfo, resume = false) { const action = resume ? this.resumePayment : this.subscribe.or(this.resumePayment) + const { checkout, subscription } = await this.startCheckout(action) + await testInfo.attach('billing-operation.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + status: subscription.status + }), + contentType: 'application/json' + }) + await testInfo.attach('stripe-checkout.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await checkout.close() + return subscription.billing_op_id + } + + async completeCheckout(session: LiveCloudBillingSession, testInfo: TestInfo) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await checkout + .getByLabel('Card number', { exact: true }) + .fill('4242424242424242') + await checkout.getByLabel('Expiration', { exact: true }).fill('1230') + await checkout + .getByRole('textbox', { + name: 'Credit or debit card CVC/CVV', + exact: true + }) + .fill('123') + await checkout + .getByPlaceholder('Full name on card', { exact: true }) + .fill('Comfy Billing Test') + await checkout + .getByRole('button', { name: 'Enter address manually', exact: true }) + .click() + await checkout.locator('#billingAddressLine1').fill('123 Test Street') + await checkout.getByLabel('City', { exact: true }).fill('San Francisco') + await checkout.getByLabel('State', { exact: true }).selectOption('CA') + await checkout.getByLabel('ZIP', { exact: true }).fill('94107') + await checkout.locator('#enableStripePass').uncheck() + await checkout.getByRole('button', { name: /^Save/ }).click() + await expect + .poll(async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return operation.status + }) + .toBe('succeeded') + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + is_active: true, + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const expectedBalance = + balanceBefore.amount_micros + Number(preview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedBalance) + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(1) + await testInfo.attach('checkout-completion.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: expectedBalance, + paymentMethodCount: methods.length + }), + contentType: 'application/json' + }) + await this.page.goto(this.frontend) + await this.attachScreenshot('checkout-completed.png') + return { + operationId: subscription.billing_op_id, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + paymentMethodCount: methods.length + } + } + + private async startCheckout(action = this.subscribe.or(this.resumePayment)) { await expect(action).toBeEnabled() const [response, checkout] = await Promise.all([ this.page.waitForResponse( @@ -186,19 +305,7 @@ export class LiveCloudCheckout { /^https:\/\/checkout\.(?:stripe\.com|comfy\.org)\/c\/pay\/cs_test_/ expect(subscription.payment_method_url).toMatch(testCheckoutUrl) await expect(checkout).toHaveURL(testCheckoutUrl) - await testInfo.attach('billing-operation.json', { - body: JSON.stringify({ - operationId: subscription.billing_op_id, - status: subscription.status - }), - contentType: 'application/json' - }) - await testInfo.attach('stripe-checkout.png', { - body: await checkout.screenshot(), - contentType: 'image/png' - }) - await checkout.close() - return subscription.billing_op_id + return { checkout, subscription } } async attachScreenshot(name: string) { diff --git a/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts new file mode 100644 index 000000000000..af892f0373dc --- /dev/null +++ b/browser_tests/fixtures/helpers/LiveCloudDisposableAccount.ts @@ -0,0 +1,33 @@ +import { randomUUID } from 'node:crypto' + +import type { APIRequestContext } from '@playwright/test' +import { expect } from '@playwright/test' +import { z } from 'zod' + +const firebaseApiKey = 'AIzaSyDa_YMeyzV0SkVe92vBZ1tVikWBmOU5KVE' + +const signUpResponseSchema = z.object({ localId: z.string().min(1) }) + +export async function createDisposableCloudAccount(request: APIRequestContext) { + const runId = randomUUID() + const credentials = { + email: `billing-e2e-checkout-completion-${runId}@comfy.org`, + password: `BillingE2E-${randomUUID()}!` + } + const response = await request + .post( + `https://identitytoolkit.googleapis.com/v1/accounts:signUp?key=${firebaseApiKey}`, + { + data: { + ...credentials, + returnSecureToken: true + } + } + ) + .catch(() => { + throw new Error('Disposable Cloud account creation failed') + }) + expect(response.status(), 'Firebase account creation').toBe(200) + const account = signUpResponseSchema.parse(await response.json()) + return { ...credentials, userId: account.localId } +} diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index e96590dd8b7a..e1b35afa84e1 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -12,7 +12,10 @@ import { installLiveCloudBillingRouting, signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' -import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + loadLiveCloudBillingConfig, + loadLiveCloudBillingEndpoints +} from '@e2e/fixtures/utils/liveCloudBillingConfig' interface FreshBillingSession { billingSession: LiveCloudBillingSession @@ -27,7 +30,7 @@ export const liveCloudBillingFixture = base.extend<{ }>({ baseURL: process.env.PLAYWRIGHT_TEST_URL, networkPolicy: async ({ baseURL }, use, testInfo) => { - const config = loadLiveCloudBillingConfig() + const config = loadLiveCloudBillingEndpoints() const origins = new Set([ new URL(baseURL ?? config.PLAYWRIGHT_TEST_URL).origin, config.PLAYWRIGHT_SETUP_API_URL, diff --git a/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts new file mode 100644 index 000000000000..74cf953f1df6 --- /dev/null +++ b/browser_tests/fixtures/liveCloudDisposableBillingFixture.ts @@ -0,0 +1,34 @@ +import { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudBilling' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBilling' +import { createDisposableCloudAccount } from '@e2e/fixtures/helpers/LiveCloudDisposableAccount' +import { liveCloudBillingFixture } from '@e2e/fixtures/liveCloudBillingFixture' +import { signInToLiveCloud } from '@e2e/fixtures/utils/liveCloudBillingContext' +import { loadLiveCloudBillingEndpoints } from '@e2e/fixtures/utils/liveCloudBillingConfig' + +interface DisposableCheckout { + billingSession: LiveCloudBillingSession + checkout: LiveCloudCheckout +} + +export const liveCloudDisposableBillingFixture = + liveCloudBillingFixture.extend<{ + disposableCheckout: DisposableCheckout + }>({ + disposableCheckout: async ({ comfyPage, context }, use, testInfo) => { + const account = await createDisposableCloudAccount(context.request) + const billingSession = await signInToLiveCloud(comfyPage.page, account) + await billingSession.assertNoCardAccount(testInfo) + const frontend = loadLiveCloudBillingEndpoints().PLAYWRIGHT_TEST_URL + await billingSession.ensureProvisioned(frontend) + await testInfo.attach('disposable-account.json', { + body: JSON.stringify({ email: account.email, userId: account.userId }), + contentType: 'application/json' + }) + await use({ + billingSession, + checkout: new LiveCloudCheckout(comfyPage, frontend) + }) + } + }) + +export { expect } from '@playwright/test' diff --git a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts index cb382c4d1906..76ee16679bdc 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingConfig.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingConfig.ts @@ -16,7 +16,7 @@ const sandboxURL = z }, 'Use a Cloud test, staging, or PR preview origin') .transform((value) => new URL(value).origin) -export const liveCloudBillingConfigSchema = z +const liveCloudBillingEndpointsSchema = z .object({ PLAYWRIGHT_TEST_URL: z .string() @@ -27,9 +27,7 @@ export const liveCloudBillingConfigSchema = z return url.href === `${url.origin}/` }, 'Use an origin without a path, query, credentials, or fragment') .transform((value) => new URL(value).origin), - PLAYWRIGHT_SETUP_API_URL: sandboxURL, - CLOUD_ACCOUNT_EMAIL: z.string().email(), - CLOUD_ACCOUNT_PASSWORD: z.string().min(1) + PLAYWRIGHT_SETUP_API_URL: sandboxURL }) .superRefine((config, ctx) => { if (!URL.canParse(config.PLAYWRIGHT_TEST_URL)) return @@ -49,6 +47,25 @@ export const liveCloudBillingConfigSchema = z } }) +export const liveCloudBillingConfigSchema = liveCloudBillingEndpointsSchema.and( + z.object({ + CLOUD_ACCOUNT_EMAIL: z.string().email(), + CLOUD_ACCOUNT_PASSWORD: z.string().min(1) + }) +) + +export function loadLiveCloudBillingEndpoints() { + const result = liveCloudBillingEndpointsSchema.safeParse(process.env) + if (!result.success) { + throw new Error( + `Cloud billing prerequisites: ${result.error.issues + .map((issue) => issue.path.join('.')) + .join(', ')}. See docs/testing/cloud-billing-e2e.md.` + ) + } + return result.data +} + export function loadLiveCloudBillingConfig() { const result = liveCloudBillingConfigSchema.safeParse(process.env) if (!result.success) { diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index f09aeac9c891..3c390eb2848c 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -7,13 +7,21 @@ import { matchesBillingResponse } from '@e2e/fixtures/helpers/LiveCloudBilling' import type { NetworkPolicy } from '@e2e/fixtures/networkIsolationFixture' -import { loadLiveCloudBillingConfig } from '@e2e/fixtures/utils/liveCloudBillingConfig' +import { + loadLiveCloudBillingConfig, + loadLiveCloudBillingEndpoints +} from '@e2e/fixtures/utils/liveCloudBillingConfig' + +interface LiveCloudCredentials { + email: string + password: string +} export async function installLiveCloudBillingRouting( context: BrowserContext, networkPolicy: NetworkPolicy ) { - const config = loadLiveCloudBillingConfig() + const config = loadLiveCloudBillingEndpoints() await context.route('**/*', async (route) => { const request = route.request() const url = new URL(request.url()) @@ -56,27 +64,37 @@ export async function installLiveCloudBillingRouting( }) } -export async function signInToLiveCloud(page: Page) { - const config = loadLiveCloudBillingConfig() +export async function signInToLiveCloud( + page: Page, + credentials?: LiveCloudCredentials +) { + const endpoints = loadLiveCloudBillingEndpoints() + const account = + credentials ?? + (() => { + const config = loadLiveCloudBillingConfig() + return { + email: config.CLOUD_ACCOUNT_EMAIL, + password: config.CLOUD_ACCOUNT_PASSWORD + } + })() await new FeatureFlagHelper(page).seedFlags({ onboarding_survey_enabled: false }) - await page.goto(`${config.PLAYWRIGHT_TEST_URL}/cloud/login`) + await page.goto(`${endpoints.PLAYWRIGHT_TEST_URL}/cloud/login`) await page .getByRole('button', { name: 'Use email instead', exact: true }) .click() await page .getByRole('textbox', { name: 'Email', exact: true }) - .fill(config.CLOUD_ACCOUNT_EMAIL) - await page - .getByLabel('Password', { exact: true }) - .fill(config.CLOUD_ACCOUNT_PASSWORD) + .fill(account.email) + await page.getByLabel('Password', { exact: true }).fill(account.password) const [response] = await Promise.all([ page.waitForResponse( (response) => matchesBillingResponse( response, - config.PLAYWRIGHT_TEST_URL, + endpoints.PLAYWRIGHT_TEST_URL, '/api/billing/status' ) && response.status() === 200 ), @@ -87,7 +105,7 @@ export async function signInToLiveCloud(page: Page) { if (!authorization) throw new Error('Missing billing authorization') return new LiveCloudBillingSession( page.request, - config.PLAYWRIGHT_SETUP_API_URL, + endpoints.PLAYWRIGHT_SETUP_API_URL, { authorization } ) } diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 92e255e04f3c..93c0fcfe60d7 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -7,7 +7,8 @@ describe('Live billing opt-in', () => { it.for([ { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, - { enabled: '1', project: 'cloud-live-paid', count: 2 } + { enabled: '1', project: 'cloud-live-paid', count: 2 }, + { enabled: '1', project: 'cloud-live-disposable', count: 1 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts b/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts new file mode 100644 index 000000000000..df4ea4e3d5e1 --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/checkoutCompletion.spec.ts @@ -0,0 +1,23 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe( + 'Real Cloud checkout completion', + { tag: ['@cloud-live'] }, + () => { + test('activates Creator and grants the quoted credits', async ({ + disposableCheckout + }, testInfo) => { + const completion = await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(completion).toMatchObject({ + planSlug: 'creator-monthly', + paymentMethodCount: 1 + }) + }) + } +) diff --git a/playwright.config.ts b/playwright.config.ts index 4d0efc281cce..10492d68be20 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -73,6 +73,10 @@ export default defineConfig({ { name: 'cloud-live-paid', testMatch: '**/tests/liveCloud/paid/*.spec.ts' + }, + { + name: 'cloud-live-disposable', + testMatch: '**/tests/liveCloud/disposable/*.spec.ts' } ].map( (project) => From d2e2a45d7b7c1851835eaa9922346ac53da3464f Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Fri, 11 Sep 2026 23:59:54 -0700 Subject: [PATCH 19/23] test: cover declined Cloud checkout --- .../fixtures/helpers/LiveCloudBilling.ts | 99 ++++++++++++++----- browser_tests/playwrightSelection.test.ts | 2 +- .../liveCloud/disposable/cardDecline.spec.ts | 19 ++++ 3 files changed, 97 insertions(+), 23 deletions(-) create mode 100644 browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index 804d345acd01..b1d796450148 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -210,28 +210,7 @@ export class LiveCloudCheckout { const { checkout, subscription } = await this.startCheckout( this.subscribe.or(this.resumePayment) ) - await checkout - .getByLabel('Card number', { exact: true }) - .fill('4242424242424242') - await checkout.getByLabel('Expiration', { exact: true }).fill('1230') - await checkout - .getByRole('textbox', { - name: 'Credit or debit card CVC/CVV', - exact: true - }) - .fill('123') - await checkout - .getByPlaceholder('Full name on card', { exact: true }) - .fill('Comfy Billing Test') - await checkout - .getByRole('button', { name: 'Enter address manually', exact: true }) - .click() - await checkout.locator('#billingAddressLine1').fill('123 Test Street') - await checkout.getByLabel('City', { exact: true }).fill('San Francisco') - await checkout.getByLabel('State', { exact: true }).selectOption('CA') - await checkout.getByLabel('ZIP', { exact: true }).fill('94107') - await checkout.locator('#enableStripePass').uncheck() - await checkout.getByRole('button', { name: /^Save/ }).click() + await this.submitCard(checkout, '4242424242424242') await expect .poll(async () => { const operation = await session.read( @@ -283,6 +262,82 @@ export class LiveCloudCheckout { } } + async declineCheckout(session: LiveCloudBillingSession, testInfo: TestInfo) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + const observeFailure = async () => { + await expect( + this.page.getByText('Your bank declined this payment', { exact: false }) + ).toBeVisible() + await this.attachScreenshot('checkout-declined.png') + } + await Promise.all([ + observeFailure(), + this.submitCard(checkout, '4000000000000341') + ]) + await expect + .poll(async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status + } + }) + .toMatchObject({ status: 'failed' }) + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.subscription_tier).toBe('FREE') + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + await testInfo.attach('checkout-decline.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros, + status: 'failed', + failureKind: 'card_declined_ui' + }), + contentType: 'application/json' + }) + return { status: 'failed', failureKind: 'card_declined_ui' } + } + + private async submitCard(checkout: Page, cardNumber: string) { + await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) + await checkout.getByLabel('Expiration', { exact: true }).fill('1230') + await checkout + .getByRole('textbox', { + name: 'Credit or debit card CVC/CVV', + exact: true + }) + .fill('123') + await checkout + .getByPlaceholder('Full name on card', { exact: true }) + .fill('Comfy Billing Test') + await checkout + .getByRole('button', { name: 'Enter address manually', exact: true }) + .click() + await checkout.locator('#billingAddressLine1').fill('123 Test Street') + await checkout.getByLabel('City', { exact: true }).fill('San Francisco') + await checkout.getByLabel('State', { exact: true }).selectOption('CA') + await checkout.getByLabel('ZIP', { exact: true }).fill('94107') + await checkout.locator('#enableStripePass').uncheck() + await checkout.getByRole('button', { name: /^Save/ }).click() + } + private async startCheckout(action = this.subscribe.or(this.resumePayment)) { await expect(action).toBeEnabled() const [response, checkout] = await Promise.all([ diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 93c0fcfe60d7..92bd6323b4bc 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -8,7 +8,7 @@ describe('Live billing opt-in', () => { { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, { enabled: '1', project: 'cloud-live-paid', count: 2 }, - { enabled: '1', project: 'cloud-live-disposable', count: 1 } + { enabled: '1', project: 'cloud-live-disposable', count: 2 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts b/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts new file mode 100644 index 000000000000..0a0c2ba270be --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/cardDecline.spec.ts @@ -0,0 +1,19 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud checkout failure', { tag: ['@cloud-live'] }, () => { + test('keeps the account on Free after a card decline', async ({ + disposableCheckout + }, testInfo) => { + const failure = await disposableCheckout.checkout.declineCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(failure).toMatchObject({ + status: 'failed', + failureKind: 'card_declined_ui' + }) + }) +}) From 6f6903e5c8fc14033e2fa4ac0d7ea05fd976bdf2 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:15:59 -0700 Subject: [PATCH 20/23] test: cover Cloud 3DS completion --- .../fixtures/helpers/LiveCloudBilling.ts | 163 +++++++++++++----- .../fixtures/liveCloudBillingFixture.ts | 2 + .../fixtures/utils/liveCloudBillingContext.ts | 1 + browser_tests/playwrightSelection.test.ts | 2 +- .../disposable/threeDSComplete.spec.ts | 22 +++ 5 files changed, 142 insertions(+), 48 deletions(-) create mode 100644 browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index b1d796450148..e5efd5bdcb8c 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -85,7 +85,7 @@ export class LiveCloudBillingSession { }) if (status.is_active) expect(status.subscription_tier).toBe('FREE') else - expect(['inactive', 'awaiting_payment_method']).toContain( + expect([undefined, 'inactive', 'awaiting_payment_method']).toContain( status.billing_status ) expect(methods).toHaveLength(0) @@ -211,55 +211,58 @@ export class LiveCloudCheckout { this.subscribe.or(this.resumePayment) ) await this.submitCard(checkout, '4242424242424242') + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + 'checkout-completed.png' + ) + } + + async completeAuthenticatedCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' await expect - .poll(async () => { - const operation = await session.read( - `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, - zBillingOpStatusResponse - ) - return operation.status - }) - .toBe('succeeded') - await expect - .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) - .toMatchObject({ - is_active: true, - plan_slug: 'creator-monthly', - subscription_tier: 'CREATOR' - }) - const expectedBalance = - balanceBefore.amount_micros + Number(preview.credits_today_cents) - await expect - .poll( - async () => - (await session.read('/api/billing/balance', zBillingBalanceResponse)) - .amount_micros + .poll(() => + checkout + .frames() + .some((frame) => frame.url().startsWith(challengeOrigin)) ) - .toBe(expectedBalance) - const methods = await session.read( - '/api/billing/payment-methods', - zListSavedPaymentMethodsResponse - ) - expect(methods).toHaveLength(1) - await testInfo.attach('checkout-completion.json', { - body: JSON.stringify({ - operationId: subscription.billing_op_id, - planSlug: preview.new_plan.slug, - grantCents: Number(preview.credits_today_cents), - balanceBeforeCents: balanceBefore.amount_micros, - balanceAfterCents: expectedBalance, - paymentMethodCount: methods.length - }), - contentType: 'application/json' + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const completeAuthentication = challenge.getByRole('button', { + name: /complete/i }) - await this.page.goto(this.frontend) - await this.attachScreenshot('checkout-completed.png') - return { - operationId: subscription.billing_op_id, - planSlug: preview.new_plan.slug, - grantCents: Number(preview.credits_today_cents), - paymentMethodCount: methods.length - } + await expect(completeAuthentication).toBeVisible() + await testInfo.attach('3ds-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await completeAuthentication.click() + return await this.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + '3ds-completed.png' + ) } async declineCheckout(session: LiveCloudBillingSession, testInfo: TestInfo) { @@ -338,6 +341,72 @@ export class LiveCloudCheckout { await checkout.getByRole('button', { name: /^Save/ }).click() } + private async verifyCheckoutCompletion( + session: LiveCloudBillingSession, + testInfo: TestInfo, + balanceBeforeCents: number, + preview: ReturnType, + operationId: string, + screenshotName: string + ) { + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + phase: operation.phase + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ status: 'succeeded' }) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + is_active: true, + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const expectedBalance = + balanceBeforeCents + Number(preview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedBalance) + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(1) + await testInfo.attach('checkout-completion.json', { + body: JSON.stringify({ + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + balanceBeforeCents, + balanceAfterCents: expectedBalance, + paymentMethodCount: methods.length + }), + contentType: 'application/json' + }) + await this.page.goto(this.frontend) + await this.attachScreenshot(screenshotName) + return { + operationId, + planSlug: preview.new_plan.slug, + grantCents: Number(preview.credits_today_cents), + paymentMethodCount: methods.length + } + } + private async startCheckout(action = this.subscribe.or(this.resumePayment)) { await expect(action).toBeEnabled() const [response, checkout] = await Promise.all([ diff --git a/browser_tests/fixtures/liveCloudBillingFixture.ts b/browser_tests/fixtures/liveCloudBillingFixture.ts index e1b35afa84e1..66ffd1aacc5f 100644 --- a/browser_tests/fixtures/liveCloudBillingFixture.ts +++ b/browser_tests/fixtures/liveCloudBillingFixture.ts @@ -42,6 +42,8 @@ export const liveCloudBillingFixture = base.extend<{ 'https://dreamboothy-dev.firebaseapp.com', 'https://checkout.stripe.com', 'https://checkout.comfy.org', + 'https://testmode-acs.stripe.com', + 'https://hooks.stripe.com', 'https://api.stripe.com', 'https://js.stripe.com', 'https://m.stripe.network', diff --git a/browser_tests/fixtures/utils/liveCloudBillingContext.ts b/browser_tests/fixtures/utils/liveCloudBillingContext.ts index 3c390eb2848c..bbeaf1f92ec9 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingContext.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingContext.ts @@ -100,6 +100,7 @@ export async function signInToLiveCloud( ), page.getByRole('button', { name: 'Sign in', exact: true }).click() ]) + await page.goto(endpoints.PLAYWRIGHT_TEST_URL) const authorization = await response.request().headerValue('authorization') expect(authorization).toBeTruthy() if (!authorization) throw new Error('Missing billing authorization') diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 92bd6323b4bc..a5e83e65f57c 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -8,7 +8,7 @@ describe('Live billing opt-in', () => { { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, { enabled: '1', project: 'cloud-live-paid', count: 2 }, - { enabled: '1', project: 'cloud-live-disposable', count: 2 } + { enabled: '1', project: 'cloud-live-disposable', count: 3 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts new file mode 100644 index 000000000000..1515e83430bc --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts @@ -0,0 +1,22 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('activates Creator after successful authentication', async ({ + disposableCheckout + }, testInfo) => { + const completion = + await disposableCheckout.checkout.completeAuthenticatedCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(completion).toMatchObject({ + planSlug: 'creator-monthly', + paymentMethodCount: 1 + }) + }) +}) From 4812ad94a96e594b0f690be0e27eec3a72918e24 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:21:58 -0700 Subject: [PATCH 21/23] test: cover Cloud 3DS failure --- .../fixtures/helpers/LiveCloudBilling.ts | 73 +++++++++++++++++++ browser_tests/playwrightSelection.test.ts | 2 +- .../disposable/threeDSFailure.spec.ts | 20 +++++ 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index e5efd5bdcb8c..5cce455a9221 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -318,6 +318,79 @@ export class LiveCloudCheckout { return { status: 'failed', failureKind: 'card_declined_ui' } } + async failAuthenticatedCheckout( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + await this.open(true) + const { checkout, subscription } = await this.startCheckout( + this.subscribe.or(this.resumePayment) + ) + await this.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' + await expect + .poll(() => + checkout + .frames() + .some((frame) => frame.url().startsWith(challengeOrigin)) + ) + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const failAuthentication = challenge.getByRole('button', { name: /fail/i }) + await expect(failAuthentication).toBeVisible() + await testInfo.attach('3ds-failure-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await failAuthentication.click() + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + declineReason: operation.decline_reason + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ + status: 'failed', + authenticationState: 'failed_retryable' + }) + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.subscription_tier).toBe('FREE') + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + await testInfo.attach('3ds-failure.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros, + authenticationState: 'failed_retryable' + }), + contentType: 'application/json' + }) + return { authenticationState: 'failed_retryable' } + } + private async submitCard(checkout: Page, cardNumber: string) { await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) await checkout.getByLabel('Expiration', { exact: true }).fill('1230') diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index a5e83e65f57c..506f1f8b1eb9 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -8,7 +8,7 @@ describe('Live billing opt-in', () => { { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, { enabled: '1', project: 'cloud-live-paid', count: 2 }, - { enabled: '1', project: 'cloud-live-disposable', count: 3 } + { enabled: '1', project: 'cloud-live-disposable', count: 4 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts new file mode 100644 index 000000000000..fd82066e16da --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts @@ -0,0 +1,20 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('keeps the account on Free after failed authentication', async ({ + disposableCheckout + }, testInfo) => { + const failure = await disposableCheckout.checkout.failAuthenticatedCheckout( + disposableCheckout.billingSession, + testInfo + ) + expect(failure).toMatchObject({ + authenticationState: 'failed_retryable' + }) + }) +}) From 179834f3c9dacae5cb5a4fe2a457b3b2bb03e95e Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:26:59 -0700 Subject: [PATCH 22/23] test: cover cancellation recovery --- .../fixtures/helpers/LiveCloudBilling.ts | 97 ++++++++++++++++++- browser_tests/playwrightSelection.test.ts | 2 +- .../disposable/cancelReactivate.spec.ts | 27 ++++++ 3 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index 5cce455a9221..499e5a6281ba 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -2,10 +2,13 @@ import { zBillingBalanceResponse, zBillingOpStatusResponse, zBillingStatusResponse, + zCancelSubscriptionAcceptedResponse, + zCancelSubscriptionResponse, zCurrentWorkspaceResponse, zListSavedPaymentMethodsResponse, zPaymentPortalResponse, zPreviewSubscribeResponse, + zResubscribeResponse, zSubscribeResponse } from '@comfyorg/ingest-types/zod' import type { @@ -15,7 +18,7 @@ import type { TestInfo } from '@playwright/test' import { expect } from '@playwright/test' -import type { z } from 'zod' +import { z } from 'zod' import type { ComfyPage } from '@e2e/fixtures/ComfyPage' @@ -42,7 +45,8 @@ export class LiveCloudBillingSession { async post( path: string, data: Record, - schema: z.ZodType + schema: z.ZodType, + acceptedStatuses = [200] ): Promise { const response = await this.request .post(new URL(path, this.backend).href, { @@ -53,7 +57,7 @@ export class LiveCloudBillingSession { .catch(() => { throw new Error(`Cloud billing request failed: ${path}`) }) - expect(response.status(), `POST ${path}`).toBe(200) + expect(acceptedStatuses, `POST ${path}`).toContain(response.status()) return schema.parse(await response.json()) } @@ -106,6 +110,21 @@ export class LiveCloudBillingSession { }) } + async expectSucceeded(operationId: string) { + await expect + .poll( + async () => { + const operation = await this.read( + `/api/billing/ops/${encodeURIComponent(operationId)}`, + zBillingOpStatusResponse + ) + return operation.status + }, + { timeout: 60_000 } + ) + .toBe('succeeded') + } + async ensureProvisioned(returnUrl: string) { await this.post( '/api/billing/payment-portal', @@ -391,6 +410,78 @@ export class LiveCloudCheckout { return { authenticationState: 'failed_retryable' } } + async verifyCancellationRecovery( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const cancel = await session.post( + '/api/billing/subscription/cancel', + { idempotency_key: crypto.randomUUID() }, + z.union([ + zCancelSubscriptionResponse, + zCancelSubscriptionAcceptedResponse + ]), + [200, 202] + ) + await session.expectSucceeded(cancel.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'creator-monthly', + subscription_tier: 'CREATOR' + }) + const canceled = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(canceled.cancel_at).toBeTruthy() + const resubscribe = await session.post( + '/api/billing/subscription/resubscribe', + { idempotency_key: crypto.randomUUID() }, + zResubscribeResponse + ) + if (resubscribe.status === 'pending') { + await session.expectSucceeded(resubscribe.billing_op_id) + } + await expect + .poll(async () => { + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + return { + cancelAt: status.cancel_at, + planSlug: status.plan_slug, + subscriptionStatus: status.subscription_status + } + }) + .toMatchObject({ + cancelAt: undefined, + planSlug: 'creator-monthly', + subscriptionStatus: 'active' + }) + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + const result = { + cancelOperationId: cancel.billing_op_id, + reactivateOperationId: resubscribe.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros + } + await testInfo.attach('cancellation-recovery.json', { + body: JSON.stringify(result), + contentType: 'application/json' + }) + return result + } + private async submitCard(checkout: Page, cardNumber: string) { await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) await checkout.getByLabel('Expiration', { exact: true }).fill('1230') diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 506f1f8b1eb9..78588ae5180d 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -8,7 +8,7 @@ describe('Live billing opt-in', () => { { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, { enabled: '1', project: 'cloud-live-paid', count: 2 }, - { enabled: '1', project: 'cloud-live-disposable', count: 4 } + { enabled: '1', project: 'cloud-live-disposable', count: 5 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts b/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts new file mode 100644 index 000000000000..d7e160658765 --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/cancelReactivate.spec.ts @@ -0,0 +1,27 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe( + 'Real Cloud subscription lifecycle', + { tag: ['@cloud-live'] }, + () => { + test.setTimeout(240_000) + + test('cancels and reactivates without changing the balance', async ({ + disposableCheckout + }, testInfo) => { + await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + const recovery = + await disposableCheckout.checkout.verifyCancellationRecovery( + disposableCheckout.billingSession, + testInfo + ) + expect(recovery.balanceAfterCents).toBe(recovery.balanceBeforeCents) + }) + } +) From 1311aa3fe8795b5992c87dbfaa7bd446685c2bb7 Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Sat, 12 Sep 2026 00:34:30 -0700 Subject: [PATCH 23/23] test: cover Cloud plan transitions --- .../fixtures/helpers/LiveCloudBilling.ts | 108 ++++++++++++++++++ browser_tests/playwrightSelection.test.ts | 2 +- .../disposable/planTransitions.spec.ts | 22 ++++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts diff --git a/browser_tests/fixtures/helpers/LiveCloudBilling.ts b/browser_tests/fixtures/helpers/LiveCloudBilling.ts index 499e5a6281ba..88c8879f5b86 100644 --- a/browser_tests/fixtures/helpers/LiveCloudBilling.ts +++ b/browser_tests/fixtures/helpers/LiveCloudBilling.ts @@ -482,6 +482,114 @@ export class LiveCloudCheckout { return result } + async verifyPlanTransitions( + session: LiveCloudBillingSession, + testInfo: TestInfo + ) { + const methods = await session.read( + '/api/billing/payment-methods', + zListSavedPaymentMethodsResponse + ) + expect(methods).toHaveLength(1) + const balanceBeforeUpgrade = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const upgradePreview = await session.post( + '/api/billing/preview-subscribe', + { plan_slug: 'pro-monthly' }, + zPreviewSubscribeResponse + ) + expect(upgradePreview).toMatchObject({ + allowed: true, + transition_type: 'upgrade' + }) + const upgradeRequest: Record = { + plan_slug: 'pro-monthly' + } + if (upgradePreview.quote_id) { + upgradeRequest.quote_id = upgradePreview.quote_id + } + if (upgradePreview.quote_version !== undefined) { + upgradeRequest.quote_version = upgradePreview.quote_version + } + if (upgradePreview.proration_at) { + upgradeRequest.proration_at = upgradePreview.proration_at + } + const upgrade = await session.post( + '/api/billing/subscribe', + upgradeRequest, + zSubscribeResponse + ) + await session.expectSucceeded(upgrade.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'pro-monthly', + subscription_tier: 'PRO' + }) + const expectedUpgradeBalance = + balanceBeforeUpgrade.amount_micros + + Number(upgradePreview.credits_today_cents) + await expect + .poll( + async () => + (await session.read('/api/billing/balance', zBillingBalanceResponse)) + .amount_micros + ) + .toBe(expectedUpgradeBalance) + const downgradePreview = await session.post( + '/api/billing/preview-subscribe', + { plan_slug: 'standard-monthly' }, + zPreviewSubscribeResponse + ) + expect(downgradePreview).toMatchObject({ + allowed: true, + transition_type: 'downgrade' + }) + const downgradeRequest: Record = { + plan_slug: 'standard-monthly' + } + if (downgradePreview.quote_id) { + downgradeRequest.quote_id = downgradePreview.quote_id + } + if (downgradePreview.quote_version !== undefined) { + downgradeRequest.quote_version = downgradePreview.quote_version + } + if (downgradePreview.proration_at) { + downgradeRequest.proration_at = downgradePreview.proration_at + } + const downgrade = await session.post( + '/api/billing/subscribe', + downgradeRequest, + zSubscribeResponse + ) + await session.expectSucceeded(downgrade.billing_op_id) + await expect + .poll(() => session.read('/api/billing/status', zBillingStatusResponse)) + .toMatchObject({ + plan_slug: 'pro-monthly', + scheduled_change: { plan_slug: 'standard-monthly' } + }) + const balanceAfterDowngrade = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfterDowngrade.amount_micros).toBe(expectedUpgradeBalance) + const result = { + upgradeOperationId: upgrade.billing_op_id, + downgradeOperationId: downgrade.billing_op_id, + balanceBeforeUpgradeCents: balanceBeforeUpgrade.amount_micros, + balanceAfterDowngradeCents: balanceAfterDowngrade.amount_micros, + scheduledPlan: 'standard-monthly' + } + await testInfo.attach('plan-transitions.json', { + body: JSON.stringify(result), + contentType: 'application/json' + }) + return result + } + private async submitCard(checkout: Page, cardNumber: string) { await checkout.getByLabel('Card number', { exact: true }).fill(cardNumber) await checkout.getByLabel('Expiration', { exact: true }).fill('1230') diff --git a/browser_tests/playwrightSelection.test.ts b/browser_tests/playwrightSelection.test.ts index 78588ae5180d..8ac20230f29d 100644 --- a/browser_tests/playwrightSelection.test.ts +++ b/browser_tests/playwrightSelection.test.ts @@ -8,7 +8,7 @@ describe('Live billing opt-in', () => { { enabled: '0', project: 'cloud-live', count: 0 }, { enabled: '1', project: 'cloud-live', count: 3 }, { enabled: '1', project: 'cloud-live-paid', count: 2 }, - { enabled: '1', project: 'cloud-live-disposable', count: 5 } + { enabled: '1', project: 'cloud-live-disposable', count: 6 } ])( 'collects $project only when explicitly enabled ($enabled)', { timeout: 90_000 }, diff --git a/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts b/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts new file mode 100644 index 000000000000..664311a5abd1 --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/planTransitions.spec.ts @@ -0,0 +1,22 @@ +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud plan transitions', { tag: ['@cloud-live'] }, () => { + test.setTimeout(240_000) + + test('upgrades now and schedules a downgrade', async ({ + disposableCheckout + }, testInfo) => { + await disposableCheckout.checkout.completeCheckout( + disposableCheckout.billingSession, + testInfo + ) + const transitions = await disposableCheckout.checkout.verifyPlanTransitions( + disposableCheckout.billingSession, + testInfo + ) + expect(transitions.scheduledPlan).toBe('standard-monthly') + }) +})