From 87988c1d235785a5b857d9de0be8b87e1103a71d Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:49:38 -0700 Subject: [PATCH 1/2] test: isolate 3DS authentication scenarios --- .../utils/liveCloudBillingPolicy.test.ts | 20 +++ .../fixtures/utils/liveCloudBillingPolicy.ts | 4 +- .../disposable/threeDSComplete.spec.ts | 23 ++++ .../disposable/threeDSFailure.spec.ts | 22 ++++ .../liveCloud/helpers/threeDSScenarios.ts | 118 ++++++++++++++++++ docs/testing/cloud-billing-e2e.md | 44 +++++++ 6 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts create mode 100644 browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts create mode 100644 browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index fbdea395459..0c38e05c1ad 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -383,3 +383,23 @@ describe('Disposable payment permissions', () => { ).toBe(false) }) }) + +describe('Stripe test challenge permissions', () => { + it('requires sandbox payment permission for challenge responses', () => { + const url = new URL('https://testmode-acs.stripe.com/authorize') + expect(isLiveCloudMutationAllowed(url, 'POST', config)).toBe(false) + expect( + isLiveCloudMutationAllowed(url, 'POST', { + ...config, + allowPayments: true + }) + ).toBe(true) + expect( + isLiveCloudMutationAllowed(url, 'POST', { + ...config, + allowPayments: true, + PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org' + }) + ).toBe(false) + }) +}) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index 1ddcee00a41..9dfbf26d25c 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -27,7 +27,8 @@ export const LIVE_CHECKOUT_ORIGINS = [ 'https://r.stripe.com', 'https://q.stripe.com', 'https://b.stripecdn.com', - 'https://newassets.hcaptcha.com' + 'https://newassets.hcaptcha.com', + 'https://testmode-acs.stripe.com' ] const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] @@ -118,6 +119,7 @@ function isOnboardingSurveyUpdate(data: unknown): boolean { } function isPaymentPostAllowed(url: URL, backend: string): boolean { + if (url.origin === 'https://testmode-acs.stripe.com') return true if ( url.origin === 'https://checkout.comfy.org' && url.pathname === '/ajax/metrics_batch' diff --git a/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts new file mode 100644 index 00000000000..19b634e26dc --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts @@ -0,0 +1,23 @@ +import { completeAuthenticatedCheckout } from '@e2e/tests/liveCloud/helpers/threeDSScenarios' +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('activates Creator after successful authentication', async ({ + disposableCheckout + }, testInfo) => { + const completion = await completeAuthenticatedCheckout( + disposableCheckout.checkout, + disposableCheckout.billingSession, + testInfo + ) + expect(completion).toMatchObject({ + planSlug: 'creator-monthly', + paymentMethodCount: 1 + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts new file mode 100644 index 00000000000..130d5c27574 --- /dev/null +++ b/browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts @@ -0,0 +1,22 @@ +import { failAuthenticatedCheckout } from '@e2e/tests/liveCloud/helpers/threeDSScenarios' +import { + expect, + liveCloudDisposableBillingFixture as test +} from '@e2e/fixtures/liveCloudDisposableBillingFixture' + +test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => { + test.setTimeout(180_000) + + test('keeps the account on Free after failed authentication', async ({ + disposableCheckout + }, testInfo) => { + const failure = await failAuthenticatedCheckout( + disposableCheckout.checkout, + disposableCheckout.billingSession, + testInfo + ) + expect(failure).toMatchObject({ + authenticationState: 'failed_retryable' + }) + }) +}) diff --git a/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts b/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts new file mode 100644 index 00000000000..74a7c2e9d49 --- /dev/null +++ b/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts @@ -0,0 +1,118 @@ +import { + zBillingBalanceResponse, + zBillingStatusResponse, + zBillingOpStatusResponse +} from '@comfyorg/ingest-types/zod' +import { expect } from '@playwright/test' +import type { TestInfo } from '@playwright/test' +import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' +import type { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudCheckout' +export async function completeAuthenticatedCheckout( + driver: LiveCloudCheckout, + session: LiveCloudBillingSession, + testInfo: TestInfo +) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + const preview = await driver.open(true) + const { checkout, subscription } = await driver.startCheckout() + await driver.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' + await expect + .poll(() => + checkout.frames().some((frame) => frame.url().startsWith(challengeOrigin)) + ) + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const completeAuthentication = challenge.getByRole('button', { + name: /complete/i + }) + await expect(completeAuthentication).toBeVisible() + await testInfo.attach('3ds-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await completeAuthentication.click() + return await driver.verifyCheckoutCompletion( + session, + testInfo, + balanceBefore.amount_micros, + preview, + subscription.billing_op_id, + '3ds-completed.png' + ) +} +export async function failAuthenticatedCheckout( + driver: LiveCloudCheckout, + session: LiveCloudBillingSession, + testInfo: TestInfo +) { + const balanceBefore = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + await driver.open(true) + const { checkout, subscription } = await driver.startCheckout() + await driver.submitCard(checkout, '4000000000003220') + const challengeOrigin = 'https://testmode-acs.stripe.com' + await expect + .poll(() => + checkout.frames().some((frame) => frame.url().startsWith(challengeOrigin)) + ) + .toBe(true) + const challenge = checkout + .frames() + .find((frame) => frame.url().startsWith(challengeOrigin)) + if (!challenge) throw new Error('3D Secure challenge did not load') + const failAuthentication = challenge.getByRole('button', { name: /fail/i }) + await expect(failAuthentication).toBeVisible() + await testInfo.attach('3ds-failure-challenge.png', { + body: await checkout.screenshot(), + contentType: 'image/png' + }) + await failAuthentication.click() + await expect + .poll( + async () => { + const operation = await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + return { + status: operation.status, + authenticationState: operation.authentication_state, + declineReason: operation.decline_reason + } + }, + { timeout: 60_000 } + ) + .toMatchObject({ + status: 'failed', + authenticationState: 'failed_retryable' + }) + const status = await session.read( + '/api/billing/status', + zBillingStatusResponse + ) + expect(status.subscription_tier).toBe('FREE') + const balanceAfter = await session.read( + '/api/billing/balance', + zBillingBalanceResponse + ) + expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) + await testInfo.attach('3ds-failure.json', { + body: JSON.stringify({ + operationId: subscription.billing_op_id, + balanceBeforeCents: balanceBefore.amount_micros, + balanceAfterCents: balanceAfter.amount_micros, + authenticationState: 'failed_retryable' + }), + contentType: 'application/json' + }) + return { authenticationState: 'failed_retryable' } +} diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 96bdcab10e3..92fdea7d193 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -82,3 +82,47 @@ Traces start after permanent-account sign-in and contain authenticated network traffic. Keep reports private. Disposable accounts require no reusable credentials. Completed-payment assertions verify saved cards through the real hosted billing portal, because Staging disables the embedded-checkout saved-card API. + +## 3DS outcomes + +PR #17545 depends only on #17481. Run `--project=cloud-live-disposable` for +successful and failed authentication. Challenge interaction, challenge permissions, +and terminal-state assertions live in this PR. Success activates Creator and +grants once; failure becomes retryable without adding credits. Earlier Test Cloud +operations remained pending; no current-head 3DS pass is claimed. + +## Required sandbox controls + +### Team fixture + +Provide two service-owned accounts in one resettable Team workspace. The owner +must have a saved Stripe test card and an active Team subscription. Both users +must also retain personal workspaces. Setup must restore membership, roles, +active workspace, subscription, and ledger balance before the suite. The tests +must prove that owner writes succeed, member writes receive authorization +errors, and selecting another workspace cannot mutate the Team ledger. + +### Clock control + +Provide a sandbox-only API that creates or resets a disposable billing customer +under a Stripe test clock, advances it to a requested boundary, and waits until +Cloud webhook and Temporal processing reach a stable checkpoint. It must cover +renewal, credit reset, pending-checkout expiry, and subscription expiry. Return +the workspace, billing operation, event, and clock identifiers for assertions. +Advancing a Stripe clock alone is insufficient. + +### Fault control + +Provide a sandbox-only, operation-scoped fault API with automatic expiry. It +must delay or fail a named billing phase once, replay duplicate and out-of-order +events, and report delivery and retry attempts. Tests must then require one +terminal operation, one charge, one ledger grant, and eventual UI and backend +agreement. + +## Evidence and safety + +Retain operation IDs, expected and observed balances, status projections, +screenshots, and blocked-egress attachments in the Playwright report. Exclude +credentials, authorization headers, cookies, and payment secrets. Basic runs +need no database, Stripe secret key, or Temporal admin access. Team, clock, and +fault cases remain blocked until the sandbox exposes the scoped controls above. From 4ff810d9a8390ae442f9189360590012ecdd497c Mon Sep 17 00:00:00 2001 From: huang47 <157390+huang47@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:52:34 -0700 Subject: [PATCH 2/2] test: authenticate card setup and invoice payments through real 3DS --- .../utils/liveCloudBillingPolicy.test.ts | 10 +- .../fixtures/utils/liveCloudBillingPolicy.ts | 23 +++- .../liveCloud/helpers/threeDSScenarios.ts | 129 +++++++++++------- docs/testing/cloud-billing-e2e.md | 9 +- 4 files changed, 117 insertions(+), 54 deletions(-) diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts index 0c38e05c1ad..e60fbc34100 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts @@ -385,8 +385,14 @@ describe('Disposable payment permissions', () => { }) describe('Stripe test challenge permissions', () => { - it('requires sandbox payment permission for challenge responses', () => { - const url = new URL('https://testmode-acs.stripe.com/authorize') + it.for([ + 'https://testmode-acs.stripe.com/authorize', + 'https://api.stripe.com/v1/3ds2/authenticate', + 'https://api.stripe.com/v1/3ds2/challenge_complete', + 'https://api.stripe.com/v1/consumers/sessions/lookup', + 'https://hooks.stripe.com/3d_secure_2/notify/acct_example/tds2_example' + ])('requires sandbox payment permission for %s', (endpoint) => { + const url = new URL(endpoint) expect(isLiveCloudMutationAllowed(url, 'POST', config)).toBe(false) expect( isLiveCloudMutationAllowed(url, 'POST', { diff --git a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts index 9dfbf26d25c..593e1494c9b 100644 --- a/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts +++ b/browser_tests/fixtures/utils/liveCloudBillingPolicy.ts @@ -28,7 +28,9 @@ export const LIVE_CHECKOUT_ORIGINS = [ 'https://q.stripe.com', 'https://b.stripecdn.com', 'https://newassets.hcaptcha.com', - 'https://testmode-acs.stripe.com' + 'https://testmode-acs.stripe.com', + 'https://invoice.stripe.com', + 'https://hooks.stripe.com' ] const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS'] @@ -119,7 +121,7 @@ function isOnboardingSurveyUpdate(data: unknown): boolean { } function isPaymentPostAllowed(url: URL, backend: string): boolean { - if (url.origin === 'https://testmode-acs.stripe.com') return true + if (isThreeDSPostAllowed(url)) return true if ( url.origin === 'https://checkout.comfy.org' && url.pathname === '/ajax/metrics_batch' @@ -153,3 +155,20 @@ function isCheckoutPostAllowed(url: URL, backend: string): boolean { (url.origin === 'https://m.stripe.com' && url.pathname === '/6') ) } + +function isThreeDSPostAllowed(url: URL): boolean { + if (url.origin === 'https://testmode-acs.stripe.com') return true + if (url.origin === 'https://api.stripe.com') { + return [ + '/v1/3ds2/authenticate', + '/v1/3ds2/challenge_complete', + '/v1/consumers/sessions/lookup' + ].includes(url.pathname) + } + return ( + url.origin === 'https://hooks.stripe.com' && + /^\/3d_secure_2\/notify\/acct_[A-Za-z0-9]+\/tds2_[A-Za-z0-9]+$/.test( + url.pathname + ) + ) +} diff --git a/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts b/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts index 74a7c2e9d49..3a185766ce4 100644 --- a/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts +++ b/browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts @@ -4,13 +4,41 @@ import { zBillingOpStatusResponse } from '@comfyorg/ingest-types/zod' import { expect } from '@playwright/test' -import type { TestInfo } from '@playwright/test' +import type { Page, TestInfo } from '@playwright/test' + import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession' import type { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudCheckout' -export async function completeAuthenticatedCheckout( + +async function answerChallenge( + page: Page, + answer: 'Complete' | 'Fail', + testInfo: TestInfo, + screenshotName: string +) { + const frames = () => + page + .frames() + .find((frame) => + frame.url().startsWith('https://testmode-acs.stripe.com') + ) + await expect.poll(() => Boolean(frames())).toBe(true) + const frame = frames() + if (!frame) throw new Error('3D Secure challenge did not load') + await frame.waitForLoadState('load') + const action = frame.getByRole('button', { name: answer, exact: true }) + await expect(action).toBeVisible() + await testInfo.attach(screenshotName, { + body: await page.screenshot(), + contentType: 'image/png' + }) + await action.press('Enter') +} + +async function authenticateInvoice( driver: LiveCloudCheckout, session: LiveCloudBillingSession, - testInfo: TestInfo + testInfo: TestInfo, + answer: 'Complete' | 'Fail' ) { const balanceBefore = await session.read( '/api/billing/balance', @@ -19,68 +47,77 @@ export async function completeAuthenticatedCheckout( const preview = await driver.open(true) const { checkout, subscription } = await driver.startCheckout() await driver.submitCard(checkout, '4000000000003220') - const challengeOrigin = 'https://testmode-acs.stripe.com' + await answerChallenge(checkout, 'Complete', testInfo, '3ds-card-setup.png') await expect - .poll(() => - checkout.frames().some((frame) => frame.url().startsWith(challengeOrigin)) + .poll( + async () => + ( + await session.read( + `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + zBillingOpStatusResponse + ) + ).authentication_state, + { timeout: 60_000 } ) - .toBe(true) - const challenge = checkout - .frames() - .find((frame) => frame.url().startsWith(challengeOrigin)) - if (!challenge) throw new Error('3D Secure challenge did not load') - const completeAuthentication = challenge.getByRole('button', { - name: /complete/i + .toBe('requires_action') + const verification = driver.page.getByRole('button', { + name: 'Complete verification', + exact: true }) - await expect(completeAuthentication).toBeVisible() - await testInfo.attach('3ds-challenge.png', { - body: await checkout.screenshot(), - contentType: 'image/png' - }) - await completeAuthentication.click() + await expect(verification).toBeVisible({ timeout: 60_000 }) + const [invoice] = await Promise.all([ + driver.page.context().waitForEvent('page'), + verification.click() + ]) + await invoice + .getByRole('button', { name: 'Confirm payment', exact: true }) + .click() + await answerChallenge( + invoice, + answer, + testInfo, + `3ds-invoice-${answer.toLowerCase()}.png` + ) + return { balanceBefore, preview, operationId: subscription.billing_op_id } +} + +export async function completeAuthenticatedCheckout( + driver: LiveCloudCheckout, + session: LiveCloudBillingSession, + testInfo: TestInfo +) { + const { balanceBefore, preview, operationId } = await authenticateInvoice( + driver, + session, + testInfo, + 'Complete' + ) return await driver.verifyCheckoutCompletion( session, testInfo, balanceBefore.amount_micros, preview, - subscription.billing_op_id, + operationId, '3ds-completed.png' ) } + export async function failAuthenticatedCheckout( driver: LiveCloudCheckout, session: LiveCloudBillingSession, testInfo: TestInfo ) { - const balanceBefore = await session.read( - '/api/billing/balance', - zBillingBalanceResponse + const { balanceBefore, operationId } = await authenticateInvoice( + driver, + session, + testInfo, + 'Fail' ) - await driver.open(true) - const { checkout, subscription } = await driver.startCheckout() - await driver.submitCard(checkout, '4000000000003220') - const challengeOrigin = 'https://testmode-acs.stripe.com' - await expect - .poll(() => - checkout.frames().some((frame) => frame.url().startsWith(challengeOrigin)) - ) - .toBe(true) - const challenge = checkout - .frames() - .find((frame) => frame.url().startsWith(challengeOrigin)) - if (!challenge) throw new Error('3D Secure challenge did not load') - const failAuthentication = challenge.getByRole('button', { name: /fail/i }) - await expect(failAuthentication).toBeVisible() - await testInfo.attach('3ds-failure-challenge.png', { - body: await checkout.screenshot(), - contentType: 'image/png' - }) - await failAuthentication.click() await expect .poll( async () => { const operation = await session.read( - `/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`, + `/api/billing/ops/${encodeURIComponent(operationId)}`, zBillingOpStatusResponse ) return { @@ -92,7 +129,7 @@ export async function failAuthenticatedCheckout( { timeout: 60_000 } ) .toMatchObject({ - status: 'failed', + status: 'pending', authenticationState: 'failed_retryable' }) const status = await session.read( @@ -107,7 +144,7 @@ export async function failAuthenticatedCheckout( expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros) await testInfo.attach('3ds-failure.json', { body: JSON.stringify({ - operationId: subscription.billing_op_id, + operationId, balanceBeforeCents: balanceBefore.amount_micros, balanceAfterCents: balanceAfter.amount_micros, authenticationState: 'failed_retryable' diff --git a/docs/testing/cloud-billing-e2e.md b/docs/testing/cloud-billing-e2e.md index 92fdea7d193..efc1a978193 100644 --- a/docs/testing/cloud-billing-e2e.md +++ b/docs/testing/cloud-billing-e2e.md @@ -86,10 +86,11 @@ portal, because Staging disables the embedded-checkout saved-card API. ## 3DS outcomes PR #17545 depends only on #17481. Run `--project=cloud-live-disposable` for -successful and failed authentication. Challenge interaction, challenge permissions, -and terminal-state assertions live in this PR. Success activates Creator and -grants once; failure becomes retryable without adding credits. Earlier Test Cloud -operations remained pending; no current-head 3DS pass is claimed. +successful and failed invoice authentication. Both scenarios complete the card-setup +challenge, then open the hosted invoice through Complete verification. Success +activates Creator and grants once. A rejected invoice challenge leaves the operation +pending with failed_retryable authentication, keeps the account on Free, and adds +no credits. Challenge interactions and permissions stay in this PR. ## Required sandbox controls