Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions browser_tests/fixtures/utils/liveCloudBillingPolicy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -383,3 +383,29 @@ describe('Disposable payment permissions', () => {
).toBe(false)
})
})

describe('Stripe test challenge permissions', () => {
it.for([
'https://testmode-acs.stripe.com/authorize',
'https://api.stripe.com/v1/3ds2/authenticate',
'https://api.stripe.com/v1/3ds2/challenge_complete',
'https://api.stripe.com/v1/consumers/sessions/lookup',
'https://hooks.stripe.com/3d_secure_2/notify/acct_example/tds2_example'
])('requires sandbox payment permission for %s', (endpoint) => {
const url = new URL(endpoint)
expect(isLiveCloudMutationAllowed(url, 'POST', config)).toBe(false)
expect(
isLiveCloudMutationAllowed(url, 'POST', {
...config,
allowPayments: true
})
).toBe(true)
expect(
isLiveCloudMutationAllowed(url, 'POST', {
...config,
allowPayments: true,
PLAYWRIGHT_SETUP_API_URL: 'https://cloud.comfy.org'
})
).toBe(false)
})
})
23 changes: 22 additions & 1 deletion browser_tests/fixtures/utils/liveCloudBillingPolicy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ export const LIVE_CHECKOUT_ORIGINS = [
'https://r.stripe.com',
'https://q.stripe.com',
'https://b.stripecdn.com',
'https://newassets.hcaptcha.com'
'https://newassets.hcaptcha.com',
'https://testmode-acs.stripe.com',
'https://invoice.stripe.com',
'https://hooks.stripe.com'
]

const SAFE_METHODS = ['GET', 'HEAD', 'OPTIONS']
Expand Down Expand Up @@ -118,6 +121,7 @@ function isOnboardingSurveyUpdate(data: unknown): boolean {
}

function isPaymentPostAllowed(url: URL, backend: string): boolean {
if (isThreeDSPostAllowed(url)) return true
if (
url.origin === 'https://checkout.comfy.org' &&
url.pathname === '/ajax/metrics_batch'
Expand Down Expand Up @@ -151,3 +155,20 @@ function isCheckoutPostAllowed(url: URL, backend: string): boolean {
(url.origin === 'https://m.stripe.com' && url.pathname === '/6')
)
}

function isThreeDSPostAllowed(url: URL): boolean {
if (url.origin === 'https://testmode-acs.stripe.com') return true
if (url.origin === 'https://api.stripe.com') {
return [
'/v1/3ds2/authenticate',
'/v1/3ds2/challenge_complete',
'/v1/consumers/sessions/lookup'
].includes(url.pathname)
}
return (
url.origin === 'https://hooks.stripe.com' &&
/^\/3d_secure_2\/notify\/acct_[A-Za-z0-9]+\/tds2_[A-Za-z0-9]+$/.test(
url.pathname
)
)
}
23 changes: 23 additions & 0 deletions browser_tests/tests/liveCloud/disposable/threeDSComplete.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { completeAuthenticatedCheckout } from '@e2e/tests/liveCloud/helpers/threeDSScenarios'
import {
expect,
liveCloudDisposableBillingFixture as test
} from '@e2e/fixtures/liveCloudDisposableBillingFixture'

test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => {
test.setTimeout(180_000)

test('activates Creator after successful authentication', async ({
disposableCheckout
}, testInfo) => {
const completion = await completeAuthenticatedCheckout(
disposableCheckout.checkout,
disposableCheckout.billingSession,
testInfo
)
expect(completion).toMatchObject({
planSlug: 'creator-monthly',
paymentMethodCount: 1
})
})
})
22 changes: 22 additions & 0 deletions browser_tests/tests/liveCloud/disposable/threeDSFailure.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { failAuthenticatedCheckout } from '@e2e/tests/liveCloud/helpers/threeDSScenarios'
import {
expect,
liveCloudDisposableBillingFixture as test
} from '@e2e/fixtures/liveCloudDisposableBillingFixture'

test.describe('Real Cloud 3D Secure', { tag: ['@cloud-live'] }, () => {
test.setTimeout(180_000)

test('keeps the account on Free after failed authentication', async ({
disposableCheckout
}, testInfo) => {
const failure = await failAuthenticatedCheckout(
disposableCheckout.checkout,
disposableCheckout.billingSession,
testInfo
)
expect(failure).toMatchObject({
authenticationState: 'failed_retryable'
})
})
})
155 changes: 155 additions & 0 deletions browser_tests/tests/liveCloud/helpers/threeDSScenarios.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
import {
zBillingBalanceResponse,
zBillingStatusResponse,
zBillingOpStatusResponse
} from '@comfyorg/ingest-types/zod'
import { expect } from '@playwright/test'
import type { Page, TestInfo } from '@playwright/test'

import type { LiveCloudBillingSession } from '@e2e/fixtures/helpers/LiveCloudBillingSession'
import type { LiveCloudCheckout } from '@e2e/fixtures/helpers/LiveCloudCheckout'

async function answerChallenge(
page: Page,
answer: 'Complete' | 'Fail',
testInfo: TestInfo,
screenshotName: string
) {
const frames = () =>
page
.frames()
.find((frame) =>
frame.url().startsWith('https://testmode-acs.stripe.com')
)
await expect.poll(() => Boolean(frames())).toBe(true)
const frame = frames()
if (!frame) throw new Error('3D Secure challenge did not load')
await frame.waitForLoadState('load')
const action = frame.getByRole('button', { name: answer, exact: true })
await expect(action).toBeVisible()
await testInfo.attach(screenshotName, {
body: await page.screenshot(),
contentType: 'image/png'
})
await action.press('Enter')
}

async function authenticateInvoice(
driver: LiveCloudCheckout,
session: LiveCloudBillingSession,
testInfo: TestInfo,
answer: 'Complete' | 'Fail'
) {
const balanceBefore = await session.read(
'/api/billing/balance',
zBillingBalanceResponse
)
const preview = await driver.open(true)
const { checkout, subscription } = await driver.startCheckout()
await driver.submitCard(checkout, '4000000000003220')
await answerChallenge(checkout, 'Complete', testInfo, '3ds-card-setup.png')
await expect
.poll(
async () =>
(
await session.read(
`/api/billing/ops/${encodeURIComponent(subscription.billing_op_id)}`,
zBillingOpStatusResponse
)
).authentication_state,
{ timeout: 60_000 }
)
.toBe('requires_action')
const verification = driver.page.getByRole('button', {
name: 'Complete verification',
exact: true
})
await expect(verification).toBeVisible({ timeout: 60_000 })
const [invoice] = await Promise.all([
driver.page.context().waitForEvent('page'),
verification.click()
])
await invoice
.getByRole('button', { name: 'Confirm payment', exact: true })
.click()
await answerChallenge(
invoice,
answer,
testInfo,
`3ds-invoice-${answer.toLowerCase()}.png`
)
return { balanceBefore, preview, operationId: subscription.billing_op_id }
}

export async function completeAuthenticatedCheckout(
driver: LiveCloudCheckout,
session: LiveCloudBillingSession,
testInfo: TestInfo
) {
const { balanceBefore, preview, operationId } = await authenticateInvoice(
driver,
session,
testInfo,
'Complete'
)
return await driver.verifyCheckoutCompletion(
session,
testInfo,
balanceBefore.amount_micros,
preview,
operationId,
'3ds-completed.png'
)
}

export async function failAuthenticatedCheckout(
driver: LiveCloudCheckout,
session: LiveCloudBillingSession,
testInfo: TestInfo
) {
const { balanceBefore, operationId } = await authenticateInvoice(
driver,
session,
testInfo,
'Fail'
)
await expect
.poll(
async () => {
const operation = await session.read(
`/api/billing/ops/${encodeURIComponent(operationId)}`,
zBillingOpStatusResponse
)
return {
status: operation.status,
authenticationState: operation.authentication_state,
declineReason: operation.decline_reason
}
},
{ timeout: 60_000 }
)
.toMatchObject({
status: 'pending',
authenticationState: 'failed_retryable'
})
const status = await session.read(
'/api/billing/status',
zBillingStatusResponse
)
expect(status.subscription_tier).toBe('FREE')
const balanceAfter = await session.read(
'/api/billing/balance',
zBillingBalanceResponse
)
expect(balanceAfter.amount_micros).toBe(balanceBefore.amount_micros)
await testInfo.attach('3ds-failure.json', {
body: JSON.stringify({
operationId,
balanceBeforeCents: balanceBefore.amount_micros,
balanceAfterCents: balanceAfter.amount_micros,
authenticationState: 'failed_retryable'
}),
contentType: 'application/json'
})
return { authenticationState: 'failed_retryable' }
}
45 changes: 45 additions & 0 deletions docs/testing/cloud-billing-e2e.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,48 @@ Traces start after permanent-account sign-in and contain authenticated network
traffic. Keep reports private. Disposable accounts require no reusable credentials.
Completed-payment assertions verify saved cards through the real hosted billing
portal, because Staging disables the embedded-checkout saved-card API.

## 3DS outcomes

PR #17545 depends only on #17481. Run `--project=cloud-live-disposable` for
successful and failed invoice authentication. Both scenarios complete the card-setup
challenge, then open the hosted invoice through Complete verification. Success
activates Creator and grants once. A rejected invoice challenge leaves the operation
pending with failed_retryable authentication, keeps the account on Free, and adds
no credits. Challenge interactions and permissions stay in this PR.

## Required sandbox controls

### Team fixture

Provide two service-owned accounts in one resettable Team workspace. The owner
must have a saved Stripe test card and an active Team subscription. Both users
must also retain personal workspaces. Setup must restore membership, roles,
active workspace, subscription, and ledger balance before the suite. The tests
must prove that owner writes succeed, member writes receive authorization
errors, and selecting another workspace cannot mutate the Team ledger.

### Clock control

Provide a sandbox-only API that creates or resets a disposable billing customer
under a Stripe test clock, advances it to a requested boundary, and waits until
Cloud webhook and Temporal processing reach a stable checkpoint. It must cover
renewal, credit reset, pending-checkout expiry, and subscription expiry. Return
the workspace, billing operation, event, and clock identifiers for assertions.
Advancing a Stripe clock alone is insufficient.

### Fault control

Provide a sandbox-only, operation-scoped fault API with automatic expiry. It
must delay or fail a named billing phase once, replay duplicate and out-of-order
events, and report delivery and retry attempts. Tests must then require one
terminal operation, one charge, one ledger grant, and eventual UI and backend
agreement.

## Evidence and safety

Retain operation IDs, expected and observed balances, status projections,
screenshots, and blocked-egress attachments in the Playwright report. Exclude
credentials, authorization headers, cookies, and payment secrets. Basic runs
need no database, Stripe secret key, or Temporal admin access. Team, clock, and
fault cases remain blocked until the sandbox exposes the scoped controls above.
Loading