Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 160 additions & 2 deletions browser_tests/fixtures/agentTurnLockFixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,18 @@ import { expect, mergeTests } from '@playwright/test'
import type { Locator, Page, WebSocketRoute } from '@playwright/test'

import type {
AgentAnswerAccepted,
AgentCancelAccepted,
AgentError,
AgentMessage,
AgentPendingAsk,
AgentTurnAccepted
} from '@comfyorg/ingest-types'
import { zAgentPostMessageRequest } from '@comfyorg/ingest-types/zod'
import {
zAgentPendingAsk,
zAgentPostMessageRequest
} from '@comfyorg/ingest-types/zod'
import { z } from 'zod'

import enMessages from '@/locales/en/main.json' with { type: 'json' }
import type { AgentWsEvent } from '@/workbench/extensions/agent/schemas/agentApiSchema'
Expand All @@ -19,6 +25,8 @@ import { TestIds } from '@e2e/fixtures/selectors'
import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute'
import { webSocketFixture } from '@e2e/fixtures/ws'

const zAnswerRequest = z.object({ selected: z.array(z.string()) })

const THREAD_ID = 'b9d0a2a1-0f2c-4f1a-9a5e-6b0f4f2c1d77'
const TURN_ID = '2dd4f367-3399-4cb4-8127-547f531c289a'
const WORKFLOW_ID = 'a81718a4-02ae-41e6-ae85-000000000001'
Expand Down Expand Up @@ -67,6 +75,46 @@ export const TURN_DONE_EVENT: AgentWsEvent = {
data: { message_id: TURN_ID, thread_id: THREAD_ID }
}

const RUN_APPROVAL_ASK_ID = `${TURN_ID}:call-run-workflow`

/**
* The option ids this ask offers. Declared separately because `AgentWsEvent`
* types `data` loosely, so reading them back off the event would be an
* `unknown` the route would have to cast.
*/
const RUN_APPROVAL_OPTION_IDS = ['run', 'cancel'] as const

/**
* The frame the server sends when a turn parks waiting for the user to approve
* a run. Shaped after `pendingRunApproval`'s reader and cloud's `asks` writer:
* the turn does not proceed until an answer posts back, so a client that drops
* this frame strands the user with no way to answer and no error.
*/
export const RUN_APPROVAL_EVENT: AgentWsEvent = {
type: 'agent_ask',
data: {
message_id: TURN_ID,
thread_id: THREAD_ID,
ask_id: RUN_APPROVAL_ASK_ID,
kind: 'run_approval',
prompt: 'Run workflow “Unsaved Workflow”?',
context: { workflow_id: WORKFLOW_ID, workflow_name: 'Unsaved Workflow' },
options: [
{
id: RUN_APPROVAL_OPTION_IDS[0],
label: enMessages.agent.runApproval.run
},
{
id: RUN_APPROVAL_OPTION_IDS[1],
label: enMessages.agent.runApproval.cancel
}
],
min_selections: 1,
max_selections: 1,
allow_other: false
}
}

/**
* The server's half of a turn, modelled on the real single-active-turn guard:
* an assistant row goes `streaming` when a turn starts and only leaves that
Expand All @@ -83,6 +131,8 @@ class TurnLockServer {
private prompt = ''
private rejected = 0
private posts = 0
private readonly answered: string[][] = []
private pendingAsk: AgentPendingAsk | undefined

get turnIsStreaming(): boolean {
return this.streaming
Expand All @@ -101,6 +151,25 @@ class TurnLockServer {
this.posts++
}

/** Every ask answer the server accepted, in order, as the selected ids. */
get answers(): string[][] {
return this.answered
}

get askIsPending(): boolean {
return this.pendingAsk !== undefined
}

recordAnswer(selected: string[]): void {
this.answered.push(selected)
this.pendingAsk = undefined
}

recordAsk(event: AgentWsEvent): void {
if (event.type === 'agent_ask')
this.pendingAsk = zAgentPendingAsk.parse(event.data)
}

completeTurn(): void {
this.streaming = false
}
Expand All @@ -124,7 +193,8 @@ class TurnLockServer {
seq: 2,
role: 'assistant',
status: this.streaming ? 'streaming' : 'complete',
workflow_id: WORKFLOW_ID
workflow_id: WORKFLOW_ID,
pending_ask: this.pendingAsk
}
]
}
Expand Down Expand Up @@ -160,6 +230,48 @@ async function routeTurnLock(
})
})

/**
* The glob accepts any thread and any ask id, so without these checks an
* answer posted against the wrong ask -- or an option this ask never offered
* -- would still be recorded and `answeredAsks()` would report success. The
* assertion this fixture exists to support is "the answer left the client for
* *this* ask", so the route has to be the thing that enforces it.
*/
await page.route('**/api/agent/threads/*/asks/*/answer', (route) => {
const url = new URL(route.request().url())
const segments = url.pathname.split('/').map(decodeURIComponent)
const threadId = segments[segments.indexOf('threads') + 1]
const askId = segments[segments.indexOf('asks') + 1]
if (threadId !== THREAD_ID || askId !== RUN_APPROVAL_ASK_ID)
return route.fulfill({
status: 404,
contentType: 'application/json',
body: JSON.stringify({ error: 'unknown thread or ask' })
})

let body: unknown
try {
body = route.request().postDataJSON()
} catch {
return route.fulfill({ status: 400, body: 'invalid JSON' })
}
const parsed = zAnswerRequest.safeParse(body)
if (!parsed.success)
return route.fulfill({ status: 400, body: 'invalid answer' })
const selected = parsed.data.selected
const offered: readonly string[] = RUN_APPROVAL_OPTION_IDS
if (selected.length !== 1 || !offered.includes(selected[0]))
return route.fulfill({
status: 422,
contentType: 'application/json',
body: JSON.stringify({ error: 'selection not offered by this ask' })
})

server.recordAnswer(selected)
const accepted: AgentAnswerAccepted = { status: 'answered' }
return route.fulfill(jsonRoute(accepted))
})

await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => {
server.completeTurn()
const accepted: AgentCancelAccepted = { status: 'cancelling' }
Expand Down Expand Up @@ -267,6 +379,16 @@ export class AgentTurnLockHarness {
ws.send(JSON.stringify(event))
}

/** Makes an ask available through transcript hydration, independently of WS delivery. */
primePendingAsk(event: AgentWsEvent): void {
this.server.recordAsk(event)
}

/** True only when a test explicitly primed transcript-based recovery. */
pendingAskIsPrimed(): boolean {
return this.server.askIsPending
}

/**
* Replays the Web Audio work an audio preview does while a turn is live:
* open an AudioContext, `decodeAudioData` real WAV bytes, close it. This is
Expand Down Expand Up @@ -303,6 +425,42 @@ export class AgentTurnLockHarness {
})
}

/**
* Every ask the user answered through the panel, flattened to the selected
* option ids. Reads the fake server rather than the DOM, so it proves the
* answer actually left the client.
*/
answeredAsks(): string[] {
return this.server.answers.flat()
}

/** Posts an answer through the browser so the real route validator sees it. */
async postAnswer(
threadId: string,
askId: string,
selected: string[]
): Promise<number> {
return await this.page.evaluate(
async ({ threadId, askId, selected }) => {
const response = await fetch(
`/api/agent/threads/${encodeURIComponent(threadId)}/asks/${encodeURIComponent(askId)}/answer`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ selected })
}
)
return response.status
},
{ threadId, askId, selected }
)
}

/** The socket the client is currently on, with no drop. */
async liveSocket(): Promise<WebSocketRoute> {
return this.getWebSocket()
}

/** Drops the live socket and resolves with the one the client reconnects on. */
async dropSocket(): Promise<WebSocketRoute> {
// Resolve the live route first: with no socket open yet both calls would
Expand Down
115 changes: 115 additions & 0 deletions browser_tests/tests/agent/agentApprovalSurvivesSocketDrop.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { expect } from '@playwright/test'

import enMessages from '@/locales/en/main.json' with { type: 'json' }
import {
RUN_APPROVAL_EVENT,
agentTurnLockTest as test
} from '@e2e/fixtures/agentTurnLockFixture'

// The worst shape a stall can take, from a real user report: they asked for an
// img2img workflow, then for img2video, and the panel went dead. They restarted
// the agent tab to get out of it.
//
// A session-id trace said the backend never hung. The turn was alive the whole
// time, parked waiting for the user to approve a tool call — and the approval
// prompt never made it into the panel. The product was working and waiting on
// the user; the user had no way to answer.
//
// The mechanism is the socket blip. `useAgentSession.onStatus(false)` answers a
// transient close with `abortActiveTurn()` + `dropBackgroundTurns()`, so the
// transport slot is null and the background map is empty. The server, which
// never learned the socket went away, then emits `agent_ask` for the approval
// it is parked on, and `agentConversationStore.ingest` has nothing to route it
// to: `ingestBackgroundTurnEvent` returns at `!entry` and the frame is gone.
// Nothing errors, the panel looks idle, the server looks healthy.
//
// Deliberately not a refresh: on a refresh the ask comes back, because the
// persisted row carries `pending_ask` and `normalizeAgentTranscript` restores
// the card. That asymmetry is why restarting the tab "fixed" it for the user
// and why this only ever showed up through a feedback form.
test.describe.configure({ timeout: 120_000 })
test.use({ connectWebSocketToServer: false })

test.describe(
'an approval ask survives a socket drop',
{ tag: ['@cloud', '@ui'] },
() => {
const PROMPT = 'turn this image into a video'

test.beforeEach(async ({ turnLock }) => {
await turnLock.openOnBlankWorkflow()
await turnLock.startTurn(PROMPT)
})

test('the approval card reaches the panel after a reconnect', async ({
turnLock
}) => {
const reconnected = await turnLock.dropSocket()

// Preconditions stay above the marker so a wiped thread or a lost prompt
// reads as a real failure rather than the expected one.
await expect(turnLock.userBubbles).toHaveText([PROMPT])

// The server is parked on this ask and will not proceed without an
// answer. `ws.send()` throws on a dead route, so pushing stays above
// test.fail() too.
turnLock.push(reconnected, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)
Comment thread
christian-byrne marked this conversation as resolved.
Resolved

test.fail()
await expect(
turnLock.panel.getByText(enMessages.agent.runApproval.question)
).toBeVisible()
await expect(
turnLock.panel.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
).toBeVisible()
})

test('the user can answer the ask that arrived after the reconnect', async ({
turnLock
}) => {
const reconnected = await turnLock.dropSocket()
await expect(turnLock.userBubbles).toHaveText([PROMPT])
turnLock.push(reconnected, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)

test.fail()
// Recovery must restore both the card and its active turn identity;
// `answerAsk` deliberately refuses to POST without `activeTurnId`.
await turnLock.panel
.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
.click({ timeout: 10_000 })
await expect.poll(() => turnLock.answeredAsks()).toEqual(['run'])
})

// Keeps the locators above honest. If the approval card never renders under
// this fixture at all — a changed testid, a gate, reworded copy — this case
// reddens and the two expected failures above stop being evidence of the
// defect they name.
test('the same ask renders when the socket never drops', async ({
turnLock
}) => {
const live = await turnLock.liveSocket()

turnLock.push(live, RUN_APPROVAL_EVENT)
expect(turnLock.pendingAskIsPrimed()).toBe(false)

await expect(
turnLock.panel.getByText(enMessages.agent.runApproval.question)
).toBeVisible()
await turnLock.panel
.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
.click()
await expect.poll(() => turnLock.answeredAsks()).toEqual(['run'])
})
}
)
3 changes: 3 additions & 0 deletions src/composables/billing/useBillingContext.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import type {
Plan
} from '@/platform/workspace/api/workspaceApi'
import {
authenticatedRemoteConfigState,
remoteConfig,
remoteConfigState
} from '@/platform/remoteConfig/remoteConfig'
Expand Down Expand Up @@ -182,6 +183,7 @@ describe('useBillingContext', () => {
beforeEach(() => {
remoteConfig.value = {}
remoteConfigState.value = 'unloaded'
authenticatedRemoteConfigState.value = 'unloaded'
mockIsPersonal.value = true
mockBillingRail.value = undefined
mockSetWorkspaceBillingRail.mockImplementation(
Expand Down Expand Up @@ -358,6 +360,7 @@ describe('useBillingContext', () => {
it('routes migrated legacy Stripe topups through workspace billing', async () => {
remoteConfig.value = { legacy_billing_migration_enabled: true }
remoteConfigState.value = 'authenticated'
authenticatedRemoteConfigState.value = 'authenticated'
mockBillingRail.value = 'legacy_stripe'

const context = useBillingContext()
Expand Down
Loading
Loading