Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 154 additions & 0 deletions browser_tests/tests/agent/agentExecutionStoryFixture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import { expect } from '@playwright/test'
import type { Page } from '@playwright/test'

import type { AgentCancelAccepted } from '@comfyorg/ingest-types'

import type { JobDetail } from '@/platform/remote/comfyui/jobs/jobTypes'

import { agentTest } from '@e2e/fixtures/agentPanelFixture'
import type { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket'
import { AgentExecutionHelper } from '@e2e/fixtures/helpers/AgentExecutionHelper'
import { AssetsHelper } from '@e2e/fixtures/helpers/AssetsHelper'
import { MultiAutogrowRealignHarness } from '@e2e/fixtures/helpers/MultiAutogrowRealignHarness'
import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute'
import {
MESSAGE_ID,
THREAD_ID
} from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture'
import enMessages from '@/locales/en/main.json' with { type: 'json' }

const JOB_CANCEL_ROUTE_PATTERN =
/\/api\/(?:interrupt|jobs\/(?:[^/]+\/)?cancel)$/

/**
* `AgentExecutionHelper` plus the two surfaces only these user-story specs
* need, kept here rather than pushed into the shared helper: a recorder for
* job interrupt/cancel requests (so "the agent's turn stopped and the job did
* not" is observable as the *absence* of a cancel), and a per-job
* `GET /api/jobs/{id}` detail route.
*
* Registered after `super.install()`, which is what makes these routes win —
* Playwright runs the most-recently-registered matching handler first — and
* the same reason the base helper installs after the harness.
*/
class AgentExecutionStoryHelper extends AgentExecutionHelper {
private readonly jobCancels: string[] = []
private readonly storyPage: Page
private readonly storyAssets: AssetsHelper

constructor(page: Page, hostSocket: AgentFollowerHostSocket) {
super(page, hostSocket)
this.storyPage = page
this.storyAssets = new AssetsHelper(page)
}

override async install(): Promise<void> {
await super.install()
await this.storyPage.route(JOB_CANCEL_ROUTE_PATTERN, (route) => {
this.jobCancels.push(new URL(route.request().url()).pathname)
return route.fulfill({ status: 200, body: '' })
})
}

/** Paths of job interrupt or cancel requests, in request order. */
jobCancelRequests(): readonly string[] {
return this.jobCancels
}

/**
* Serves `GET /api/jobs/{jobId}` — the detail record, which is separate from
* the list row `enqueueServerRun` and `upsertJob` maintain. Its `workflow`
* field is the body that was POSTed to `/api/prompt`, stored verbatim
* (cloud `services/ingest/.../prompt.go`, `SetWorkflowJSON`), and it is the
* only thing the "open as workflow" action has to rebuild a graph from.
* Without this route that fetch 404s, so a spec about what the action does
* with the detail must supply one rather than read a failed fetch as a
* product verdict.
*/
async mockJobDetail(jobId: string, detail: JobDetail): Promise<void> {
await this.storyAssets.mockJobDetail(jobId, detail)
}
}

interface AgentExecutionStoryRig {
harness: MultiAutogrowRealignHarness
execution: AgentExecutionStoryHelper
}

/**
* The execution rig of `agentExecutionFixture`, but with no turn sent yet:
* the harness's message mock serves exactly one `message_id`, so a user-story
* spec whose subject IS a turn — "run it", or stopping one mid-flight — must
* drive that turn itself rather than spend the only one on a binding
* handshake. The panel is open and targeting the workflow; sending the
* story's message is what acks the turn and binds the CRDT follower.
*/
export const agentExecutionStoryTest = agentTest.extend<{
rig: AgentExecutionStoryRig
}>({
rig: async ({ page }, use) => {
const harness = new MultiAutogrowRealignHarness(page)
await harness.setUp({ settings: { 'Comfy.Queue.QPOV2': false } })
const execution = new AgentExecutionStoryHelper(page, harness.hostSocket)
await execution.install()
const accepted: AgentCancelAccepted = { status: 'cancelling' }
await page.route('**/api/agent/threads/*/messages/*/cancel', (route) =>
route.fulfill({ ...jsonRoute(accepted), status: 202 })
)
await harness.targetActiveWorkflow()
await use({ harness, execution })
}
})

/**
* Sends the story's user message and waits until the turn is live: the
* assistant placeholder is rendered (`agent_message_done` is only routed while
* the store holds the ack's `message_id`), the turn's ack has bound the CRDT
* follower, and the workflow the agent would run is visible on the canvas.
*/
export async function startStoryTurn(
harness: MultiAutogrowRealignHarness,
text: string
): Promise<void> {
// `harness.agentPanel.composer`, not a hand-rolled locator: the composer's
// accessible name is the `agent.placeholder` /
// `agent.placeholderWithoutSkills` pair, and `AGENT_COMPOSER_LABEL` tracks
// both. A literal here silently matches nothing and the fill times out.
await harness.agentPanel.composer.fill(text)
await harness.panel
.getByRole('button', { name: enMessages.agent.send })
.click()
await expect(
harness.panel.getByText(enMessages.agent.thinking).first()
).toBeVisible()
await harness.hostSocket.waitForSubscribe()
await expect(harness.targetNode).toBeVisible()
}

/** Ends the live turn from the host and waits for the panel to go idle. */
export async function settleStoryTurn(
harness: MultiAutogrowRealignHarness
): Promise<void> {
harness.hostSocket.send({
type: 'agent_message_done',
data: { message_id: MESSAGE_ID, thread_id: THREAD_ID }
})
await expect(
harness.panel.getByRole('button', { name: enMessages.agent.stop })
).toHaveCount(0)
}

/** The agent's `run` tool call frame, `running` or settled. */
export function runToolCallFrame(status: 'running' | 'success') {
return {
type: 'agent_tool_call' as const,
data: {
tool_call_id: 'call-run',
tool_name: 'run',
status,
...(status === 'success' && { duration_ms: 1200 }),
message_id: MESSAGE_ID,
thread_id: THREAD_ID
}
}
}
52 changes: 52 additions & 0 deletions browser_tests/tests/agent/agentFailedRunIsSurfaced.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { expect } from '@playwright/test'

import {
agentExecutionStoryTest as test,
runToolCallFrame,
settleStoryTurn,
startStoryTurn
} from '@e2e/tests/agent/agentExecutionStoryFixture'
import { QueuePanel } from '@e2e/fixtures/components/QueuePanel'
import { TARGET_ID } from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture'
import { TestIds } from '@e2e/fixtures/selectors'
import enMessages from '@/locales/en/main.json' with { type: 'json' }

test.describe(
'Agent failed run',
{ tag: ['@cloud', '@agent', '@vue-nodes'] },
() => {
test('a run that fails after the turn ends names its error and leaves the panel idle', async ({
rig: { harness, execution },
page
}) => {
const jobId = 'agent-surfaced-failed-run-1'
const failureMessage = 'deterministic fixture failure'
await startStoryTurn(harness, 'run it')
harness.hostSocket.send(runToolCallFrame('running'))
await execution.enqueueServerRun(jobId)
await execution.startJob(jobId)
harness.hostSocket.send(runToolCallFrame('success'))
await settleStoryTurn(harness)

await execution.failJob(jobId, {
nodeId: TARGET_ID,
message: failureMessage
})

const queuePanel = new QueuePanel(page)
await queuePanel.open()
await queuePanel.jobRow(jobId).hover()
const details = page.getByTestId(TestIds.queue.jobDetailsPopover)
await expect(
details.getByText(enMessages.queue.jobDetails.errorMessage)
).toBeVisible()
await expect(details.getByText(failureMessage)).toBeVisible()
await expect(
harness.panel.getByRole('button', { name: enMessages.agent.stop })
).toHaveCount(0)
await expect(
harness.panel.getByText(enMessages.agent.thinking)
).toHaveCount(0)
})
}
)
170 changes: 170 additions & 0 deletions browser_tests/tests/agent/agentGoldenPathRunIt.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
import { expect } from '@playwright/test'

import type { AgentAnswerAccepted } from '@comfyorg/ingest-types'
import { zAgentAnswerRequest } from '@comfyorg/ingest-types/zod'

import {
agentExecutionStoryTest as test,
runToolCallFrame,
settleStoryTurn,
startStoryTurn
} from '@e2e/tests/agent/agentExecutionStoryFixture'
import { QueuePanel } from '@e2e/fixtures/components/QueuePanel'
import { nextFrame } from '@e2e/fixtures/utils/timing'
import {
MESSAGE_ID,
SOURCE_NODE_ID,
THREAD_ID,
WORKFLOW_ID
} from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture'
import enMessages from '@/locales/en/main.json' with { type: 'json' }

const JOB_ID = 'agent-golden-path-run-1'
const OUTPUT_FILENAME = 'agent-golden-path-out.png'
const ASK_ID = `${MESSAGE_ID}:call-run`
const ANSWER_PATH = `/api/agent/threads/${THREAD_ID}/asks/${encodeURIComponent(ASK_ID)}/answer`

const ids = { message_id: MESSAGE_ID, thread_id: THREAD_ID }

/**
* GOLDEN PATH (user-story matrix rank 26): "run it" submits the workflow and
* shows me the output.
*
* The journey, end to end as the user sees it: ask the agent to run the
* workflow, approve the run when the agent asks, watch the run land on the
* queue, and see the finished output on the canvas node and in job history.
* The run is submitted by the agent's `run` tool on the server — approval is
* answered over REST and the browser never POSTs `/api/prompt` — which is why
* the job enters through `enqueueServerRun` rather than a client submission.
*
* Harness verification for every lever used here is
* `agentExecutionLifecycle.spec.ts`; this spec asserts the product journey on
* top of that surface, not the fixture itself.
*/
test.describe(
'Agent golden path',
{ tag: ['@cloud', '@agent', '@vue-nodes'] },
() => {
test('"run it" submits the workflow and shows me the output', async ({
rig: { harness, execution },
page
}) => {
// Run approval is answered over REST; the parked turn then resumes on
// the socket (`agent_ask_resolved` is the canonical resolution).
const answers: { method: string; path: string; selected: string[] }[] = []
await page.route('**/api/agent/threads/*/asks/*/answer', (route) => {
const body = zAgentAnswerRequest.parse(route.request().postDataJSON())
answers.push({
method: route.request().method(),
path: new URL(route.request().url()).pathname,
selected: body.selected
})
const accepted: AgentAnswerAccepted = {
status: 'answered',
selected: body.selected
}
return route.fulfill({
status: 202,
contentType: 'application/json',
body: JSON.stringify(accepted)
})
})

// The user asks the agent to run the workflow.
await startStoryTurn(harness, 'run it')

// The agent asks for approval to run, and the user approves.
harness.hostSocket.send({
type: 'agent_ask',
data: {
...ids,
ask_id: ASK_ID,
kind: 'run_approval',
context: {
workflow_id: WORKFLOW_ID,
workflow_name: 'Unsaved Workflow'
},
prompt: 'Run workflow “Unsaved Workflow”?',
options: [
{ id: 'run', label: 'Run' },
{ id: 'cancel', label: 'Cancel' }
],
min_selections: 1,
max_selections: 1,
allow_other: false
}
})
await expect(
harness.panel.getByText(enMessages.agent.runApproval.leadBound)
).toBeVisible()
await harness.panel
.getByRole('button', {
name: enMessages.agent.runApproval.run,
exact: true
})
.click()
await expect.poll(() => answers.length).toBe(1)
expect(answers[0]).toEqual({
method: 'POST',
path: ANSWER_PATH,
selected: ['run']
})

// The turn resumes: the run tool submits the workflow server-side and
// the job starts, then the agent's turn completes.
harness.hostSocket.send({
type: 'agent_ask_resolved',
data: { ...ids, ask_id: ASK_ID, status: 'answered', selected: ['run'] }
})
harness.hostSocket.send(runToolCallFrame('running'))
await execution.enqueueServerRun(JOB_ID)
await execution.startJob(JOB_ID)
harness.hostSocket.send(runToolCallFrame('success'))
await settleStoryTurn(harness)

// "Submits the workflow": the run is a real job the user can see on the
// queue surface. It was submitted by the agent on the server — the
// browser itself never POSTed /api/prompt.
const queuePanel = new QueuePanel(page)
await queuePanel.open()
await expect(queuePanel.jobRow(JOB_ID)).toBeVisible()
expect(execution.submittedPrompts()).toHaveLength(0)

// "Shows me the output": the finished run's image renders on the node
// (completed onto the source node — media type is inferred from a
// node's inputs, and the target's video inputs would render an image
// output as a <video>), and the job survives as a history row.
await execution.completeJob(JOB_ID, {
nodeId: String(SOURCE_NODE_ID),
filename: OUTPUT_FILENAME
})
await nextFrame(page)
const nodeImage = harness.vueNodes
.getNodeLocator(String(SOURCE_NODE_ID))
.getByRole('img', { name: 'View image 1 of 1' })
await expect(nodeImage).toHaveAttribute(
'src',
new RegExp(OUTPUT_FILENAME)
)
await expect(nodeImage).toBeVisible()
await queuePanel.overlay
.getByRole('button', { name: 'Completed', exact: true })
.click()
const historyRow = queuePanel.jobRow(JOB_ID)
await expect(historyRow).toBeVisible()
const historyImage = historyRow.getByRole('img')
await expect(historyImage).toHaveAttribute(
'src',
/[?&]filename=agent-golden-path-out\.png/
)
await expect
.poll(() =>
historyImage.evaluate(
(image: HTMLImageElement) =>
image.complete && image.naturalWidth > 0
)
)
.toBe(true)
})
}
)
Loading
Loading