diff --git a/browser_tests/tests/agent/agentExecutionStoryFixture.ts b/browser_tests/tests/agent/agentExecutionStoryFixture.ts new file mode 100644 index 00000000000..0a0f01ddcbc --- /dev/null +++ b/browser_tests/tests/agent/agentExecutionStoryFixture.ts @@ -0,0 +1,154 @@ +import { expect } from '@playwright/test' +import type { Page } from '@playwright/test' + +import type { AgentCancelAccepted } from '@comfyorg/ingest-types' + +import type { JobDetail } from '@/platform/remote/comfyui/jobs/jobTypes' + +import { agentTest } from '@e2e/fixtures/agentPanelFixture' +import type { AgentFollowerHostSocket } from '@e2e/fixtures/agentFollowerHostSocket' +import { AgentExecutionHelper } from '@e2e/fixtures/helpers/AgentExecutionHelper' +import { AssetsHelper } from '@e2e/fixtures/helpers/AssetsHelper' +import { MultiAutogrowRealignHarness } from '@e2e/fixtures/helpers/MultiAutogrowRealignHarness' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { + MESSAGE_ID, + THREAD_ID +} from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture' +import enMessages from '@/locales/en/main.json' with { type: 'json' } + +const JOB_CANCEL_ROUTE_PATTERN = + /\/api\/(?:interrupt|jobs\/(?:[^/]+\/)?cancel)$/ + +/** + * `AgentExecutionHelper` plus the two surfaces only these user-story specs + * need, kept here rather than pushed into the shared helper: a recorder for + * job interrupt/cancel requests (so "the agent's turn stopped and the job did + * not" is observable as the *absence* of a cancel), and a per-job + * `GET /api/jobs/{id}` detail route. + * + * Registered after `super.install()`, which is what makes these routes win — + * Playwright runs the most-recently-registered matching handler first — and + * the same reason the base helper installs after the harness. + */ +class AgentExecutionStoryHelper extends AgentExecutionHelper { + private readonly jobCancels: string[] = [] + private readonly storyPage: Page + private readonly storyAssets: AssetsHelper + + constructor(page: Page, hostSocket: AgentFollowerHostSocket) { + super(page, hostSocket) + this.storyPage = page + this.storyAssets = new AssetsHelper(page) + } + + override async install(): Promise { + await super.install() + await this.storyPage.route(JOB_CANCEL_ROUTE_PATTERN, (route) => { + this.jobCancels.push(new URL(route.request().url()).pathname) + return route.fulfill({ status: 200, body: '' }) + }) + } + + /** Paths of job interrupt or cancel requests, in request order. */ + jobCancelRequests(): readonly string[] { + return this.jobCancels + } + + /** + * Serves `GET /api/jobs/{jobId}` — the detail record, which is separate from + * the list row `enqueueServerRun` and `upsertJob` maintain. Its `workflow` + * field is the body that was POSTed to `/api/prompt`, stored verbatim + * (cloud `services/ingest/.../prompt.go`, `SetWorkflowJSON`), and it is the + * only thing the "open as workflow" action has to rebuild a graph from. + * Without this route that fetch 404s, so a spec about what the action does + * with the detail must supply one rather than read a failed fetch as a + * product verdict. + */ + async mockJobDetail(jobId: string, detail: JobDetail): Promise { + await this.storyAssets.mockJobDetail(jobId, detail) + } +} + +interface AgentExecutionStoryRig { + harness: MultiAutogrowRealignHarness + execution: AgentExecutionStoryHelper +} + +/** + * The execution rig of `agentExecutionFixture`, but with no turn sent yet: + * the harness's message mock serves exactly one `message_id`, so a user-story + * spec whose subject IS a turn — "run it", or stopping one mid-flight — must + * drive that turn itself rather than spend the only one on a binding + * handshake. The panel is open and targeting the workflow; sending the + * story's message is what acks the turn and binds the CRDT follower. + */ +export const agentExecutionStoryTest = agentTest.extend<{ + rig: AgentExecutionStoryRig +}>({ + rig: async ({ page }, use) => { + const harness = new MultiAutogrowRealignHarness(page) + await harness.setUp({ settings: { 'Comfy.Queue.QPOV2': false } }) + const execution = new AgentExecutionStoryHelper(page, harness.hostSocket) + await execution.install() + const accepted: AgentCancelAccepted = { status: 'cancelling' } + await page.route('**/api/agent/threads/*/messages/*/cancel', (route) => + route.fulfill({ ...jsonRoute(accepted), status: 202 }) + ) + await harness.targetActiveWorkflow() + await use({ harness, execution }) + } +}) + +/** + * Sends the story's user message and waits until the turn is live: the + * assistant placeholder is rendered (`agent_message_done` is only routed while + * the store holds the ack's `message_id`), the turn's ack has bound the CRDT + * follower, and the workflow the agent would run is visible on the canvas. + */ +export async function startStoryTurn( + harness: MultiAutogrowRealignHarness, + text: string +): Promise { + // `harness.agentPanel.composer`, not a hand-rolled locator: the composer's + // accessible name is the `agent.placeholder` / + // `agent.placeholderWithoutSkills` pair, and `AGENT_COMPOSER_LABEL` tracks + // both. A literal here silently matches nothing and the fill times out. + await harness.agentPanel.composer.fill(text) + await harness.panel + .getByRole('button', { name: enMessages.agent.send }) + .click() + await expect( + harness.panel.getByText(enMessages.agent.thinking).first() + ).toBeVisible() + await harness.hostSocket.waitForSubscribe() + await expect(harness.targetNode).toBeVisible() +} + +/** Ends the live turn from the host and waits for the panel to go idle. */ +export async function settleStoryTurn( + harness: MultiAutogrowRealignHarness +): Promise { + harness.hostSocket.send({ + type: 'agent_message_done', + data: { message_id: MESSAGE_ID, thread_id: THREAD_ID } + }) + await expect( + harness.panel.getByRole('button', { name: enMessages.agent.stop }) + ).toHaveCount(0) +} + +/** The agent's `run` tool call frame, `running` or settled. */ +export function runToolCallFrame(status: 'running' | 'success') { + return { + type: 'agent_tool_call' as const, + data: { + tool_call_id: 'call-run', + tool_name: 'run', + status, + ...(status === 'success' && { duration_ms: 1200 }), + message_id: MESSAGE_ID, + thread_id: THREAD_ID + } + } +} diff --git a/browser_tests/tests/agent/agentFailedRunIsSurfaced.spec.ts b/browser_tests/tests/agent/agentFailedRunIsSurfaced.spec.ts new file mode 100644 index 00000000000..f62b18da26e --- /dev/null +++ b/browser_tests/tests/agent/agentFailedRunIsSurfaced.spec.ts @@ -0,0 +1,52 @@ +import { expect } from '@playwright/test' + +import { + agentExecutionStoryTest as test, + runToolCallFrame, + settleStoryTurn, + startStoryTurn +} from '@e2e/tests/agent/agentExecutionStoryFixture' +import { QueuePanel } from '@e2e/fixtures/components/QueuePanel' +import { TARGET_ID } from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture' +import { TestIds } from '@e2e/fixtures/selectors' +import enMessages from '@/locales/en/main.json' with { type: 'json' } + +test.describe( + 'Agent failed run', + { tag: ['@cloud', '@agent', '@vue-nodes'] }, + () => { + test('a run that fails after the turn ends names its error and leaves the panel idle', async ({ + rig: { harness, execution }, + page + }) => { + const jobId = 'agent-surfaced-failed-run-1' + const failureMessage = 'deterministic fixture failure' + await startStoryTurn(harness, 'run it') + harness.hostSocket.send(runToolCallFrame('running')) + await execution.enqueueServerRun(jobId) + await execution.startJob(jobId) + harness.hostSocket.send(runToolCallFrame('success')) + await settleStoryTurn(harness) + + await execution.failJob(jobId, { + nodeId: TARGET_ID, + message: failureMessage + }) + + const queuePanel = new QueuePanel(page) + await queuePanel.open() + await queuePanel.jobRow(jobId).hover() + const details = page.getByTestId(TestIds.queue.jobDetailsPopover) + await expect( + details.getByText(enMessages.queue.jobDetails.errorMessage) + ).toBeVisible() + await expect(details.getByText(failureMessage)).toBeVisible() + await expect( + harness.panel.getByRole('button', { name: enMessages.agent.stop }) + ).toHaveCount(0) + await expect( + harness.panel.getByText(enMessages.agent.thinking) + ).toHaveCount(0) + }) + } +) diff --git a/browser_tests/tests/agent/agentGoldenPathRunIt.spec.ts b/browser_tests/tests/agent/agentGoldenPathRunIt.spec.ts new file mode 100644 index 00000000000..f6d549616b3 --- /dev/null +++ b/browser_tests/tests/agent/agentGoldenPathRunIt.spec.ts @@ -0,0 +1,170 @@ +import { expect } from '@playwright/test' + +import type { AgentAnswerAccepted } from '@comfyorg/ingest-types' +import { zAgentAnswerRequest } from '@comfyorg/ingest-types/zod' + +import { + agentExecutionStoryTest as test, + runToolCallFrame, + settleStoryTurn, + startStoryTurn +} from '@e2e/tests/agent/agentExecutionStoryFixture' +import { QueuePanel } from '@e2e/fixtures/components/QueuePanel' +import { nextFrame } from '@e2e/fixtures/utils/timing' +import { + MESSAGE_ID, + SOURCE_NODE_ID, + THREAD_ID, + WORKFLOW_ID +} from '@e2e/fixtures/data/agent/agentCrdtMultiAutogrowRealignFixture' +import enMessages from '@/locales/en/main.json' with { type: 'json' } + +const JOB_ID = 'agent-golden-path-run-1' +const OUTPUT_FILENAME = 'agent-golden-path-out.png' +const ASK_ID = `${MESSAGE_ID}:call-run` +const ANSWER_PATH = `/api/agent/threads/${THREAD_ID}/asks/${encodeURIComponent(ASK_ID)}/answer` + +const ids = { message_id: MESSAGE_ID, thread_id: THREAD_ID } + +/** + * GOLDEN PATH (user-story matrix rank 26): "run it" submits the workflow and + * shows me the output. + * + * The journey, end to end as the user sees it: ask the agent to run the + * workflow, approve the run when the agent asks, watch the run land on the + * queue, and see the finished output on the canvas node and in job history. + * The run is submitted by the agent's `run` tool on the server — approval is + * answered over REST and the browser never POSTs `/api/prompt` — which is why + * the job enters through `enqueueServerRun` rather than a client submission. + * + * Harness verification for every lever used here is + * `agentExecutionLifecycle.spec.ts`; this spec asserts the product journey on + * top of that surface, not the fixture itself. + */ +test.describe( + 'Agent golden path', + { tag: ['@cloud', '@agent', '@vue-nodes'] }, + () => { + test('"run it" submits the workflow and shows me the output', async ({ + rig: { harness, execution }, + page + }) => { + // Run approval is answered over REST; the parked turn then resumes on + // the socket (`agent_ask_resolved` is the canonical resolution). + const answers: { method: string; path: string; selected: string[] }[] = [] + await page.route('**/api/agent/threads/*/asks/*/answer', (route) => { + const body = zAgentAnswerRequest.parse(route.request().postDataJSON()) + answers.push({ + method: route.request().method(), + path: new URL(route.request().url()).pathname, + selected: body.selected + }) + const accepted: AgentAnswerAccepted = { + status: 'answered', + selected: body.selected + } + return route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify(accepted) + }) + }) + + // The user asks the agent to run the workflow. + await startStoryTurn(harness, 'run it') + + // The agent asks for approval to run, and the user approves. + harness.hostSocket.send({ + type: 'agent_ask', + data: { + ...ids, + ask_id: ASK_ID, + kind: 'run_approval', + context: { + workflow_id: WORKFLOW_ID, + workflow_name: 'Unsaved Workflow' + }, + prompt: 'Run workflow “Unsaved Workflow”?', + options: [ + { id: 'run', label: 'Run' }, + { id: 'cancel', label: 'Cancel' } + ], + min_selections: 1, + max_selections: 1, + allow_other: false + } + }) + await expect( + harness.panel.getByText(enMessages.agent.runApproval.leadBound) + ).toBeVisible() + await harness.panel + .getByRole('button', { + name: enMessages.agent.runApproval.run, + exact: true + }) + .click() + await expect.poll(() => answers.length).toBe(1) + expect(answers[0]).toEqual({ + method: 'POST', + path: ANSWER_PATH, + selected: ['run'] + }) + + // The turn resumes: the run tool submits the workflow server-side and + // the job starts, then the agent's turn completes. + harness.hostSocket.send({ + type: 'agent_ask_resolved', + data: { ...ids, ask_id: ASK_ID, status: 'answered', selected: ['run'] } + }) + harness.hostSocket.send(runToolCallFrame('running')) + await execution.enqueueServerRun(JOB_ID) + await execution.startJob(JOB_ID) + harness.hostSocket.send(runToolCallFrame('success')) + await settleStoryTurn(harness) + + // "Submits the workflow": the run is a real job the user can see on the + // queue surface. It was submitted by the agent on the server — the + // browser itself never POSTed /api/prompt. + const queuePanel = new QueuePanel(page) + await queuePanel.open() + await expect(queuePanel.jobRow(JOB_ID)).toBeVisible() + expect(execution.submittedPrompts()).toHaveLength(0) + + // "Shows me the output": the finished run's image renders on the node + // (completed onto the source node — media type is inferred from a + // node's inputs, and the target's video inputs would render an image + // output as a