diff --git a/browser_tests/fixtures/utils/agentAttachmentHelpers.ts b/browser_tests/fixtures/utils/agentAttachmentHelpers.ts new file mode 100644 index 00000000000..d914434d477 --- /dev/null +++ b/browser_tests/fixtures/utils/agentAttachmentHelpers.ts @@ -0,0 +1,127 @@ +import { expect } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' + +import { MIME_ASSET_INFO } from '@/platform/assets/schemas/mediaAssetSchema' +import type { + MediaKind, + parseAssetInfo +} from '@/platform/assets/schemas/mediaAssetSchema' + +type DraggedAssetInfo = NonNullable> + +export interface ExpectedAssetPreview { + filename: string + ref?: string + kind: Extract + width?: number + height?: number +} + +export async function dropAssets( + page: Page, + target: Locator, + assets: ExpectedAssetPreview[], + type: DraggedAssetInfo['type'] = 'output' +) { + for (const { filename, ref = filename, kind } of assets) { + const payload: DraggedAssetInfo = { + filename, + subfolder: '', + type, + display_name: filename, + attachment_ref: ref, + media_kind: kind + } + const dataTransfer = await page.evaluateHandle( + ({ mime, payload }) => { + const transfer = new DataTransfer() + transfer.setData(mime, JSON.stringify(payload)) + return transfer + }, + { mime: MIME_ASSET_INFO, payload } + ) + await target.dispatchEvent('drop', { dataTransfer }) + await dataTransfer.dispose() + } +} + +export async function expectAssets( + panel: Locator, + assets: ExpectedAssetPreview[], + timeout = 10_000 +) { + const deadline = Date.now() + timeout + const remaining = () => Math.max(1, deadline - Date.now()) + const userMessage = panel + .getByTestId('user-message-bubble') + .last() + .locator('..') + const previews = userMessage + .getByTestId('reply-asset-group') + .locator('img, video') + await expect(previews).toHaveCount(assets.length, { timeout: remaining() }) + for (const [index, asset] of assets.entries()) { + const preview = previews.nth(index) + await expect(preview).toBeVisible({ timeout: remaining() }) + await expect(preview.locator('..')).toHaveAccessibleName(asset.filename, { + timeout: remaining() + }) + await expect + .poll( + () => + preview.evaluate((element) => + element instanceof HTMLImageElement + ? element.naturalWidth > 0 && element.naturalHeight > 0 + : element instanceof HTMLVideoElement + ? element.videoWidth > 0 && element.videoHeight > 0 + : false + ), + { timeout: remaining() } + ) + .toBe(true) + } + await expect + .poll( + () => + previews.evaluateAll((elements) => + elements.map((element) => { + if (element instanceof HTMLImageElement) { + return { + kind: 'image', + label: element.alt, + ref: new URL( + element.currentSrc, + location.href + ).searchParams.get('filename'), + width: element.naturalWidth, + height: element.naturalHeight + } + } + if (element instanceof HTMLVideoElement) { + return { + label: element.parentElement?.getAttribute('aria-label'), + ref: element.currentSrc + ? new URL(element.currentSrc, location.href).searchParams.get( + 'filename' + ) + : null, + kind: 'video', + width: element.videoWidth, + height: element.videoHeight + } + } + return null + }) + ), + { timeout: remaining() } + ) + .toEqual( + assets.map(({ filename, ref = filename, kind, width, height }) => ({ + kind, + label: filename, + ref, + width: width ?? expect.any(Number), + height: height ?? expect.any(Number) + })) + ) +} diff --git a/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts b/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts index 4b4d55979dc..6392822a26b 100644 --- a/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts +++ b/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts @@ -3,9 +3,13 @@ import { expect } from '@playwright/test' import type { AgentMessage } from '@comfyorg/ingest-types' import enMessages from '@/locales/en/main.json' with { type: 'json' } -import { MIME_ASSET_INFO } from '@/platform/assets/schemas/mediaAssetSchema' import { promptHistoryTest as test } from '@e2e/fixtures/agentPromptHistoryFixture' +import { + dropAssets, + expectAssets +} from '@e2e/fixtures/utils/agentAttachmentHelpers' +import type { ExpectedAssetPreview } from '@e2e/fixtures/utils/agentAttachmentHelpers' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' import { assetPath } from '@e2e/fixtures/utils/paths' @@ -29,142 +33,233 @@ import { assetPath } from '@e2e/fixtures/utils/paths' test.describe.configure({ timeout: 120_000 }) test.use({ connectWebSocketToServer: false }) -test( - 'keeps a user message asset preview after a browser refresh', - { tag: ['@cloud', '@ui'] }, - async ({ page, promptHistory, workflowSelection }, testInfo) => { - const droppedFilename = 'ComfyUI_00002_.png' - - await page.route( - `**/view?filename=${droppedFilename}&type=input`, - (route) => route.fulfill({ path: assetPath('image64x64.webp') }) - ) - - // The real ingest API returns the message row's `content` map verbatim, - // including `attachment_refs` once a turn's attachments resolve - // server-side (services/agent/server/agent_handler.go's getMessages sets - // messageResponse.Content = m.Content directly). Stand in for that GET - // response here since the shared prompt-history fixture's own mock does - // not yet carry attachments through to `content` at all. POST still goes - // through the fixture's handler unmodified (route.fallback()). - await page.route('**/api/agent/threads/*/messages', (route) => { - if (route.request().method() !== 'GET') return route.fallback() - const request = promptHistory.requests.at(0) - if (!request) return route.fallback() - const threadId = new URL(route.request().url()).pathname - .split('/') - .at(-2)! - const turnId = 'e2e-attachment-turn' - const messages: AgentMessage[] = [ - { - id: 'e2e-attachment-user', - thread_id: threadId, - turn_id: turnId, - seq: 1, - role: 'user', - status: 'complete', - workflow_id: request.workflow_id, - content: { - text: request.content, - attachments: request.attachments, - attachment_refs: (request.attachments ?? []).map((name) => ({ - name, - id: 'asset-e2e-attachment', - kind: 'image' - })) - } - }, - { - id: turnId, - thread_id: threadId, - turn_id: turnId, - seq: 2, - role: 'assistant', - status: 'complete', - content: { text: 'Looks good.' } - } +const firstImage = { + filename: 'ComfyUI_00002_.png', + kind: 'image', + width: 64, + height: 64 +} satisfies ExpectedAssetPreview +const secondImage = { + filename: 'ComfyUI_00003_.png', + kind: 'image', + width: 32, + height: 32 +} satisfies ExpectedAssetPreview +const video = { + filename: 'ComfyUI_00004_.mp4', + ref: 'asset-video.mp4', + kind: 'video' +} satisfies ExpectedAssetPreview +const references = [ + { name: firstImage.filename, id: 'asset-first', kind: 'image' }, + { name: secondImage.filename, id: 'asset-second', kind: 'image' } +] + +for (const scenario of [ + { + name: 'one image from filenames', + assets: [firstImage], + historyContent: { attachments: [firstImage.filename] } + }, + { + name: 'two distinct images from filenames and references', + assets: [firstImage, secondImage], + historyContent: { + attachments: [firstImage.filename, secondImage.filename], + attachment_refs: references + } + }, + { + name: 'two distinct images from references only', + assets: [firstImage, secondImage], + historyContent: { attachment_refs: references } + }, + { + name: 'an image and video from references only', + assets: [firstImage, video], + historyAssets: [firstImage, { ...video, ref: video.filename }], + historyContent: { + attachment_refs: [ + references[0], + { name: video.filename, id: 'asset-video', kind: 'video' } ] - return route.fulfill(jsonRoute(messages)) - }) + } + } +]) { + test( + `keeps ${scenario.name} after refresh and chat switching`, + { tag: ['@cloud', '@ui'] }, + async ({ page, promptHistory, workflowSelection }, testInfo) => { + const filenames = scenario.assets.map((asset) => asset.filename) + let attachmentThreadId: string | undefined - await page - .getByRole('button', { - name: enMessages.agent.entryButton, - exact: true - }) - .click() - await page - .getByRole('button', { - name: enMessages.sideToolbar.newBlankWorkflow, - exact: true + await page.route( + `**/view?filename=${firstImage.filename}&type=input`, + (route) => route.fulfill({ path: assetPath('image64x64.webp') }) + ) + await page.route( + `**/view?filename=${secondImage.filename}&type=input`, + (route) => route.fulfill({ path: assetPath('image32x32.webp') }) + ) + await page.route(`**/view?filename=${video.ref}&type=input`, (route) => + route.fulfill({ + path: assetPath('workflowInMedia/workflow.mp4'), + contentType: 'video/mp4' + }) + ) + await page.route( + `**/view?filename=${video.filename}&type=input`, + (route) => + route.fulfill({ + path: assetPath('workflowInMedia/workflow.mp4'), + contentType: 'video/mp4' + }) + ) + + // The real ingest API returns the message row's `content` map verbatim, + // including `attachment_refs` once a turn's attachments resolve + // server-side (services/agent/server/agent_handler.go's getMessages sets + // messageResponse.Content = m.Content directly). Stand in for that GET + // response here since the shared prompt-history fixture's own mock does + // not yet carry attachments through to `content` at all. POST still goes + // through the fixture's handler unmodified (route.fallback()). + await page.route('**/api/agent/threads/*/messages', (route) => { + const threadId = new URL(route.request().url()).pathname + .split('/') + .at(-2)! + if (route.request().method() === 'POST') { + return route.fallback().then(() => { + attachmentThreadId = threadId + }) + } + if (route.request().method() !== 'GET') return route.fallback() + if (!attachmentThreadId) return route.fallback() + if (threadId !== attachmentThreadId) { + return route.fulfill(jsonRoute([])) + } + const request = promptHistory.requests.at(0) + if (!request) throw new Error('Attachment request was not recorded') + const turnId = 'e2e-attachment-turn' + const messages: AgentMessage[] = [ + { + id: 'e2e-attachment-user', + thread_id: threadId, + turn_id: turnId, + seq: 1, + role: 'user', + status: 'complete', + workflow_id: request.workflow_id, + content: { + text: request.content, + ...scenario.historyContent + } + }, + { + id: turnId, + thread_id: threadId, + turn_id: turnId, + seq: 2, + role: 'assistant', + status: 'complete', + content: { text: 'Looks good.' } + } + ] + return route.fulfill(jsonRoute(messages)) }) - .click() - const panel = page.locator('#agent-panel-root') - await expect(panel).toBeVisible() - await panel - .getByRole('button', { name: enMessages.agent.switchWorkflow }) - .click() - await page - .getByRole('menuitemradio', { name: 'Unsaved Workflow', exact: true }) - .click() - await expect.poll(() => workflowSelection.savedPaths.length).toBe(1) - workflowSelection.finishSave(true) - - // Simulate dropping a generated asset card into the composer, the same - // way agentBatchOutputAttachment.spec.ts does: the real drag source puts - // this shape on the DataTransfer. - await panel.dispatchEvent('drop', { - dataTransfer: await page.evaluateHandle( - ({ mime, filename }) => { - const dataTransfer = new DataTransfer() - dataTransfer.setData( - mime, - JSON.stringify({ - filename, - subfolder: '', - type: 'output', - attachment_ref: filename, - media_kind: 'image' - }) - ) - return dataTransfer - }, - { mime: MIME_ASSET_INFO, filename: droppedFilename } + + await page + .getByRole('button', { + name: enMessages.agent.entryButton, + exact: true + }) + .click() + await page + .getByRole('button', { + name: enMessages.sideToolbar.newBlankWorkflow, + exact: true + }) + .click() + const panel = page.locator('#agent-panel-root') + await expect(panel).toBeVisible() + await panel + .getByRole('button', { name: enMessages.agent.switchWorkflow }) + .click() + await page + .getByRole('menuitemradio', { name: 'Unsaved Workflow', exact: true }) + .click() + await expect.poll(() => workflowSelection.savedPaths.length).toBe(1) + workflowSelection.finishSave(true) + + // Simulate dropping a generated asset card into the composer, the same + // way agentBatchOutputAttachment.spec.ts does: the real drag source puts + // this shape on the DataTransfer. + const composer = panel.getByRole('textbox', { name: /^Describe ideas/ }) + await dropAssets(page, composer, scenario.assets, 'input') + for (const filename of filenames) { + await expect(panel.getByTestId('composer-asset-section')).toContainText( + filename + ) + } + + await composer.pressSequentially('compare these attachments') + await panel + .getByRole('button', { name: enMessages.agent.send, exact: true }) + .click() + await expect.poll(() => promptHistory.requests.length).toBe(1) + expect(promptHistory.requests[0].attachments).toEqual( + scenario.assets.map((asset) => + 'ref' in asset && asset.ref ? asset.ref : asset.filename + ) ) - }) - - const composer = panel.getByRole('textbox', { name: /^Describe ideas/ }) - await composer.pressSequentially('check this image') - await panel - .getByRole('button', { name: enMessages.agent.send, exact: true }) - .click() - await expect.poll(() => promptHistory.requests.length).toBe(1) - - const image = panel.getByRole('img', { name: droppedFilename }) - await expect(image).toBeVisible() - await expect(image).toHaveJSProperty('naturalWidth', 64) - await panel.screenshot({ path: testInfo.outputPath('before-reload.png') }) - - // Settle the (WS-less) turn so the reload below is not racing a - // permanently-streaming assistant message. - await panel - .getByRole('button', { name: enMessages.agent.stop, exact: true }) - .click() - - await page.reload() - await expect( - page.getByTestId('integrated-tab-bar-actions') - ).toHaveAttribute('data-agent-gate-settled', 'true', { timeout: 30_000 }) - const reopenedPanel = page.locator('#agent-panel-root') - await expect(reopenedPanel).toBeVisible({ timeout: 30_000 }) - - const reopenedImage = reopenedPanel.getByRole('img', { - name: droppedFilename - }) - await expect(reopenedImage).toBeVisible({ timeout: 10_000 }) - await expect(reopenedImage).toHaveJSProperty('naturalWidth', 64) - await reopenedPanel.screenshot({ - path: testInfo.outputPath('after-reload.png') - }) - } -) + + await expectAssets(panel, scenario.assets) + await panel.screenshot({ path: testInfo.outputPath('before-reload.png') }) + + // Settle the (WS-less) turn so the reload below is not racing a + // permanently-streaming assistant message. + await panel + .getByRole('button', { name: enMessages.agent.stop, exact: true }) + .click() + + await page.reload() + await expect( + page.getByTestId('integrated-tab-bar-actions') + ).toHaveAttribute('data-agent-gate-settled', 'true', { timeout: 30_000 }) + const reopenedPanel = page.locator('#agent-panel-root') + await expect(reopenedPanel).toBeVisible({ timeout: 30_000 }) + + const historyAssets = + 'historyAssets' in scenario && scenario.historyAssets + ? scenario.historyAssets + : scenario.assets + await expectAssets(reopenedPanel, historyAssets) + await reopenedPanel.screenshot({ + path: testInfo.outputPath('after-reload.png') + }) + + await reopenedPanel + .getByRole('button', { name: enMessages.agent.newChat }) + .click() + await expect( + reopenedPanel.getByTestId('user-message-bubble') + ).toHaveCount(0) + await expectAssets(reopenedPanel, []) + await reopenedPanel + .getByRole('button', { name: enMessages.agent.showChatHistory }) + .click() + await reopenedPanel + .getByRole('button', { + name: 'Inline reference round trip', + exact: true + }) + .click() + await expect( + reopenedPanel.getByTestId('user-message-bubble') + ).toContainText('compare these attachments') + await expectAssets(reopenedPanel, historyAssets) + await reopenedPanel.screenshot({ + path: testInfo.outputPath('after-chat-switch.png') + }) + } + ) +}