diff --git a/browser_tests/assets/silence.wav b/browser_tests/assets/silence.wav new file mode 100644 index 00000000000..555f153a976 Binary files /dev/null and b/browser_tests/assets/silence.wav differ diff --git a/browser_tests/fixtures/components/AgentPanel.ts b/browser_tests/fixtures/components/AgentPanel.ts index 1f613987e4e..d03d2a83579 100644 --- a/browser_tests/fixtures/components/AgentPanel.ts +++ b/browser_tests/fixtures/components/AgentPanel.ts @@ -21,6 +21,7 @@ export class AgentPanel { public readonly workflowPicker: Locator public readonly fileInput: Locator public readonly composerAssetSection: Locator + public readonly scrollAssetsRight: Locator public readonly attachmentChips: Locator public readonly composer: Locator public readonly composerPromptArea: Locator @@ -64,6 +65,9 @@ export class AgentPanel { }) this.fileInput = this.root.getByTestId('agent-file-input') this.composerAssetSection = this.root.getByTestId('composer-asset-section') + this.scrollAssetsRight = this.root.getByRole('button', { + name: enMessages.g.scrollRight + }) this.attachmentChips = this.root.getByTestId('agent-attachment-chip') this.composer = this.root.getByRole('textbox', { name: /^Describe ideas/ }) this.composerPromptArea = this.root.getByTestId('composer-inline-input') @@ -86,13 +90,67 @@ export class AgentPanel { .getByRole('listitem') } + async scrollAssetsToEnd(): Promise { + const count = await this.attachmentChips.count() + for ( + let step = 0; + step < count && (await this.scrollAssetsRight.isEnabled()); + step++ + ) { + const target = await this.composerAssetSection.evaluate((element) => + Math.min( + element.scrollWidth - element.clientWidth, + element.scrollLeft + element.clientWidth + ) + ) + await this.scrollAssetsRight.click() + await expect + .poll(() => + this.composerAssetSection.evaluate((element) => element.scrollLeft) + ) + .toBeCloseTo(target, 0) + } + await expect(this.scrollAssetsRight).toBeDisabled() + } + activityRow(label: string): Locator { return this.activityRows.getByText(label, { exact: true }) } + assetPreview(name: string): Locator { + return this.page.getByRole('dialog', { name, exact: true }) + } + + previewAssetButton(name: string): Locator { + return this.attachmentChip(name).getByRole('button', { + name: enMessages.agent.previewAsset.replace('{name}', name), + exact: true + }) + } + + async expectAttachmentFullyVisible(name: string): Promise { + const tray = this.root.getByRole('region', { + name: enMessages.assetBrowser.assets, + exact: true + }) + await expect + .poll(async () => { + const [card, viewport] = await Promise.all([ + this.attachmentChip(name).boundingBox(), + tray.boundingBox() + ]) + return ( + !!card && + !!viewport && + card.y >= viewport.y && + card.y + card.height <= viewport.y + viewport.height + ) + }) + .toBe(true) + } + /** - * The composer attachment carrying `name`. Matches on the chip's own - * attribute rather than its text, which truncates at `max-w-32`. + * The composer attachment carrying `name`. * * `name` is a filename and may legitimately contain a quote or backslash, so * it is escaped for the double-quoted CSS string rather than interpolated diff --git a/browser_tests/fixtures/data/assetFixtures.ts b/browser_tests/fixtures/data/assetFixtures.ts index c6fc819d6ea..41f1181cdf6 100644 --- a/browser_tests/fixtures/data/assetFixtures.ts +++ b/browser_tests/fixtures/data/assetFixtures.ts @@ -75,6 +75,17 @@ export const AGENT_VIDEO_ASSET: Asset = createOutputAsset({ last_access_time: '2026-09-18T00:00:00.000Z' }) +export const AGENT_AUDIO_ASSET: Asset = createOutputAsset({ + id: '33333333-3333-4333-a333-333333333333', + hash: 'blake3:3333333333333333333333333333333333333333333333333333333333333333', + name: 'agent_generated_audio.wav', + mime_type: 'audio/wav', + size: 24_044, + created_at: '2026-09-18T00:00:00.000Z', + updated_at: '2026-09-18T00:00:00.000Z', + last_access_time: '2026-09-18T00:00:00.000Z' +}) + export const STALE_TEMP_FILENAME = 'ComfyUI_temp_thsmm_00001_.png' export const STALE_TEMP_CARD_TEXT = STALE_TEMP_FILENAME.replace(/\.[^.]+$/, '') export const STALE_TEMP_ASSET: Asset = createOutputAsset({ diff --git a/browser_tests/fixtures/helpers/DragDropHelper.ts b/browser_tests/fixtures/helpers/DragDropHelper.ts index 06740fc8811..532f635b67a 100644 --- a/browser_tests/fixtures/helpers/DragDropHelper.ts +++ b/browser_tests/fixtures/helpers/DragDropHelper.ts @@ -1,7 +1,7 @@ -import { readFileSync } from 'fs' +import { readFileSync, statSync } from 'fs' import { basename } from 'path' -import type { Page } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' import type { Position } from '@e2e/fixtures/types' import { getMimeType } from '@e2e/fixtures/utils/mimeTypeUtil' @@ -11,6 +11,28 @@ import { nextFrame } from '@e2e/fixtures/utils/timing' export class DragDropHelper { constructor(private readonly page: Page) {} + async dropFilesOn(target: Locator, filePaths: string[]): Promise { + const files = filePaths.map((filePath) => ({ + bytes: Array.from(readFileSync(filePath)), + name: basename(filePath), + type: getMimeType(basename(filePath)), + lastModified: statSync(filePath).mtimeMs + })) + const transfer = await this.page.evaluateHandle((files) => { + const transfer = new DataTransfer() + for (const { bytes, name, type, lastModified } of files) + transfer.items.add( + new File([new Uint8Array(bytes)], name, { type, lastModified }) + ) + return transfer + }, files) + try { + await target.dispatchEvent('drop', { dataTransfer: transfer }) + } finally { + await transfer.dispose() + } + } + async dragAndDropExternalResource( options: { fileName?: string diff --git a/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts b/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts new file mode 100644 index 00000000000..f5244341920 --- /dev/null +++ b/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict' + +import { expect } from '@playwright/test' + +import { + agentTest as test, + bootAgentApp +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { AssetsSidebarTab } from '@e2e/fixtures/components/SidebarTab' +import { + AGENT_AUDIO_ASSET, + AGENT_VIDEO_ASSET +} from '@e2e/fixtures/data/assetFixtures' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.describe('Asset preview playback', { tag: '@cloud' }, () => { + for (const { kind, asset, file, contentType } of [ + { + kind: 'video', + asset: AGENT_VIDEO_ASSET, + file: 'plain_video.mp4', + contentType: 'video/mp4' + }, + { + kind: 'audio', + asset: AGENT_AUDIO_ASSET, + file: 'silence.wav', + contentType: 'audio/wav' + } + ] as const) { + test(`stops ${kind} at close and reopens paused during the exit animation`, async ({ + page, + agentFlagEnabled + }) => { + await page.route(`**/api/assets/${asset.id}/content**`, (route) => + route.fulfill({ path: assetPath(file), contentType }) + ) + await bootAgentApp(page, agentFlagEnabled, { + assets: { assets: [asset], total: 1, has_more: false } + }) + const panel = new AgentPanel(page) + await panel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(panel.root) + await panel.composer.fill('Keep this draft') + const trigger = panel.previewAssetButton(asset.name) + await trigger.press('Enter') + const player = panel.assetPreview(asset.name).locator(kind) + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.readyState) + ) + .toBeGreaterThanOrEqual(2) + await player.press('Space') + await expect + .poll(() => player.evaluate((media: HTMLMediaElement) => media.paused)) + .toBe(false) + const playingElement = await player.elementHandle() + assert(playingElement) + try { + await page.keyboard.press('Escape') + expect( + await playingElement.evaluate( + (media) => media instanceof HTMLMediaElement && media.paused + ) + ).toBe(true) + await page.keyboard.press('Enter') + await expect(trigger).toHaveAttribute('aria-expanded', 'true') + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (media: HTMLMediaElement) => + media.paused && media.currentTime === 0 + ) + ) + .toBe(true) + await player.press('Space') + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.paused) + ) + .toBe(false) + const reopenedElement = await player.elementHandle() + assert(reopenedElement) + try { + await panel.composer.click() + expect( + await reopenedElement.evaluate( + (media) => media instanceof HTMLMediaElement && media.paused + ) + ).toBe(true) + } finally { + await reopenedElement.dispose() + } + await expect(trigger).toHaveAttribute('aria-expanded', 'false') + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + } finally { + await playingElement.dispose() + } + }) + } +}) diff --git a/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts b/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts new file mode 100644 index 00000000000..e81ba4bdb9b --- /dev/null +++ b/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict' + +import { expect } from '@playwright/test' + +import { + agentTest as test, + bootAgentApp +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { AssetsSidebarTab } from '@e2e/fixtures/components/SidebarTab' +import { + AGENT_AUDIO_ASSET, + AGENT_VIDEO_ASSET +} from '@e2e/fixtures/data/assetFixtures' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.describe('Single asset tray preview', { tag: '@cloud' }, () => { + test.beforeEach(async ({ page, agentFlagEnabled }) => { + await page.route( + `**/api/assets/${AGENT_AUDIO_ASSET.id}/content**`, + (route) => + route.fulfill({ + path: assetPath('silence.wav'), + contentType: 'audio/wav' + }) + ) + await page.route( + `**/api/assets/${AGENT_VIDEO_ASSET.id}/content**`, + (route) => + route.fulfill({ + path: assetPath('plain_video.mp4'), + contentType: 'video/mp4' + }) + ) + await bootAgentApp(page, agentFlagEnabled, { + assets: { + assets: [AGENT_AUDIO_ASSET, AGENT_VIDEO_ASSET], + total: 2, + has_more: false + } + }) + const panel = new AgentPanel(page) + await panel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.waitForAssets(2) + await assets.getAssetCardByName('agent_generated_audio').dragTo(panel.root) + await expect(panel.attachmentChip(AGENT_AUDIO_ASSET.name)).toBeVisible() + await assets.getAssetCardByName('agent_generated_video').dragTo(panel.root) + await expect(panel.attachmentChips).toHaveCount(2) + await panel.composer.fill('Keep this draft') + }) + + test('replaces a clicked preview on hover and preserves keyboard switching and dismissal', async ({ + page + }) => { + const panel = new AgentPanel(page) + const audio = panel.previewAssetButton(AGENT_AUDIO_ASSET.name) + const video = panel.previewAssetButton(AGENT_VIDEO_ASSET.name) + await audio.click() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toBeVisible() + await video.hover() + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toBeVisible() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toHaveCount(0) + await expect(audio).toHaveAttribute('aria-expanded', 'false') + await panel.composer.hover() + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toHaveCount(0) + + await audio.press('Enter') + await expect( + panel.assetPreview(AGENT_AUDIO_ASSET.name).locator('audio') + ).toBeFocused() + await video.press('Space') + await expect( + panel.assetPreview(AGENT_VIDEO_ASSET.name).locator('video') + ).toBeFocused() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toHaveCount(0) + await page.keyboard.press('Escape') + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toHaveCount(0) + await expect(video).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + }) + + for (const { kind, asset, nextAsset } of [ + { kind: 'audio', asset: AGENT_AUDIO_ASSET, nextAsset: AGENT_VIDEO_ASSET }, + { kind: 'video', asset: AGENT_VIDEO_ASSET, nextAsset: AGENT_AUDIO_ASSET } + ] as const) { + test(`stops native ${kind} playback when hover replaces it and reopens paused`, async ({ + page + }) => { + const panel = new AgentPanel(page) + const trigger = panel.previewAssetButton(asset.name) + await trigger.click() + const player = panel.assetPreview(asset.name).locator(kind) + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.readyState) + ) + .toBeGreaterThanOrEqual(2) + await player.press('Space') + await expect + .poll(() => player.evaluate((media: HTMLMediaElement) => media.paused)) + .toBe(false) + const playing = await player.elementHandle() + assert(playing) + try { + await panel.previewAssetButton(nextAsset.name).hover() + await expect(panel.assetPreview(nextAsset.name)).toBeVisible() + await expect + .poll(() => + playing.evaluate( + (media) => + media instanceof HTMLMediaElement && + media.paused && + media.currentTime === 0 + ) + ) + .toBe(true) + await expect(panel.assetPreview(asset.name)).toHaveCount(0) + await trigger.press('Enter') + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (media: HTMLMediaElement) => + media.paused && media.currentTime === 0 + ) + ) + .toBe(true) + await expect(panel.assetPreview(nextAsset.name)).toHaveCount(0) + await page.keyboard.press('Escape') + await expect(trigger).toBeFocused() + await expect(panel.assetPreview(asset.name)).toHaveCount(0) + } finally { + await playing.dispose() + } + }) + } +}) diff --git a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts b/browser_tests/tests/agent/agentAttachJsonFile.spec.ts index a9e5605627e..456529ec19c 100644 --- a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts +++ b/browser_tests/tests/agent/agentAttachJsonFile.spec.ts @@ -41,13 +41,18 @@ test.describe( // that was reported (the OS picker hid .json files) and the actual fix // (AGENT_ATTACH_ACCEPT now lists .json/application/json). await fileInput.setInputFiles(assetPath('default.json')) - await expect(assetSection).toContainText('default.json') + const attachment = assetSection.getByRole('group', { + name: 'default.json', + exact: true + }) + await expect(attachment).toBeVisible() await expect( assetSection.locator(`[aria-label="${enMessages.agent.uploading}"]`) ).toHaveCount(0) - await assetSection - .getByRole('button', { name: enMessages.agent.remove, exact: true }) + await attachment.hover() + await attachment + .getByRole('button', { name: 'Remove default.json', exact: true }) .click() await expect(assetSection).toHaveCount(0) }) diff --git a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts index f59fee1e7e4..a71501b0416 100644 --- a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts +++ b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts @@ -40,7 +40,9 @@ test.describe('Agent attachment cancellation', { tag: '@cloud' }, () => { }) await uploadReceived await expect( - panel.getByTestId('composer-asset-section').getByText('pending.mp4') + panel + .getByTestId('composer-asset-section') + .getByRole('group', { name: 'pending.mp4' }) ).toBeVisible() await expect(send).toBeDisabled() @@ -51,9 +53,9 @@ test.describe('Agent attachment cancellation', { tag: '@cloud' }, () => { await openButton.click() await expect(composer).toHaveText('Keep this draft') - await expect(panel.getByText('pending.mp4', { exact: true })).toHaveCount( - 0 - ) + await expect( + panel.getByRole('group', { name: 'pending.mp4' }) + ).toHaveCount(0) await expect(send).toBeEnabled() } finally { releaseUpload() diff --git a/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts index 8e18cb97743..53fd750fbc0 100644 --- a/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts +++ b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts @@ -41,24 +41,29 @@ test.describe('Agent composer bulk attachments', { tag: '@cloud' }, () => { ) await expect(agentPanel.attachmentChips).toHaveCount(ATTACHMENT_COUNT) + await expect(agentPanel.attachmentChips.last()).not.toBeInViewport({ + ratio: 1 + }) + await agentPanel.scrollAssetsToEnd() + await expect(agentPanel.attachmentChips.last()).toBeInViewport({ ratio: 1 }) await expect(agentPanel.sendButton).toBeEnabled() // ratio: 1 rather than the default ratio: 0 — a Send button clipped down to // a sliver is the FE-3202 symptom, and it satisfies ratio: 0. await expect(agentPanel.sendButton).toBeInViewport({ ratio: 1 }) - // The cap is only a fix if what it hides stays reachable: `overflow-hidden` - // in its place strands 32 of the 36 chips yet satisfies every other - // assertion here, so this scrolls to the last chip rather than asserting a - // class. The baseline keeps the wheel the only thing that can move the list. - const scrollTop = () => - agentPanel.composerAssetSection.evaluate((element) => element.scrollTop) - await expect.poll(scrollTop).toBe(0) await agentPanel.composerAssetSection.hover() - await page.mouse.wheel(0, 2000) - await expect.poll(scrollTop).toBeGreaterThan(0) + await page.mouse.wheel(-10000, 0) + await expect(agentPanel.attachmentChips.first()).toBeInViewport({ + ratio: 1 + }) + await expect(agentPanel.attachmentChips.last()).not.toBeInViewport({ + ratio: 1 + }) + await page.mouse.wheel(10000, 0) await expect( agentPanel.attachmentChip(`reference-${ATTACHMENT_COUNT}.png`) ).toBeInViewport({ ratio: 1 }) + await expect(agentPanel.sendButton).toBeInViewport({ ratio: 1 }) // Click, not Enter: the keyboard path posts even when the button has been // pushed out of reach, so only a real click proves Send is usable. diff --git a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts index 26cf330b2a2..6d51327f22a 100644 --- a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts +++ b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts @@ -1,3 +1,5 @@ +import assert from 'node:assert/strict' + import { expect } from '@playwright/test' import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' @@ -36,12 +38,16 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { ) await bootAgentApp(page, agentFlagEnabled, { - assets: { assets: [AGENT_VIDEO_ASSET], total: 1, has_more: false } + assets: { + assets: [{ ...AGENT_VIDEO_ASSET, display_name: null }], + total: 1, + has_more: false + } }) }) for (const view of ['grid', 'list'] as const) { - test(`attaches an asset dragged onto the composer from ${view} view`, async ({ + test(`re-attaches a library asset without a display name from ${view} view`, async ({ page }) => { const agentPanel = new AgentPanel(page) @@ -53,6 +59,7 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { if (view === 'list') { await assets.openSettingsMenu() await assets.listViewOption.click() + await assets.closeSettingsMenu() } const source = @@ -68,9 +75,171 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { // PM-1157/PM-1158 failure, and it would pass a bare count assertion. await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() await expect(agentPanel.attachmentChips).toHaveCount(1) + await source.dragTo(agentPanel.root) + await expect(agentPanel.attachmentChips).toHaveCount(1) + const duplicateNotice = page.getByRole('alert').filter({ + hasText: `${ASSET_NAME} is already in the asset tray` + }) + await expect(duplicateNotice).toHaveCount(1) + await duplicateNotice.getByRole('button', { name: 'Close' }).click() + await agentPanel.composer.fill('Keep this draft @agent_generated') + await agentPanel.root + .getByRole('menuitem', { name: ASSET_NAME, exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) + await agentPanel.attachmentChip(ASSET_NAME).hover() + await agentPanel + .attachmentChip(ASSET_NAME) + .getByRole('button', { name: `Remove ${ASSET_NAME}`, exact: true }) + .click() + await expect(agentPanel.attachmentChips).toHaveCount(0) + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await expect(agentPanel.composer).toContainText('Keep this draft') + await source.dragTo(agentPanel.root) + await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() + await expect(agentPanel.attachmentChips).toHaveCount(1) + await agentPanel.composer.press('End') + await agentPanel.composer.pressSequentially('@agent_generated') + await agentPanel.root + .getByRole('menuitem', { name: ASSET_NAME, exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) }) } + test('reuses a decoded video poster and supports native keyboard playback with cleanup', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(agentPanel.root) + const poster = agentPanel + .attachmentChip(ASSET_NAME) + .getByRole('img', { name: ASSET_NAME, exact: true }) + await expect(poster).toHaveAttribute('src', /^blob:/) + await expect + .poll(() => + poster.evaluate( + (image: HTMLImageElement) => image.complete && image.naturalWidth > 0 + ) + ) + .toBe(true) + const posterUrl = await poster.getAttribute('src') + assert(posterUrl) + await agentPanel.composer.fill('Animate @agent_generated') + const suggestion = agentPanel.root.getByRole('menuitem', { + name: ASSET_NAME, + exact: true + }) + await expect(suggestion.locator('img')).toHaveAttribute('src', posterUrl) + await suggestion.click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip').locator('img') + ).toHaveAttribute('src', posterUrl) + await agentPanel.composer.hover() + await expect(agentPanel.assetPreview(ASSET_NAME)).toHaveCount(0) + await agentPanel.composer.pressSequentially(' with motion') + await expect(agentPanel.composer).toBeFocused() + await agentPanel.composerAssetSection.focus() + await agentPanel.composerAssetSection.press('Tab') + const trigger = agentPanel.previewAssetButton(ASSET_NAME) + await expect(trigger).toBeFocused() + await trigger.press('Enter') + const preview = agentPanel.assetPreview(ASSET_NAME) + const player = preview.locator('video') + await expect(preview).toBeVisible() + await expect(player).toBeFocused() + await expect(player).toHaveAttribute('controls', '') + await expect(player).not.toHaveAttribute('autoplay') + await expect(player).toHaveAttribute('poster', posterUrl) + await expect + .poll(() => + player.evaluate((video: HTMLVideoElement) => video.readyState) + ) + .toBeGreaterThanOrEqual(2) + await expect + .poll(() => player.evaluate((video: HTMLVideoElement) => video.paused)) + .toBe(true) + await player.press('Space') + await expect + .poll(() => player.evaluate((video: HTMLVideoElement) => video.paused)) + .toBe(false) + await expect + .poll(() => + player.evaluate((video: HTMLVideoElement) => video.currentTime) + ) + .toBeGreaterThan(0) + const playingElement = await player.elementHandle() + assert(playingElement) + await player.press('Escape') + await expect(preview).toHaveCount(0) + await expect(trigger).toBeFocused() + await expect + .poll(() => + playingElement.evaluate( + (video) => + video instanceof HTMLVideoElement && + video.paused && + !video.isConnected + ) + ) + .toBe(true) + await playingElement.dispose() + await trigger.press('Space') + await expect(player).toBeVisible() + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (video: HTMLVideoElement) => video.paused && video.currentTime === 0 + ) + ) + .toBe(true) + await player.press('Escape') + await expect(preview).toHaveCount(0) + await expect(agentPanel.composer).toContainText(' with motion') + }) + + test.describe('a short viewport', () => { + test.use({ viewport: { width: 1024, height: 520 } }) + + test('keeps the complete asset card visible as the prompt grows', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(agentPanel.root) + await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() + await agentPanel.expectAttachmentFullyVisible(ASSET_NAME) + const original = await agentPanel.attachmentChip(ASSET_NAME).boundingBox() + assert(original) + await agentPanel.composer.fill( + 'Keep this line and the attached video.\n'.repeat(40) + ) + await agentPanel.expectAttachmentFullyVisible(ASSET_NAME) + await expect + .poll( + async () => + (await agentPanel.attachmentChip(ASSET_NAME).boundingBox())?.height + ) + .toBe(original.height) + await expect(agentPanel.attachmentChips).toHaveCount(1) + await expect(agentPanel.composer).toContainText( + 'Keep this line and the attached video.' + ) + }) + }) + // A filename is user data and may contain a quote or a backslash, which would // break the attribute selector the chip is matched by. Exercised end to end // rather than unit-tested, so a regression in the escaping shows up as the diff --git a/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts b/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts index 27d4ac49398..c5d61db5362 100644 --- a/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts +++ b/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts @@ -60,9 +60,10 @@ test.describe( const removeButton = panel.getByRole('button', { name: `Remove KSampler #${node.id} reference` }) - await expect(removeButton).toBeVisible() - await expect(placeholder).toHaveCount(0) + await expect(placeholder).toBeHidden() + await panel.getByTestId('node-reference-chip').hover() + await expect(removeButton).toBeVisible() await removeButton.click() await expect(removeButton).toHaveCount(0) diff --git a/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts b/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts new file mode 100644 index 00000000000..dd273091746 --- /dev/null +++ b/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts @@ -0,0 +1,101 @@ +import { expect } from '@playwright/test' + +import { bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { inlineReferencesTest as test } from '@e2e/fixtures/agentInlineReferencesFixture' +import { DragDropHelper } from '@e2e/fixtures/helpers/DragDropHelper' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test( + 'keeps repeated file drops in one tray item while pending and after uploading', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.composer.fill('Keep this draft') + const drops = new DragDropHelper(page) + const file = assetPath('test_upload_image.png') + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect( + panel.root.getByRole('status', { name: 'Uploading' }) + ).toBeVisible() + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + const notice = page + .getByRole('alert') + .filter({ hasText: 'test_upload_image.png is already in the asset tray' }) + await expect(notice).toHaveCount(1) + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + await notice.getByRole('button', { name: 'Close' }).click() + assetUpload.finish() + await expect( + panel.root.getByRole('status', { name: 'Uploading' }) + ).toHaveCount(0) + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect(notice).toHaveCount(1) + await notice.getByRole('button', { name: 'Close' }).click() + await panel.composer.press('End') + await panel.composer.pressSequentially(' @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .click() + await panel.composer.pressSequentially(' @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .click() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(2) + await panel.attachmentChip('test_upload_image.png').hover() + await panel + .attachmentChip('test_upload_image.png') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + .click() + await expect(panel.attachmentChips).toHaveCount(0) + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect(panel.composer).toContainText('Keep this draft') + } +) + +test( + 'shows one plural duplicate notice while continuing a mixed pending batch', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.composer.fill('Keep this draft') + const drops = new DragDropHelper(page) + const file = assetPath('test_upload_image.png') + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await drops.dropFilesOn(panel.root, [file, file, assetPath('silence.wav')]) + await expect(panel.attachmentChips).toHaveCount(2) + await expect(panel.attachmentChip('test_upload_image.png')).toBeVisible() + await expect(panel.attachmentChip('silence.wav')).toBeVisible() + await expect( + page.getByRole('alert').filter({ hasText: 'Skipped 2 duplicate assets' }) + ).toHaveCount(1) + await expect( + panel.root.getByRole('status', { name: 'Uploading', exact: true }) + ).toHaveCount(2) + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + assetUpload.finish() + await expect( + panel.root.getByRole('status', { name: 'Uploading', exact: true }) + ).toHaveCount(0) + await expect(panel.attachmentChips).toHaveCount(2) + } +) diff --git a/browser_tests/tests/agent/agentComposerTouch.spec.ts b/browser_tests/tests/agent/agentComposerTouch.spec.ts new file mode 100644 index 00000000000..67806570a3f --- /dev/null +++ b/browser_tests/tests/agent/agentComposerTouch.spec.ts @@ -0,0 +1,67 @@ +import { expect } from '@playwright/test' + +import { bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { inlineReferencesTest as test } from '@e2e/fixtures/agentInlineReferencesFixture' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.use({ hasTouch: true, viewport: { width: 1024, height: 768 } }) + +test( + 'discloses filenames and removes a staged asset without hover', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.fileInput.setInputFiles(assetPath('test_upload_image.png')) + await panel.fileInput.setInputFiles({ + name: 'notes.txt', + mimeType: 'text/plain', + buffer: Buffer.from('Notes') + }) + assetUpload.finish() + await expect(panel.attachmentChips).toHaveCount(2) + await panel.composer.fill('Keep this draft @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .tap() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) + await panel + .attachmentChip('notes.txt') + .getByRole('button', { name: 'Preview notes.txt', exact: true }) + .tap() + const preview = page.getByRole('dialog', { name: 'notes.txt', exact: true }) + await expect(preview.getByText('notes.txt', { exact: true })).toBeVisible() + await panel.previewAssetButton('test_upload_image.png').tap() + await expect(panel.assetPreview('test_upload_image.png')).toBeVisible() + await expect(preview).toHaveCount(0) + await panel.previewAssetButton('notes.txt').tap() + await expect(preview).toBeVisible() + await expect(panel.assetPreview('test_upload_image.png')).toHaveCount(0) + await panel.composer.tap() + await expect(preview).toHaveCount(0) + const remove = panel + .attachmentChip('test_upload_image.png') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + await expect + .poll(() => + remove.evaluate((element) => + element.checkVisibility({ checkOpacity: true }) + ) + ) + .toBe(true) + await remove.tap() + await expect(panel.attachmentChip('test_upload_image.png')).toHaveCount(0) + await expect(panel.attachmentChip('notes.txt')).toBeVisible() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await expect(panel.composer).toContainText('Keep this draft') + } +) diff --git a/browser_tests/tests/agent/agentInlineReferences.spec.ts b/browser_tests/tests/agent/agentInlineReferences.spec.ts index b5d0181a6cf..e637a51b56e 100644 --- a/browser_tests/tests/agent/agentInlineReferences.spec.ts +++ b/browser_tests/tests/agent/agentInlineReferences.spec.ts @@ -19,7 +19,7 @@ test.use({ }) test( - 'keeps inline node and asset removal synchronized with the upper rows and Undo', + 'keeps asset attachments in the tray while inline references follow prompt editing and Undo', { tag: ['@cloud', '@ui'] }, async ({ page, workflowSelection, assetUpload }) => { await page.locator('#comfy-file-input').setInputFiles({ @@ -45,39 +45,50 @@ test( await panel .getByTestId('agent-file-input') .setInputFiles(assetPath('test_upload_image.png')) - const text = 'Adjust Color balance #12 to match test_upload_image.png ' - await expect(editor).toHaveText(text) + const tray = panel.getByTestId('composer-asset-section') + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await expect(tray).not.toContainText('test_upload_image.png') + await expect(editor).toHaveText('Adjust Color balance #12 to match ') + await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await expect( panel.getByRole('button', { name: enMessages.agent.send, exact: true }) ).toBeDisabled() + await editor.pressSequentially('@test_upload') await panel - .getByTestId('composer-asset-section') - .getByRole('button', { name: enMessages.agent.remove, exact: true }) + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) .click() - assetUpload.finish() - await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) - await editor.press('ControlOrMeta+z') + const text = 'Adjust Color balance #12 to match test_upload_image.png ' await expect(editor).toHaveText(text) + assetUpload.finish() await expect( panel.getByRole('button', { name: enMessages.agent.send, exact: true }) ).toBeEnabled() + await editor.press('Backspace') + await editor.press('Backspace') + await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await editor.press('ControlOrMeta+z') + await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(1) + await editor.getByTestId('node-reference-chip').hover() await panel .getByRole('button', { name: 'Remove Color balance #12 reference' }) .click() - await expect(editor).toHaveText('Adjust to match test_upload_image.png ') + await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await editor.press('ControlOrMeta+z') - await expect(editor).toHaveText(text) - await expect(panel.getByTestId('composer-node-section')).toHaveText( - 'Color balance' - ) + await expect(editor.getByTestId('node-reference-chip')).toHaveCount(1) + await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await editor.press('ControlOrMeta+a') await editor.press('ControlOrMeta+c') await expect .poll(() => page.evaluate(() => navigator.clipboard.readText())) .toBe( - 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png] ' + 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png]' ) await editor.press('ArrowRight') await page @@ -95,15 +106,37 @@ test( await expect.poll(() => workflowSelection.savedPaths.length).toBe(2) workflowSelection.finishSave(true) await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) + await editor.pressSequentially(' @') + await panel + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + panel.getByText(enMessages.agent.noNodesToReference, { exact: true }) + ).toBeVisible() + await expect( + panel.getByRole('menuitem', { name: /Color balance/ }) + ).toHaveCount(0) + await editor.press('Escape') await editor.press('ControlOrMeta+a') await editor.press('Backspace') await editor.press('ControlOrMeta+v') await expect(editor).toHaveText( - 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png] ' + 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png]' ) await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await tray.getByRole('group', { name: 'test_upload_image.png' }).hover() + await panel + .getByTestId('composer-asset-section') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + .click() await expect(panel.getByTestId('composer-asset-section')).toHaveCount(0) await expect( panel.getByRole('button', { name: enMessages.agent.switchWorkflow }) diff --git a/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts b/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts new file mode 100644 index 00000000000..bdef27d587f --- /dev/null +++ b/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts @@ -0,0 +1,108 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { zComfyWorkflow } from '@/platform/workflow/validation/schemas/workflowSchema' +import workflow from '@e2e/assets/nodes/single_ksampler.json' with { type: 'json' } +import { + agentTest as test, + bootAgentApp, + loadIntoBootWorkflow, + mockWorkflowPersistence +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { Topbar } from '@e2e/fixtures/components/Topbar' +import { agentReplayNodeDefs } from '@e2e/fixtures/data/agentReplayNodeDefs' +import { nextFrame } from '@e2e/fixtures/utils/timing' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' + +test.describe( + 'Node mention freshness (FE-3220)', + { tag: ['@cloud', '@vue-nodes'] }, + () => { + test.beforeEach(async ({ page, agentFlagEnabled }) => { + await bootAgentApp(page, agentFlagEnabled, { + vueNodes: true, + objectInfo: { KSampler: agentReplayNodeDefs.KSampler }, + beforeNavigate: (page) => + mockWorkflowPersistence(page, 'a81718a4-02ae-41e6-ae85-c33b7bb880f6') + }) + await loadIntoBootWorkflow(page, zComfyWorkflow.parse(workflow)) + const agentPanel = new AgentPanel(page) + await agentPanel.open() + await agentPanel.selectWorkflow() + }) + + test('refreshes nodes after renaming, deleting and undoing in the target workflow', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + const vueNodes = new VueNodeHelpers(page) + await agentPanel.composer.fill('Keep this draft @') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByRole('menuitem', { name: 'KSampler', exact: true }) + ).toBeVisible() + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await vueNodes.renameNode('3', 'Color grade') + await nextFrame(page) + await expect( + vueNodes.getNodeLocator('3').getByTestId('node-title') + ).toHaveText('Color grade') + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByRole('menuitem', { + name: 'Color grade', + exact: true + }) + ).toBeVisible() + await expect( + agentPanel.root.getByRole('menuitem', { name: 'KSampler', exact: true }) + ).toHaveCount(0) + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await vueNodes.deleteNode('3') + await nextFrame(page) + await expect(vueNodes.getNodeLocator('3')).toHaveCount(0) + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByText(enMessages.agent.noNodesToReference, { + exact: true + }) + ).toBeVisible() + await expect( + agentPanel.root.getByRole('menuitem', { + name: 'Color grade', + exact: true + }) + ).toHaveCount(0) + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await new Topbar(page).triggerTopbarCommand(['Edit', 'Undo']) + await nextFrame(page) + await expect( + vueNodes.getNodeLocator('3').getByTestId('node-title') + ).toHaveText('Color grade') + await expect(agentPanel.composer).toHaveText('Keep this draft ') + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await agentPanel.root + .getByRole('menuitem', { name: 'Color grade', exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('node-reference-chip') + ).toHaveText('Color grade #3') + await expect(agentPanel.composer).toContainText('Keep this draft') + }) + } +) diff --git a/browser_tests/tests/agent/agentNodeSelectionChipRemoval.spec.ts b/browser_tests/tests/agent/agentNodeSelectionChipRemoval.spec.ts index 3b93788ac9d..c721b1e7c16 100644 --- a/browser_tests/tests/agent/agentNodeSelectionChipRemoval.spec.ts +++ b/browser_tests/tests/agent/agentNodeSelectionChipRemoval.spec.ts @@ -43,11 +43,10 @@ test.describe( y: y + height / 2 }) - await expect( - panel.getByRole('button', { - name: `Remove KSampler #${node.id} reference` - }) - ).toBeVisible() + await expect(panel.getByTestId('node-reference-chip')).toBeVisible() + await expect + .poll(() => comfyPage.nodeOps.getSelectedNodeIds()) + .toEqual([node.id]) }) const removeButton = panel.getByRole('button', { @@ -55,8 +54,13 @@ test.describe( }) await test.step('Remove the reference and clear its highlight', async () => { + await panel.getByTestId('node-reference-chip').hover() + await expect(removeButton).toBeVisible() await removeButton.click() await expect(removeButton).toHaveCount(0) + await expect + .poll(() => comfyPage.nodeOps.getSelectedNodeIds()) + .toEqual([]) await expect .poll(async () => { const canvas = await comfyPage.canvas.screenshot({ diff --git a/browser_tests/tests/agent/agentPanel.spec.ts b/browser_tests/tests/agent/agentPanel.spec.ts index e549b58cd28..6afc2231fb0 100644 --- a/browser_tests/tests/agent/agentPanel.spec.ts +++ b/browser_tests/tests/agent/agentPanel.spec.ts @@ -621,7 +621,9 @@ test.describe('In-App Agent panel', { tag: '@cloud' }, () => { }) expect((await uploadResponse).ok()).toBe(true) await expect( - panel.getByTestId('composer-asset-section').getByText('movie.mp4') + panel + .getByTestId('composer-asset-section') + .getByRole('group', { name: 'movie.mp4' }) ).toBeVisible() await expect.poll(() => uploadCount).toBe(1) @@ -633,9 +635,9 @@ test.describe('In-App Agent panel', { tag: '@cloud' }, () => { await expect( page.getByText('too-large.mp4 is larger than 24 MB') ).toBeVisible() - await expect(panel.getByText('too-large.mp4', { exact: true })).toHaveCount( - 0 - ) + await expect( + panel.getByRole('group', { name: 'too-large.mp4' }) + ).toHaveCount(0) await expect.poll(() => uploadCount).toBe(1) }) diff --git a/browser_tests/tests/agent/agentUploadedImageAttachment.spec.ts b/browser_tests/tests/agent/agentUploadedImageAttachment.spec.ts index 391b8691053..018a967dfe5 100644 --- a/browser_tests/tests/agent/agentUploadedImageAttachment.spec.ts +++ b/browser_tests/tests/agent/agentUploadedImageAttachment.spec.ts @@ -47,7 +47,12 @@ test( await panel .getByTestId('agent-file-input') .setInputFiles(assetPath('image32x32.webp')) - await expect(panel.getByTestId('asset-reference-chip')).toHaveCount(1) + await expect( + panel + .getByTestId('composer-asset-section') + .getByRole('group', { name: 'image32x32.webp', exact: true }) + ).toBeVisible() + await expect(panel.getByTestId('asset-reference-chip')).toHaveCount(0) await panel .getByRole('textbox', { name: /^Describe ideas/ }) .pressSequentially('use this upload') diff --git a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md new file mode 100644 index 00000000000..203dbcd3ca6 --- /dev/null +++ b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md @@ -0,0 +1,53 @@ +# ADR-AGENT-COMPOSER-0040: Independent Assets and Prompt References + +Date: 2026-10-05 + +## Status + +Proposed + +## Context + +An asset added to the agent composer must be sent even without an inline +mention. Users may mention one asset repeatedly, delete or undo an occurrence, +and continue typing while an upload or send completes. Deriving attachments +from prompt references makes deletion remove the asset from the request and +makes repeated mentions duplicate it. + +## Decision + +The existing composer store owns the ordered included asset IDs and asset +metadata. Prompt references identify occurrences at text positions. Adding an +asset stages it; mentioning an included asset inserts an occurrence. Editor +history may restore occurrences only for assets still included in the tray. +Removing a tray item retires its identity and removes every occurrence; an +explicit new drop gets a fresh identity. + +Send snapshots retain both the included assets and the prompt. An untouched +failed draft restores both; newer user input wins. Local preview resources +remain owned by the store while included or held by a pending snapshot, and are +released when no longer retained. UI hover/focus highlighting and native scroll +measurements remain local to components and never change the draft. + +Playable media URLs and image poster URLs are distinct metadata. The store owns +one cancellable video-thumbnail job per asset/source and publishes its result to +the tray and all prompt occurrences. UI surfaces render that poster; only tray +hover mounts a player. A source change, removal or disposal cancels obsolete +capture. Pending snapshots retain media resources and failed recovery preserves +a poster completed for the same source while the send was pending. + +## Alternatives considered + +- Continue deriving attachments from references: cannot express an unmentioned + included asset or repeated references without changing send semantics. +- Keep a separate tray in a component: loses assets across remount and makes + upload completion, Undo and failed-send recovery depend on mounted UI. +- Create another persistent asset store: duplicates the existing composer's + submission lifecycle without a separate sharing requirement. + +## Consequences + +Attachment metadata is resolved through one owner when applying editor history. +Removing an inline occurrence keeps the included file. Send payloads contain +one entry per included asset while preserving every inline mention. Callers +stage and reference assets through distinct store commands. diff --git a/docs/adr/README.md b/docs/adr/README.md index 64f74e27647..b1b0dd80f78 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -11,6 +11,7 @@ An Architecture Decision Record captures an important architectural decision mad | ADR | Title | Status | Date | | ---------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | ---------- | ---------- | | [AGENT-BINDING-0035](AGENT-BINDING-0035-document-identity-gates-persisted-workflow-tab-bindings.md) | Document Identity Gates Persisted Workflow Tab Bindings | Proposed | 2026-09-19 | +| [AGENT-COMPOSER-0040](AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md) | Independent Assets and Prompt References | Proposed | 2026-10-05 | | [AGENT-CONTEXT-0028](AGENT-CONTEXT-0028-separate-workflow-references-from-editor-tabs.md) | Separate Workflow References from Editor Tabs | Proposed | 2026-09-07 | | [AGENT-HISTORY-0038](AGENT-HISTORY-0038-wait-for-chat-and-workflow-readiness.md) | Wait for Chat and Workflow Readiness | Proposed | 2026-09-25 | | [AGENT-OBSERVABILITY-0034](AGENT-OBSERVABILITY-0034-frontend-journey-effect-contract.md) | Frontend Journey Effect Contract | Proposed | 2026-09-17 | diff --git a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts index eaa3189970a..052a3a34373 100644 --- a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts +++ b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts @@ -230,7 +230,9 @@ describe('AssetsSidebarListView', () => { display_name: 'Clip', attachment_ref: 'clip.mp4', media_kind: 'video', - preview_url: undefined + preview_url: undefined, + media_url: + 'http://localhost:3000/api/view?filename=clip.mp4&type=output&subfolder=' }), MIME_ASSET_INFO ) diff --git a/src/locales/en/main.json b/src/locales/en/main.json index e300434f899..7a855c370d2 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5544,6 +5544,7 @@ "failedDetail": "Your onboarding state could not be reset. Please try again." }, "agent": { + "previewAsset": "Preview {name}", "title": "Comfy Agent", "diagnosticReport": { "retry": "Retry copy report", @@ -5641,6 +5642,7 @@ "assetNotAttachable": "Images, video, audio, 3D models, and text files can be attached", "attachmentTooLarge": "{name} is larger than {limit}", "attachmentUploadFailed": "{name} could not be uploaded", + "attachmentsAlreadyAdded": "{name} is already in the asset tray | Skipped {count} duplicate assets", "mention": "Mention a node", "changeWorkflowForChat": "Change the workflow that the agent can edit", "chatOptions": "Chat options", @@ -5690,7 +5692,11 @@ "focusNode": "Show on canvas", "focusNodeLabel": "Show {node} on canvas", "removeNodeLabel": "Remove {node} reference", + "removeAssetReference": "Remove {name} reference", + "removeAsset": "Remove {name}", + "previewAsset": "Preview {name}", "uploading": "Uploading", + "uploadingAttachments": "Uploading {count} attachment | Uploading {count} attachments", "noNodesToMention": "No nodes in this workflow", "friend": "there", "sendFailed": "Message failed to send", diff --git a/src/platform/assets/schemas/mediaAssetSchema.ts b/src/platform/assets/schemas/mediaAssetSchema.ts index deab1462d8c..a65a000d542 100644 --- a/src/platform/assets/schemas/mediaAssetSchema.ts +++ b/src/platform/assets/schemas/mediaAssetSchema.ts @@ -5,7 +5,7 @@ import { zResultItem } from '@/platform/remote/comfyui/execution/types' import { assetItemSchema } from './assetSchema' -const zMediaKindSchema = z.enum([ +export const zMediaKindSchema = z.enum([ 'video', 'audio', 'image', @@ -56,7 +56,8 @@ const zDraggedAssetInfo = zResultItem.and( z.object({ attachment_ref: z.string().min(1).optional(), media_kind: zMediaKindSchema.optional(), - preview_url: z.string().url().optional() + preview_url: z.string().url().optional(), + media_url: z.string().url().optional() }) ) diff --git a/src/platform/assets/utils/assetDragUtil.test.ts b/src/platform/assets/utils/assetDragUtil.test.ts new file mode 100644 index 00000000000..76004b96598 --- /dev/null +++ b/src/platform/assets/utils/assetDragUtil.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it, vi } from 'vitest' + +import { useFeatureFlags } from '@/composables/useFeatureFlags' +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { api } from '@/scripts/api' +import { getDroppedAsset } from '@/utils/eventUtils' + +import { startAssetDrag } from './assetDragUtil' + +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@/scripts/api')) + +function drag(asset: AssetItem) { + const dataTransfer = new DataTransfer() + const event = new DragEvent('dragstart', { cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return getDroppedAsset(dataTransfer) +} + +describe('asset drag media sources', () => { + it.for([ + { + label: 'server poster', + assetsEnabled: true, + preview_id: 'poster-id', + thumbnail_url: undefined, + preview_url: undefined, + poster: 'http://localhost:8188/api/assets/poster-id/content', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'distinct thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '/view?filename=poster.png', + preview_url: '/view?filename=clip.mp4', + poster: 'http://localhost:8188/api/view?filename=poster.png', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'absolute CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: 'https://cdn.example/thumb.webp?signature=poster', + preview_url: '/view?filename=clip.mp4', + poster: 'https://cdn.example/thumb.webp?signature=poster', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'protocol-relative CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '//cdn.example/thumb.webp', + preview_url: '/view?filename=clip.mp4', + poster: 'http://cdn.example/thumb.webp', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'history video without a poster', + assetsEnabled: false, + preview_id: undefined, + thumbnail_url: '/view?filename=clip.mp4', + preview_url: '/view?filename=clip.mp4', + poster: undefined, + media: 'http://localhost:3000/view?filename=clip.mp4' + }, + { + label: 'assets API video without a poster', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: undefined, + preview_url: undefined, + poster: undefined, + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + } + ])( + 'preserves distinct file/image sources for $label', + ({ + assetsEnabled, + preview_id, + thumbnail_url, + preview_url, + poster, + media + }) => { + vi.mocked(useFeatureFlags().flags).assetsEnabled = assetsEnabled + vi.mocked(api.apiURL).mockImplementation( + (path) => `http://localhost:8188/api${path}` + ) + const asset = { + id: 'video-id', + name: 'clip.mp4', + display_name: 'My clip', + tags: ['input'], + created_at: '2026-10-05T00:00:00Z', + updated_at: '2026-10-05T00:00:00Z', + preview_id, + thumbnail_url, + preview_url + } satisfies AssetItem + expect(drag(asset)).toMatchObject({ + name: 'My clip', + kind: 'video', + mediaUrl: media, + previewUrl: poster + }) + } + ) +}) diff --git a/src/platform/assets/utils/assetDragUtil.ts b/src/platform/assets/utils/assetDragUtil.ts index 9487c359402..4a4623dfcfc 100644 --- a/src/platform/assets/utils/assetDragUtil.ts +++ b/src/platform/assets/utils/assetDragUtil.ts @@ -9,6 +9,7 @@ * row that starts a drag carrying nothing. */ import { getMediaTypeFromFilename } from '@/utils/formatUtil' +import { api } from '@/scripts/api' import { getAssetType } from '../composables/media/assetMappers' import { getOutputAssetMetadata } from '../schemas/assetMetadataSchema' @@ -18,6 +19,39 @@ import { getAssetUrlFilename } from './assetMetadataUtils' import { resolvePreviewUrl } from './assetPreviewUtil' import { getAssetFileUrl } from './assetUrlUtil' +function resolveThumbnailUrl(url: string): URL | null { + const source = + URL.canParse(url) || url.startsWith('//') ? url : api.apiURL(url) + return URL.parse(source, location.href) +} + +function assetMediaSources(asset: AssetItem) { + const mediaKind = getMediaTypeFromFilename(asset.name) + const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) + const playable = mediaKind === 'video' || mediaKind === 'audio' + const mediaUrl = playable + ? URL.parse( + getAssetFileUrl(asset, { disposition: 'inline' }), + location.href + ) + : undefined + const posterUrl = asset.preview_id + ? previewUrl + : asset.thumbnail_url && asset.thumbnail_url !== asset.preview_url + ? resolveThumbnailUrl(asset.thumbnail_url) + : undefined + return { + media_kind: mediaKind, + preview_url: + mediaKind === 'image' + ? previewUrl?.toString() + : mediaKind === 'video' + ? posterUrl?.toString() + : undefined, + media_url: mediaUrl?.toString() + } +} + /** * Start a native drag carrying `asset`. * @@ -43,9 +77,6 @@ export function startAssetDrag( if (!dataTransfer) return const output = getOutputAssetMetadata(asset.user_metadata)?.allOutputs?.[0] - const mediaKind = getMediaTypeFromFilename(asset.name) - const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) - const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) const assetInfo = { ...(output?.filename ? { @@ -57,14 +88,14 @@ export function startAssetDrag( : { filename: asset.name, type: getAssetType(asset.tags), - display_name: asset.display_name + display_name: asset.display_name ?? undefined }), attachment_ref: getAssetUrlFilename(asset), - media_kind: mediaKind, - preview_url: mediaKind === 'image' ? previewUrl?.toString() : undefined + ...assetMediaSources(asset) } dataTransfer.setData(MIME_ASSET_INFO, JSON.stringify(assetInfo)) + const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) if (!fileUrl) return dataTransfer.setData('text/uri-list', fileUrl.toString()) diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts index 3dc5be6e58a..af1822bd411 100644 --- a/src/utils/eventUtils.ts +++ b/src/utils/eventUtils.ts @@ -7,6 +7,7 @@ export interface DroppedAsset { ref?: string kind?: MediaKind previewUrl?: string + mediaUrl?: string } export function getDroppedAsset( @@ -27,7 +28,8 @@ export function getDroppedAsset( uri, ref, kind: asset?.media_kind, - previewUrl: asset?.preview_url + previewUrl: asset?.preview_url, + mediaUrl: asset?.media_url } : undefined } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index e7c69e26700..14f49056a3b 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -54,6 +54,8 @@ import { useWorkflowTabActivityStore } from '@/stores/workflowTabActivityStore' import { useSidebarTabStore } from '@/stores/workspace/sidebarTabStore' import { useToastStore } from '@/platform/updates/common/toastStore' import { useAssetsStore } from '@/stores/assetsStore' +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { startAssetDrag } from '@/platform/assets/utils/assetDragUtil' import { getFilenameDetails } from '@/utils/formatUtil' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import { StorageKeys } from '@/platform/workflow/persistence/base/storageKeys' @@ -2122,6 +2124,30 @@ function fileOfSize(name: string, size: number, type: string): File { return file } +function assetPanelDrag( + displayName: string | null, + overrides: Partial = {} +) { + const asset = { + id: 'library-source', + name: 'library.png', + hash: 'stored-library.png', + display_name: displayName, + tags: ['input'], + created_at: '2026-10-03T00:00:00Z', + updated_at: '2026-10-03T00:00:00Z', + ...overrides + } satisfies AssetItem + const dataTransfer = new DataTransfer() + const event = new DragEvent('dragstart', { cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return { + types: Array.from(dataTransfer.types), + getData: (type: string) => dataTransfer.getData(type) + } +} + // DES-527 replaced the paperclip and @ buttons with a single + menu, so every // attach or node-mention gesture now starts by opening it. async function openAddMenu(): Promise { @@ -2179,6 +2205,7 @@ function setupNodeSelectionCanvas() { selectItems, deselect, deselectAll, + setDirty: vi.fn(), animateToBounds: vi.fn(), canvas: canvasElement, ds: createTestDragAndScale(900, 700) @@ -2367,6 +2394,215 @@ describe('AgentPanelRoot attach flow', () => { useAgentWorkflowTabBindingStore().bind('wf-42', tab.path) }) + it.for([ + { displayName: null, label: 'library.png' }, + { displayName: 'My library image', label: 'My library image' } + ])( + 'stages an assets-panel drop with display name $displayName from the real producer payload', + async ({ displayName, label }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep this draft') + await nextTick() + const target = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) + const data = assetPanelDrag(displayName) + + expect(dispatchDrag(target, 'dragover', data)).toBe(true) + expect(dispatchDrag(target, 'drop', data)).toBe(true) + expect( + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: label } + ) + ).toBeInTheDocument() + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: label, + ref: 'stored-library.png', + previewUrl: 'http://localhost:3000/api/assets/library-source/content' + }) + ]) + expect(store.prompt).toEqual({ text: 'Keep this draft', references: [] }) + expect(uploaded).toEqual([]) + } + ) + + it.for([ + { + filename: 'clip.mp4', + label: 'My clip', + kind: 'video', + previewId: 'poster', + previewUrl: 'http://localhost:3000/api/assets/poster/content', + indicatorLabel: 'My clip', + indicatorSource: 'http://localhost:3000/api/assets/poster/content' + }, + { + filename: 'song.mp3', + label: 'My recording', + kind: 'audio', + previewId: undefined, + previewUrl: undefined, + indicatorLabel: 'Audio', + indicatorSource: null + } + ])( + 'preserves playable $kind metadata from the assets panel through the chat tray', + async ({ + filename, + label, + kind, + previewId, + previewUrl, + indicatorLabel, + indicatorSource + }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep typing') + await nextTick() + const data = assetPanelDrag(label, { + name: filename, + hash: `stored-${filename}`, + preview_id: previewId + }) + expect(dispatchDrag(screen.getByRole('textbox'), 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: label }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: label, + mediaKind: kind, + previewUrl, + mediaUrl: + 'http://localhost:3000/api/assets/library-source/content?disposition=inline' + }) + ]) + expect( + within(trayItem) + .getByRole('img', { name: indicatorLabel }) + .getAttribute('src') + ).toBe(indicatorSource) + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(uploaded).toEqual([]) + } + ) + + it('deduplicates an assets-panel item that needs fetching to identify its media type', async () => { + const data = assetPanelDrag('Recording.mp3', { name: 'recording' }) + expect( + JSON.parse(data.getData('application/x-comfy-asset-info')) + ).toMatchObject({ media_kind: 'other' }) + const source = data.getData('text/uri-list') + const fetched: string[] = [] + vi.stubGlobal( + 'fetch', + vi.fn(async (input, init) => { + if (String(input) === source) { + fetched.push(source) + return new Response(new Blob(['audio'], { type: 'audio/mpeg' })) + } + if (init?.method === 'POST') return json(200, { name: 'recording.mp3' }) + if (String(input).includes('/assets')) + return json(200, { assets: [], total: 0, has_more: false }) + return json(200, agentThreadList()) + }) + ) + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + dispatchDrag(target, 'drop', data) + await vi.waitFor(() => expect(store.attachments[0]?.uploading).toBe(false)) + dispatchDrag(target, 'drop', data) + await nextTick() + expect(store.attachments).toHaveLength(1) + expect(fetched).toEqual([source]) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ + severity: 'info', + detail: 'Recording.mp3 is already in the asset tray' + }) + ]) + }) + + it('allows a removed library asset to be explicitly dropped again while retaining retirement of its old identity', async () => { + stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + const data = assetPanelDrag('library.png') + expect(dispatchDrag(target, 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: 'library.png' }) + const oldAttachment = store.attachments[0] + assert.exists(oldAttachment) + store.referenceAttachment(oldAttachment.id) + const oldPrompt = store.prompt + await userEvent.hover(trayItem) + await userEvent.click( + within(trayItem).getByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'library.png' }) + }) + ) + expect(store.attachments).toEqual([]) + + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + const newAttachment = store.attachments[0] + assert.exists(newAttachment) + expect(newAttachment.id).not.toBe(oldAttachment.id) + store.updateAttachment(oldAttachment.id, { ref: 'too-late.png' }) + store.applyEditorPrompt(oldPrompt) + expect(store.attachments).toEqual([newAttachment]) + expect(newAttachment.ref).toBe('stored-library.png') + expect(store.prompt.references).toEqual([]) + expect(useToastStore().messagesToAdd).toEqual([]) + expect(dispatchDrag(target, 'drop', data)).toBe(true) + expect(store.attachments).toHaveLength(1) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ + severity: 'info', + detail: 'library.png is already in the asset tray' + }) + ]) + }) + + it('allows the same library asset to be dropped into a new draft after sending', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => + init?.method === 'POST' + ? json(202, { thread_id: 'th-1', message_id: 'm-1' }) + : json(200, agentThreadList()) + ) + ) + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + const data = assetPanelDrag('library.png') + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + const previousId = store.attachments[0]?.id + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(store.attachments).toEqual([]) + + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: 'library.png', + ref: 'stored-library.png' + }) + ]) + expect(store.attachments[0]?.id).not.toBe(previousId) + expect(store.prompt.references).toEqual([]) + }) + it('uploads a picked file, stages its ref, and forwards it on the next send', async () => { const messageBodies: unknown[] = [] const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { @@ -2402,8 +2638,9 @@ describe('AgentPanelRoot attach flow', () => { await userEvent.upload(input, file) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() @@ -2413,7 +2650,7 @@ describe('AgentPanelRoot attach flow', () => { expect(messageBodies).toHaveLength(1) expect(messageBodies[0]).toMatchObject({ - content: '@[Image: cat.png] make it pop', + content: 'make it pop', attachments: ['uploaded_cat.png'] }) expect(useTelemetry()!.trackAgentMessageSent).toHaveBeenCalledWith({ @@ -2520,8 +2757,9 @@ describe('AgentPanelRoot attach flow', () => { ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'movie.mp4' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'movie.mp4' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['movie.mp4'])) @@ -2603,7 +2841,9 @@ describe('AgentPanelRoot attach flow', () => { detail: 'movie.mp4 is larger than 24 MB' }) ) - expect(screen.queryByText('movie.mp4')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'movie.mp4' }) + ).not.toBeInTheDocument() expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() }) @@ -2617,8 +2857,9 @@ describe('AgentPanelRoot attach flow', () => { dispatchDrag(screen.getByRole('textbox'), 'drop', { files: [image] }) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'huge.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'huge.png' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['huge.png'])) @@ -2636,8 +2877,9 @@ describe('AgentPanelRoot attach flow', () => { ).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'movie.mp4' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'movie.mp4' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['movie.mp4'])) @@ -2668,8 +2910,9 @@ describe('AgentPanelRoot attach flow', () => { ).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - name + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: name } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual([name])) @@ -2691,8 +2934,9 @@ describe('AgentPanelRoot attach flow', () => { await userEvent.paste(clipboard) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'image.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'image.png' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['image.png'])) @@ -2712,8 +2956,9 @@ describe('AgentPanelRoot attach flow', () => { await userEvent.paste(clipboard) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'image.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'image.png' } ) ).toBeInTheDocument() expect(screen.getByRole('textbox')).toHaveTextContent( @@ -2898,7 +3143,7 @@ describe('AgentPanelRoot attach flow', () => { await vi.waitFor(() => expect(refresh).toHaveBeenCalledTimes(2)) }) - it('lets a removed upload finish without reattaching until Undo', async () => { + it('keeps a removed tray upload retired after completion and stale editor history', async () => { const signals: AbortSignal[] = [] let finishUpload: (response: Response) => void = () => {} const upload = new Promise((resolve) => { @@ -2927,7 +3172,9 @@ describe('AgentPanelRoot attach flow', () => { const prompt = composer.prompt await userEvent.click( - await screen.findByRole('button', { name: i18n.global.t('agent.remove') }) + await screen.findByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'cat.png' }) + }) ) expect(signals[0].aborted).toBe(false) @@ -2937,13 +3184,7 @@ describe('AgentPanelRoot attach flow', () => { await vi.waitFor(() => expect(refresh).toHaveBeenCalledOnce()) expect(composer.attachments).toEqual([]) composer.applyEditorPrompt(prompt) - expect(composer.attachments).toEqual([ - expect.objectContaining({ - name: 'cat.png', - ref: 'uploaded-cat.png', - uploading: false - }) - ]) + expect(composer.attachments).toEqual([]) }) it('uses the server limit for audio rejection copy', async () => { @@ -2979,26 +3220,58 @@ describe('AgentPanelRoot attach flow', () => { ) }) - it('shows the asset drop target during a trusted drag and clears it on leave', async () => { - stubUploadFetch() - renderWithSelectedTarget() - await nextTick() - const target = screen.getByRole('textbox') - const data = { - types: ['application/x-comfy-asset-info', 'text/uri-list'] + it.for([ + { tray: 'empty', attachments: [] }, + { + tray: 'with an asset', + attachments: [{ id: 'kept', name: 'kept.png', ref: 'kept.png' }] } + ])( + 'shows panel-wide drop feedback across child transitions while preserving the composer (tray=$tray)', + async ({ attachments }) => { + stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.replaceDraft({ + text: 'Keep this draft', + workflowReferences: [], + attachments + }) + await nextTick() + const prompt = store.prompt + const header = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) + const composer = screen.getByTestId('composer-input-box') + const data = { + types: ['application/x-comfy-asset-info', 'text/uri-list'] + } + expect(screen.queryByRole('status')).not.toBeInTheDocument() - dispatchDrag(target, 'dragenter', data) - await nextTick() + dispatchDrag(header, 'dragenter', data) + await nextTick() - const dropTarget = screen.getByRole('status') - expect(dropTarget).toHaveTextContent('Drag and drop assets here') + const dropTarget = screen.getByRole('status') + expect(dropTarget).toHaveTextContent('Drag and drop assets here') + expect(composer).not.toContainElement(dropTarget) + expect(screen.getByRole('textbox')).toHaveTextContent('Keep this draft') + expect(store.prompt).toEqual(prompt) + expect(screen.queryAllByRole('group', { name: 'kept.png' })).toHaveLength( + attachments.length + ) - dispatchDrag(dropTarget, 'dragleave', data) - await nextTick() + dispatchDrag(composer, 'dragenter', data) + dispatchDrag(header, 'dragleave', data) + await nextTick() + expect(screen.getByRole('status')).toBe(dropTarget) - expect(screen.queryByRole('status')).not.toBeInTheDocument() - }) + dispatchDrag(composer, 'dragleave', data) + await nextTick() + expect(screen.queryByRole('status')).not.toBeInTheDocument() + expect(store.prompt).toEqual(prompt) + expect(store.attachments).toEqual(attachments) + } + ) it('rejects URI-only drags without showing or claiming the asset target', async () => { const uploaded = stubUploadFetch() @@ -3078,8 +3351,9 @@ describe('AgentPanelRoot attach flow', () => { expect(dropTarget).not.toBeInTheDocument() expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - filename + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: filename } ) ).toBeInTheDocument() @@ -3089,7 +3363,7 @@ describe('AgentPanelRoot attach flow', () => { expect(messageBodies).toHaveLength(1) expect(messageBodies[0]).toMatchObject({ - content: `@[${mime === 'image/png' ? 'Image' : 'Video'}: ${filename}] describe this`, + content: 'describe this', attachments: [`uploaded_${filename}`] }) } @@ -3139,13 +3413,15 @@ describe('AgentPanelRoot attach flow', () => { expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - filename + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: filename } ) ).toBeInTheDocument() expect( - within(screen.getByTestId('composer-asset-section')).getAllByText( - filename + within(screen.getByTestId('composer-asset-section')).getAllByRole( + 'group', + { name: filename } ) ).toHaveLength(1) expect( @@ -3208,8 +3484,9 @@ describe('AgentPanelRoot attach flow', () => { expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'gen.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'gen.png' } ) ).toBeInTheDocument() expect( @@ -3273,23 +3550,77 @@ describe('AgentPanelRoot attach flow', () => { stubUploadFetch() renderWithSelectedTarget() await nextTick() - const target = screen.getByRole('textbox') + const target = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) const workflow = new File(['{}'], 'flow.json', { type: 'application/json' }) expect(dispatchDrag(target, 'drop', { files: [workflow] })).toBe(false) - expect(screen.queryByText('flow.json')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'flow.json' }) + ).not.toBeInTheDocument() const asset = new File(['x'], 'cat.png', { type: 'image/png' }) expect(dispatchDrag(target, 'drop', { files: [asset] })).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() }) + it.for([ + { copies: 1, notice: 'cat.png is already in the asset tray' }, + { copies: 2, notice: 'Skipped 2 duplicate assets' } + ])('reports $copies skipped files once', async ({ copies, notice }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + store.setText('Keep this draft') + const target = screen.getByRole('textbox') + await userEvent.click(target) + dispatchDrag(target, 'drop', { + files: [ + new File(['image'], 'cat.png', { type: 'image/png', lastModified: 1 }) + ] + }) + await vi.waitFor(() => expect(store.attachments[0]?.uploading).toBe(false)) + expect(useToastStore().messagesToAdd).toEqual([]) + dispatchDrag(target, 'drop', { + files: [ + ...Array.from( + { length: copies }, + () => + new File(['image'], 'cat.png', { + type: 'image/png', + lastModified: 1 + }) + ), + new File(['another image'], 'dog.png', { type: 'image/png' }) + ] + }) + await nextTick() + expect( + within(screen.getByTestId('composer-asset-section')).getAllByRole( + 'group', + { + name: 'cat.png' + } + ) + ).toHaveLength(1) + await vi.waitFor(() => expect(uploaded).toEqual(['cat.png', 'dog.png'])) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ severity: 'info', detail: notice }) + ]) + expect(screen.getByRole('group', { name: 'dog.png' })).toBeInTheDocument() + expect(target).toHaveFocus() + expect(store.prompt).toEqual({ text: 'Keep this draft', references: [] }) + }) + it('attaches only the assets out of a mixed drop', async () => { const uploaded = stubUploadFetch() renderWithSelectedTarget() @@ -3306,8 +3637,9 @@ describe('AgentPanelRoot attach flow', () => { dispatchDrag(screen.getByRole('textbox'), 'drop', { files }) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['cat.png'])) @@ -3361,7 +3693,9 @@ describe('AgentPanelRoot attach flow', () => { if (outcome === 'cancelled') { await userEvent.click( - screen.getByRole('button', { name: i18n.global.t('agent.remove') }) + screen.getByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'gen.png' }) + }) ) resolveAsset(new Response(new Blob(['asset'], { type: 'image/png' }))) } else { @@ -3410,7 +3744,9 @@ describe('AgentPanelRoot attach flow', () => { // The rejected upload's chip leaves; only a committed one stays settled. await vi.waitFor(() => - expect(screen.queryByText('bad.png')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'bad.png' }) + ).not.toBeInTheDocument() ) await vi.waitFor(() => expect( @@ -3423,8 +3759,9 @@ describe('AgentPanelRoot attach flow', () => { expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() } else { expect( - within(screen.getByTestId('composer-asset-section')).getByText( - 'good.png' + within(screen.getByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'good.png' } ) ).toBeInTheDocument() await vi.waitFor(() => @@ -3465,8 +3802,9 @@ describe('AgentPanelRoot attach flow', () => { ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() expect( @@ -3517,8 +3855,9 @@ describe('AgentPanelRoot attach flow', () => { screen.getByTestId('agent-file-input'), file ) - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) await sendFromComposer('second message') @@ -3593,8 +3932,9 @@ describe('AgentPanelRoot attach flow', () => { file ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() @@ -3602,7 +3942,9 @@ describe('AgentPanelRoot attach flow', () => { vi.mocked(reportError).mockClear() failUpload() await vi.waitFor(() => - expect(screen.queryByText('cat.png')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'cat.png' }) + ).not.toBeInTheDocument() ) expect(revoke).toHaveBeenCalledTimes(1) // A rejected file is the user's to fix; raising the server-error overlay @@ -3645,7 +3987,7 @@ describe('AgentPanelRoot attach flow', () => { revoke.mockRestore() }) - it('keeps a dismissed durable preview available for Undo until the editor unmounts', async () => { + it('keeps a durable tray preview through inline deletion, Undo and unmount', async () => { const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) vi.stubGlobal( 'fetch', @@ -3675,7 +4017,9 @@ describe('AgentPanelRoot attach flow', () => { file ) const assetSection = await screen.findByTestId('composer-asset-section') - expect(within(assetSection).getByText('cat.png')).toBeInTheDocument() + expect( + within(assetSection).getByRole('group', { name: 'cat.png' }) + ).toBeInTheDocument() await waitFor(() => expect( within(assetSection).getByRole('img', { name: 'cat.png' }) @@ -3683,10 +4027,17 @@ describe('AgentPanelRoot attach flow', () => { ) expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:mock-url') - await userEvent.click( - screen.getByRole('button', { name: i18n.global.t('agent.remove') }) - ) - expect(screen.queryByText('cat.png')).not.toBeInTheDocument() + const composer = useAgentComposerStore() + composer.referenceAttachment(composer.attachments[0].id) + await screen.findByTestId('asset-reference-chip') + await userEvent.click(screen.getByRole('textbox')) + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: 'cat.png' + }) + ).toBeVisible() screen.getByRole('textbox').focus() await userEvent.keyboard('{Control>}z{/Control}') expect( @@ -3695,6 +4046,9 @@ describe('AgentPanelRoot attach flow', () => { { name: 'cat.png' } ) ).toHaveAttribute('src', '/api/view?filename=uploaded_cat.png&type=input') + expect(screen.getByTestId('asset-reference-chip')).toHaveTextContent( + 'cat.png' + ) expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:mock-url') view.unmount() expect(revoke).toHaveBeenCalledTimes(1) @@ -9934,21 +10288,105 @@ describe('AgentPanelRoot workflow binding', () => { it.for([ ...[ - 'untouched', - 'new-draft', - 'cleared-draft', - 'removed-reference', - 'removed-attachment', - 'new-chat', - 'history' - ].flatMap((nextAction) => - ['mounted', 'reopened'].map((panel) => ({ nextAction, panel })) + { + nextAction: 'untouched', + expectedDraft: '', + act: () => Promise.resolve() + }, + { + nextAction: 'new-draft', + expectedDraft: 'New input', + act: async (textbox: HTMLElement) => { + await userEvent.click(textbox) + await userEvent.paste('New input') + } + }, + { + nextAction: 'cleared-draft', + expectedDraft: '', + act: async (textbox: HTMLElement) => { + await userEvent.click(textbox) + await userEvent.paste('New input') + await userEvent.clear(textbox) + } + }, + { + nextAction: 'removed-reference', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { + name: i18n.global.t('agent.addToPrompt') + }) + ) + await userEvent.click( + screen.getByRole('menuitem', { name: 'Workflows' }) + ) + await userEvent.click( + await screen.findByRole('menuitem', { name: 'reference' }) + ) + await userEvent.click( + screen.getByRole('button', { name: 'Remove reference reference' }) + ) + } + }, + { + nextAction: 'removed-attachment', + expectedDraft: '', + act: async () => { + useAgentComposerStore().addAttachment({ + id: 'upload-2', + name: 'new.png', + ref: 'new.png' + }) + await userEvent.click( + await screen.findByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'new.png' }) + }) + ) + } + }, + { + nextAction: 'new-chat', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { name: i18n.global.t('agent.newChat') }) + ) + } + }, + { + nextAction: 'history', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { + name: i18n.global.t('agent.showChatHistory') + }) + ) + await userEvent.click(await screen.findByText('Earlier chat')) + } + } + ].flatMap((scenario) => + ['mounted', 'reopened'].map((panel) => ({ ...scenario, panel })) ), - { nextAction: 'untouched', panel: 'closed' }, - { nextAction: 'stop', panel: 'reopened' } + { + nextAction: 'untouched', + panel: 'closed', + expectedDraft: '', + act: () => Promise.resolve() + }, + { + nextAction: 'stop', + panel: 'reopened', + expectedDraft: '', + act: async () => { + await userEvent.click(screen.getByRole('button', { name: 'Stop' })) + } + } ])( 'settles a submission without losing composer intent: $nextAction ($panel)', - async ({ nextAction, panel }) => { + async ({ nextAction, panel, expectedDraft, act }) => { setupWorkflowContext({ targetId: 'wf-42', references: [ @@ -10013,6 +10451,11 @@ describe('AgentPanelRoot workflow binding', () => { ] }) composer.setNodes([{ id: '12', title: 'KSampler' }]) + composer.setInsertionPoint({ + textOffset: composer.draft.length, + referenceIndex: composer.prompt.references.length + }) + composer.referenceAttachment('upload-1') const originalDraft = composer.draft const originalReferences = [...composer.workflowReferences] await userEvent.click(screen.getByRole('button', { name: 'Send' })) @@ -10032,54 +10475,8 @@ describe('AgentPanelRoot workflow binding', () => { textbox = screen.getByRole('textbox') expect(screen.getByRole('button', { name: 'Stop' })).toBeVisible() } - if (nextAction === 'new-draft' || nextAction === 'cleared-draft') - await userEvent.click(textbox) - await userEvent.paste('New input') - if (nextAction === 'cleared-draft') await userEvent.clear(textbox) - if (nextAction === 'removed-reference') { - await userEvent.click( - screen.getByRole('button', { - name: i18n.global.t('agent.addToPrompt') - }) - ) - await userEvent.click( - screen.getByRole('menuitem', { name: 'Workflows' }) - ) - await userEvent.click( - await screen.findByRole('menuitem', { name: 'reference' }) - ) - await userEvent.click( - screen.getByRole('button', { name: 'Remove reference reference' }) - ) - } - if (nextAction === 'removed-attachment') { - composer.addAttachment({ - id: 'upload-2', - name: 'new.png', - ref: 'new.png' - }) - await userEvent.click( - await screen.findByRole('button', { - name: i18n.global.t('agent.remove') - }) - ) - } - if (nextAction === 'new-chat') - await userEvent.click( - screen.getByRole('button', { name: i18n.global.t('agent.newChat') }) - ) - if (nextAction === 'history') { - await userEvent.click( - screen.getByRole('button', { - name: i18n.global.t('agent.showChatHistory') - }) - ) - await userEvent.click(await screen.findByText('Earlier chat')) - } - if (nextAction === 'stop') { - await userEvent.click(screen.getByRole('button', { name: 'Stop' })) - expect(cancellations).toHaveLength(0) - } + await act(textbox) + expect(cancellations).toHaveLength(0) finishSend( nextAction === 'stop' ? json(202, ack('wf-42', 'm-stopped')) @@ -10126,9 +10523,7 @@ describe('AgentPanelRoot workflow binding', () => { ] }) } else { - expect(useAgentComposerStore().draft).toBe( - nextAction === 'new-draft' ? 'New input' : '' - ) + expect(useAgentComposerStore().draft).toBe(expectedDraft) expect(composer.attachments).toEqual([]) expect( screen.queryByRole('button', { name: 'Open reference' }) @@ -10271,6 +10666,37 @@ describe('AgentPanelRoot workflow binding', () => { expect(canvasStore.selectedItems).toEqual([state.nodes[0]]) }) + it.for([1, 2])( + 'redraws the canvas after removing an inline node from %i selected nodes', + async (count) => { + makeTab() + mockMessagesEndpoint('wf-42') + const state = setupOwnedSelectionCanvas() + const other = new LGraphNode('VAE Decode') + state.subgraph.add(other) + vi.spyOn(state.canvas, 'animateToBounds').mockImplementation(() => {}) + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + await enterNodeSelectionMode() + const selected = [state.subgraphNode, other].slice(0, count) + state.canvas.selectItems(selected) + syncFakeSelection() + await nextTick() + state.canvas.dirty_canvas = false + + await userEvent.click( + screen.getByRole('button', { name: 'Remove KSampler #12 reference' }) + ) + + expect( + screen.queryByRole('button', { name: 'Remove KSampler #12 reference' }) + ).toBeNull() + expect([...state.canvas.selectedItems]).toEqual(selected.slice(1)) + expect(state.subgraphNode.selected).toBe(false) + expect(state.canvas.dirty_canvas).toBe(true) + } + ) + it('does not expose a canvas-focus action on a reference chip', async () => { makeTab() setupNodeSelectionCanvas() @@ -10328,6 +10754,50 @@ describe('AgentPanelRoot workflow binding', () => { expect([...state.selectedItems]).toEqual([rootTwin]) }) + it('updates the open node picker when the viewed graph changes without editing the prompt', async () => { + makeTab('wf-42') + const state = setupOwnedSelectionCanvas() + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + await openMentionPicker() + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + const prompt = useAgentComposerStore().prompt + + viewGraph(state.canvas, state.rootGraph) + + expect( + await screen.findByRole('menuitem', { name: 'Root node' }) + ).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().prompt).toEqual(prompt) + expect(useAgentComposerStore().nodes).toEqual([]) + }) + + it('lists target nodes loaded after opening the reference menu', async () => { + makeTab('wf-42') + const state = setupNodeSelectionCanvas() + showRootGraph(state) + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + + const textbox = screen.getByRole('textbox') + await userEvent.click(textbox) + await userEvent.paste('Use @') + showRootGraph(state, state.nodes) + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + await userEvent.click(screen.getByRole('menuitem', { name: 'KSampler' })) + expect(useAgentComposerStore().nodes).toEqual([ + { id: '12', locatorId: '12', title: 'KSampler' } + ]) + await userEvent.paste('next') + expect(useAgentComposerStore().draft).toBe('Use next') + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler' + ) + }) + it('excludes referenced nodes from the mention picker', async () => { makeTab() setupNodeSelectionCanvas() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 82847728f3f..26c1dd0a886 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -11,7 +11,6 @@ import { onBeforeUnmount, onMounted, provide, - readonly, ref, watch } from 'vue' @@ -602,7 +601,9 @@ let nodeReferenceWorkflow = selectionTags.value.length : null function viewedGraphNodes() { - return app.canvas?.graph?.nodes ?? app.graph?.nodes ?? [] + return ( + (canvasStore.currentGraph ?? app.canvas?.graph ?? app.graph)?.nodes ?? [] + ) } function mentionableNodes(): SelectedNode[] { @@ -1572,11 +1573,9 @@ function onNewChat(source?: 'new_chat_button' | 'history_delete'): void { else agentPanelStore.startFollowingVisibleWorkflow() } -const panelRef = ref>() const fileInput = ref() const assetDragActive = ref(false) let assetDragDepth = 0 -provide('agentAssetDragActive', readonly(assetDragActive)) watch( selectedTarget, @@ -1666,11 +1665,24 @@ const attachment = useAttachment({ // must not raise the server-error overlay. onError: (message) => toast.add({ severity: 'warn', detail: message, life: 5000 }), + onDuplicate: notifyDuplicateAttachments, stage: composerStore.addAttachment, update: composerStore.updateAttachment, remove: composerStore.removeAttachment }) +function notifyDuplicateAttachments(names: string[]): void { + toast.add({ + severity: 'info', + detail: t( + 'agent.attachmentsAlreadyAdded', + { name: names[0], count: names.length }, + names.length + ), + life: 3500 + }) +} + onBeforeUnmount(() => runPanelTeardown({ cancelAllUploads: () => { @@ -1710,6 +1722,7 @@ function onRemoveSelectionTag(id: string): void { removeSelectionTag(id) if (node) { canvasStore.canvas?.deselect(node) + canvasStore.canvas?.setDirty(true) } } @@ -1766,21 +1779,33 @@ async function attachDroppedAsset(event: DragEvent): Promise { return false } + const sourceKey = asset.ref ? `asset:${asset.ref}` : `uri:${asset.uri}` if (asset.ref && asset.kind !== 'other') { - return ( - panelRef.value?.addAttachment({ - id: `asset:${asset.ref}`, - name: asset.name, - ref: asset.ref, - previewUrl: asset.previewUrl - }) ?? false - ) + if (composerStore.attachments.some((item) => item.ref === asset.ref)) { + notifyDuplicateAttachments([asset.name]) + return false + } + const added = composerStore.addAttachment({ + id: `asset:${crypto.randomUUID()}`, + name: asset.name, + ref: asset.ref, + sourceKey, + previewUrl: asset.previewUrl, + mediaUrl: asset.mediaUrl, + mediaKind: asset.kind + }) + if (!added) notifyDuplicateAttachments([asset.name]) + return added } - const result = await attachment.addDeferredFile(asset.name, async () => { - const file = await fetchDroppedAsset(asset) - return file && isAgentAttachable(file) ? file : undefined - }) + const result = await attachment.addDeferredFile( + asset.name, + async () => { + const file = await fetchDroppedAsset(asset) + return file && isAgentAttachable(file) ? file : undefined + }, + sourceKey + ) if (result === 'unsupported') toast.add({ severity: 'warn', @@ -1822,7 +1847,7 @@ async function onPanelDrop(event: DragEvent): Promise {
{ @change="onFilesPicked" /> { +
+
{ }) import { i18n } from '@/i18n' -import type { ComposerAttachment } from '../../composables/agent/useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import { toTurnId } from '../../schemas/agentApiSchema' import type { WorkflowReference } from '../../types/workflowReference' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' diff --git a/src/workbench/extensions/agent/components/agent/AgentPanel.vue b/src/workbench/extensions/agent/components/agent/AgentPanel.vue index d04b743c73e..c2bc6359279 100644 --- a/src/workbench/extensions/agent/components/agent/AgentPanel.vue +++ b/src/workbench/extensions/agent/components/agent/AgentPanel.vue @@ -22,7 +22,7 @@ import type { WorkflowReferenceOption } from '../../types/workflowReference' import type { TurnId } from '../../schemas/agentApiSchema' -import type { ComposerAttachment } from '../../composables/agent/useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import type { SelectedNode } from '../../composables/agent/useCanvasSelection' import { DEFAULT_AGENT_PAYWALL_PRESENTATION } from '@/workbench/extensions/agent/services/agent/agentPaywallPresentation' import type { diff --git a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts new file mode 100644 index 00000000000..7b66d93fb20 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts @@ -0,0 +1,740 @@ +import userEvent from '@testing-library/user-event' +import { + fireEvent, + render, + screen, + waitFor, + within +} from '@testing-library/vue' +import { describe, expect, it, vi } from 'vitest' +import { defineComponent, h, nextTick } from 'vue' + +import { i18n } from '@/i18n' + +import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' +import Composer from './Composer.vue' +import { setupInlinePromptEditorDom } from './composer/inlinePromptEditorTestSetup' + +setupInlinePromptEditorDom() + +const asset = { + id: 'source', + name: 'source.png', + ref: 'uploaded-source.png', + previewUrl: 'https://example.com/source.png' +} + +type MediaSurfaces = { + chip: HTMLElement + trayItem: HTMLElement + menuItem: HTMLElement +} + +function setup() { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + const send = vi.fn() + const Host = defineComponent({ + setup: () => () => + h(Composer, { + hasWorkflowTarget: true, + selectionTags: store.nodes, + onSend: send + }) + }) + const view = render(Host, { + global: { plugins: [i18n], directives: { tooltip: () => {} } } + }) + return { ...view, store, send, editor: screen.getByRole('textbox') } +} + +function trackResizes() { + const callbacks = new Map void>>() + vi.stubGlobal( + 'ResizeObserver', + class implements ResizeObserver { + private targets = new Set() + private notify: () => void + constructor(callback: ResizeObserverCallback) { + this.notify = () => callback([], this) + } + observe(target: Element) { + this.targets.add(target) + const listeners = callbacks.get(target) ?? new Set<() => void>() + listeners.add(this.notify) + callbacks.set(target, listeners) + } + unobserve(target: Element) { + callbacks.get(target)?.delete(this.notify) + this.targets.delete(target) + } + disconnect() { + for (const target of this.targets) this.unobserve(target) + } + } + ) + return (target: Element) => + callbacks.get(target)?.forEach((notify) => notify()) +} + +function setTrayGeometry( + tray: HTMLElement, + width: number, + contentWidth: number +) { + const dimensions = { width, contentWidth } + let left = 0 + Object.defineProperties(tray, { + clientWidth: { get: () => dimensions.width }, + scrollWidth: { get: () => dimensions.contentWidth }, + scrollLeft: { + get: () => { + left = Math.min( + left, + Math.max(0, dimensions.contentWidth - dimensions.width) + ) + return left + }, + set: (value: number) => { + left = value + } + }, + scrollTo: { + value: (options: ScrollToOptions) => { + tray.scrollLeft = options.left ?? tray.scrollLeft + tray.dispatchEvent(new Event('scroll')) + } + } + }) + return dimensions +} + +describe('composer asset tray', () => { + it('updates every inline video thumbnail without changing its references or opening a preview', async () => { + const user = userEvent.setup() + const { store } = setup() + store.addAttachment({ + id: 'video', + name: 'My video', + ref: '', + mediaKind: 'video', + mediaUrl: 'blob:video', + uploading: true + }) + store.referenceAttachment('video') + store.referenceAttachment('video') + const chips = await screen.findAllByTestId('asset-reference-chip') + expect( + chips.map((chip) => + within(chip) + .getByRole('img', { name: 'Video' }) + .getAttribute('aria-label') + ) + ).toEqual(['Video', 'Video']) + store.updateAttachment('video', { + ref: 'stored.mp4', + mediaUrl: '/stored.mp4', + previewUrl: '/poster.png', + uploading: false + }) + await waitFor(() => + expect( + screen + .getAllByTestId('asset-reference-chip') + .map((chip) => within(chip).getByAltText('').getAttribute('src')) + ).toEqual(['/poster.png', '/poster.png']) + ) + await user.hover(screen.getAllByTestId('asset-reference-chip')[0]) + expect(screen.getByRole('group', { name: 'My video' })).toHaveAttribute( + 'data-highlighted', + 'true' + ) + expect( + screen.queryByRole('region', { name: 'My video' }) + ).not.toBeInTheDocument() + expect(store.prompt.references).toHaveLength(2) + expect(store.attachments).toHaveLength(1) + }) + + it('shows the audio type of a renamed inline attachment', async () => { + const { store } = setup() + store.addAttachment({ + id: 'audio', + name: 'Recording', + ref: 'song.mp3', + mediaKind: 'audio', + mediaUrl: '/song.mp3' + }) + store.referenceAttachment('audio') + const chip = await screen.findByTestId('asset-reference-chip') + expect(within(chip).getByRole('img', { name: 'Audio' })).toBeInTheDocument() + }) + + it.for([ + { + name: 'My video', + mediaKind: 'video', + previewUrl: '/poster.png', + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => [ + within(chip).getByAltText(''), + within(trayItem).getByAltText('My video'), + within(menuItem).getByAltText('') + ], + sources: ['/poster.png', '/poster.png', '/poster.png'], + labels: [null, null, null] + }, + { + name: 'Recording', + mediaKind: 'audio', + previewUrl: undefined, + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => + [chip, trayItem, menuItem].map((surface) => + within(surface).getByRole('img', { name: 'Audio' }) + ), + sources: [null, null, null], + labels: ['Audio', 'Audio', 'Audio'] + } + ] as const)( + 'uses the same media indicator for $mediaKind in the tray, menu and inline', + async ({ name, mediaKind, previewUrl, indicators, sources, labels }) => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment({ + id: 'media', + name, + ref: '/file', + mediaKind, + previewUrl + }) + store.referenceAttachment('media') + const chip = await screen.findByTestId('asset-reference-chip') + await user.click(editor) + await user.keyboard('@') + const menuItem = await screen.findByRole('menuitem', { name }) + const trayItem = screen.getByRole('group', { name }) + const images = indicators({ chip, trayItem, menuItem }) + expect(images.map((image) => image.getAttribute('src'))).toEqual(sources) + expect(images.map((image) => image.getAttribute('aria-label'))).toEqual( + labels + ) + await user.click(menuItem) + expect(await screen.findAllByTestId('asset-reference-chip')).toHaveLength( + 2 + ) + expect(store.attachments).toHaveLength(1) + } + ) + + it('announces all pending uploads outside the scroll viewport and preserves the draft on Enter', async () => { + const user = userEvent.setup() + const { store, editor, send } = setup() + const status = screen.getByTestId('composer-upload-status') + expect(status).toBeEmptyDOMElement() + expect(status).toHaveAttribute('aria-live', 'polite') + expect(status).toHaveAttribute('aria-atomic', 'true') + store.addAttachment({ + id: 'first', + name: 'first.png', + ref: '', + uploading: true + }) + store.addAttachment({ + id: 'second', + name: 'second.png', + ref: '', + uploading: true + }) + await user.type(editor, 'Keep my draft') + expect(status).toHaveTextContent('Uploading 2 attachments') + expect( + within(screen.getByTestId('composer-asset-section')).queryByTestId( + 'composer-upload-status' + ) + ).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Send' })).toBeDisabled() + await user.keyboard('{Enter}') + expect(send).not.toHaveBeenCalled() + expect(store.draft).toBe('Keep my draft') + store.updateAttachment('first', { ref: 'first.png', uploading: false }) + await waitFor(() => + expect(status).toHaveTextContent('Uploading 1 attachment') + ) + store.removeAttachment('second') + await waitFor(() => expect(status).toBeEmptyDOMElement()) + expect(screen.getByTestId('composer-upload-status')).toBe(status) + expect(screen.getByRole('button', { name: 'Send' })).toBeEnabled() + await user.keyboard('{Enter}') + expect(send).toHaveBeenCalledExactlyOnceWith('Keep my draft', [ + expect.objectContaining({ id: 'first', ref: 'first.png' }) + ]) + }) + + it('offers bounded page scrolling without changing the draft or included assets', async () => { + const resize = trackResizes() + const { store, editor } = setup() + const assets = Array.from({ length: 8 }, (_, index) => ({ + id: `asset-${index}`, + name: `asset-${index}.png`, + ref: `asset-${index}.png` + })) + store.replaceDraft({ + text: '', + attachments: assets, + workflowReferences: [] + }) + await userEvent.type(editor, 'Keep this draft') + const tray = screen.getByTestId('composer-asset-section') + setTrayGeometry(tray, 250, 720) + resize(tray) + await fireEvent.scroll(tray) + + const previous = screen.getByRole('button', { name: 'Scroll Left' }) + const next = screen.getByRole('button', { name: 'Scroll Right' }) + expect(previous).toBeDisabled() + expect(next).toBeEnabled() + await userEvent.click(next) + expect(tray.scrollLeft).toBe(250) + expect(previous).toBeEnabled() + expect(next).toBeEnabled() + await userEvent.keyboard('{Enter}') + expect(tray.scrollLeft).toBe(470) + expect(next).toBeDisabled() + await userEvent.click(previous) + expect(tray.scrollLeft).toBe(220) + expect(next).toBeEnabled() + expect(store.draft).toBe('Keep this draft') + expect(store.attachments).toEqual(assets) + }) + + it('updates overflow controls when the tray is resized', async () => { + const resize = trackResizes() + const { store } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await nextTick() + const tray = screen.getByTestId('composer-asset-section') + const dimensions = setTrayGeometry(tray, 240, 192) + resize(tray) + await nextTick() + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + + dimensions.width = 160 + resize(tray) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + + dimensions.width = 240 + resize(tray) + await nextTick() + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + expect(screen.queryByRole('button', { name: 'Scroll Left' })).toBeNull() + }) + + it('clears overflow controls when removing assets makes the tray fit', async () => { + const resize = trackResizes() + const { store } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await nextTick() + const tray = screen.getByTestId('composer-asset-section') + const dimensions = setTrayGeometry(tray, 160, 192) + resize(tray) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + + await userEvent.click(screen.getByRole('button', { name: 'Scroll Right' })) + expect(tray.scrollLeft).toBe(32) + + dimensions.contentWidth = 104 + await userEvent.click( + within(screen.getByRole('group', { name: 'other.png' })).getByRole( + 'button', + { name: 'Remove other.png' } + ) + ) + + await waitFor(() => + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + ) + expect(store.attachments).toEqual([asset]) + + dimensions.contentWidth = 192 + store.addAttachment({ id: 'new', name: 'new.png', ref: 'new.png' }) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + expect(screen.getByRole('button', { name: 'Scroll Left' })).toBeDisabled() + store.removeAttachment('new') + store.removeAttachment(asset.id) + await nextTick() + expect(screen.queryByTestId('composer-asset-section')).toBeNull() + }) + + it('stages assets without changing the prompt, and sends unmentioned assets', async () => { + const { store, editor, send } = setup() + await userEvent.type(editor, 'Inspect these images') + store.addAttachment(asset) + await screen.findByRole('img', { name: asset.name }) + + expect(editor).toHaveTextContent('Inspect these images') + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect(store.draft).toBe('Inspect these images') + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith('Inspect these images', [asset]) + }) + + it.for([ + { + via: 'pointer', + pick: () => + userEvent.click(screen.getByRole('menuitem', { name: asset.name })) + }, + { via: 'keyboard', pick: () => userEvent.keyboard('{Enter}') } + ])( + 'inserts a matching asset at the caret via $via and retains the tray', + async ({ pick }) => { + const { store, editor, send } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await userEvent.type(editor, 'Use @source') + const menu = screen.getByRole('menu', { name: 'Add to prompt' }) + expect( + within(menu).queryByRole('menuitem', { name: 'other.png' }) + ).toBeNull() + await pick() + await screen.findByTestId('asset-reference-chip') + expect(editor.textContent).toBe('Use source.png ') + expect(store.attachments).toEqual([ + asset, + { id: 'other', name: 'other.png', ref: 'other.png' } + ]) + expect( + within(screen.getByTestId('composer-asset-section')).getByRole( + 'group', + { name: asset.name } + ) + ).toBeVisible() + await userEvent.keyboard('as the subject') + expect(editor.textContent).toBe('Use source.png as the subject') + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith( + 'Use @[Image: source.png] as the subject', + [asset, { id: 'other', name: 'other.png', ref: 'other.png' }] + ) + } + ) + + it('keeps included assets through inline deletion, Undo and remount', async () => { + const { store, editor, unmount } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}{Backspace}{Backspace}') + await waitFor(() => + expect( + screen.queryByTestId('asset-reference-chip') + ).not.toBeInTheDocument() + ) + expect(store.attachments).toEqual([asset]) + await userEvent.keyboard('{Control>}z{/Control}') + await screen.findByTestId('asset-reference-chip') + expect(store.attachments).toEqual([asset]) + unmount() + setup() + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: asset.name + }) + ).toBeVisible() + expect(screen.getByTestId('asset-reference-chip')).toHaveTextContent( + asset.name + ) + }) + + it('allows repeated asset mentions, deletes only one, and sends the attachment once', async () => { + const { store, editor, send } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('then @source') + expect(screen.getByRole('menuitem', { name: asset.name })).toBeVisible() + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('and @source') + await userEvent.keyboard('{Enter}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(3) + ) + expect(editor.textContent).toBe( + 'Use source.png then source.png and source.png ' + ) + + await userEvent.keyboard('{Backspace}{Backspace}') + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + expect(store.attachments).toEqual([asset]) + await userEvent.keyboard('{Control>}z{/Control}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(3) + ) + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith( + 'Use @[Image: source.png] then @[Image: source.png] and @[Image: source.png]', + [asset] + ) + }) + + it('removes only the chosen inline asset with its hover button, retains the tray, and supports Undo and continued typing', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, 'Use @source') + await user.keyboard('{Enter}then @source{Enter}after') + const references = await screen.findAllByTestId('asset-reference-chip') + expect(references).toHaveLength(2) + await user.hover(references[0]) + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.click( + within(references[0]).getByRole('button', { + name: 'Remove source.png reference' + }) + ) + + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(1) + expect(editor.textContent).toBe('Use then source.png after') + expect(store.attachments).toEqual([asset]) + expect(editor).toHaveFocus() + expect(trayItem).not.toHaveAttribute('data-highlighted') + await user.keyboard('{Control>}z{/Control}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + ) + expect(editor.textContent).toBe('Use source.png then source.png after') + expect(trayItem).not.toHaveAttribute('data-highlighted') + await user.keyboard('continued ') + expect(editor).toHaveTextContent('continued') + expect(store.attachments).toEqual([asset]) + }) + + it('keeps typing uninterrupted after asset insertion and highlights its tray item on inline hover without opening a preview', async () => { + vi.useFakeTimers() + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, 'Use @source') + await user.keyboard('{Enter}') + const chip = screen.getByTestId('asset-reference-chip') + + chip.dispatchEvent( + new PointerEvent('pointerenter', { pointerType: 'mouse' }) + ) + await nextTick() + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + expect(editor).toHaveFocus() + await user.keyboard('and keep typing') + expect(editor.textContent).toBe('Use source.png and keep typing') + + await user.hover(chip) + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + expect(editor).toHaveFocus() + + await user.unhover(chip) + expect(trayItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset]) + }) + + it('highlights the tray while an inline asset has focus or hover, without opening a preview', async () => { + vi.useFakeTimers() + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}') + + await user.tab() + const chip = screen.getByTestId('asset-reference-chip') + expect(chip).toHaveFocus() + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + + await user.hover(chip) + await user.tab({ shift: true }) + expect(editor).toHaveFocus() + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(chip) + expect(trayItem).not.toHaveAttribute('data-highlighted') + }) + + it('maps repeated inline references to their tray asset and clears only the activity that ended', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + const other = { id: 'other', name: 'other.png', ref: 'other.png' } + store.addAttachment(asset) + store.addAttachment(other) + await user.type(editor, '@source') + await user.keyboard('{Enter}then @source{Enter}and @other{Enter}') + const references = screen.getAllByTestId('asset-reference-chip') + expect(references).toHaveLength(3) + const tray = within(screen.getByTestId('composer-asset-section')) + const sourceItem = tray.getByRole('group', { name: asset.name }) + const otherItem = tray.getByRole('group', { name: other.name }) + + await user.tab() + expect(references[0]).toHaveFocus() + await user.hover(references[1]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).not.toHaveAttribute('data-highlighted') + await user.unhover(references[1]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + + await user.hover(references[2]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(references[2]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).not.toHaveAttribute('data-highlighted') + await user.click(editor) + expect(sourceItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset, other]) + }) + + it('retains the highlight from a hovered occurrence when another focused occurrence is removed', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}then @source{Enter}') + const references = screen.getAllByTestId('asset-reference-chip') + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + + await user.tab() + expect(references[0]).toHaveFocus() + await user.hover(references[1]) + await user.tab() + expect( + within(references[0]).getByRole('button', { + name: 'Remove source.png reference' + }) + ).toHaveFocus() + await user.keyboard('{Enter}') + + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(1) + expect(editor).toHaveFocus() + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(references[1]) + expect(trayItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset]) + }) + + it('opens a large asset preview from tray hover and closes it on leave or unmount', async () => { + const user = userEvent.setup() + const { store, editor, unmount } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}') + await screen.findByTestId('asset-reference-chip') + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + + const trigger = screen.getByRole('button', { + name: `Preview ${asset.name}` + }) + + await user.hover(trigger) + const preview = await screen.findByRole('dialog', { name: asset.name }) + expect( + within(preview).getByRole('img', { name: asset.name }) + ).toHaveAttribute('src', asset.previewUrl) + await user.unhover(trigger) + await user.pointer({ + target: screen.getByRole('button', { name: 'Send' }), + coords: { clientX: 500, clientY: 500 } + }) + await waitFor(() => + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + ) + + await user.hover(trigger) + await screen.findByRole('dialog', { name: asset.name }) + unmount() + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + }) + + it('inserts an asset in the middle of a sentence and preserves the following text', async () => { + const { store, editor } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use as background') + await userEvent.keyboard('{Control>}a{/Control}{ArrowLeft}{ArrowRight>4/}') + await userEvent.paste('@source') + await userEvent.keyboard('{Enter}') + + await screen.findByTestId('asset-reference-chip') + expect(editor.textContent).toBe('Use source.png as background') + expect(store.prompt.text).toBe('Use as background') + }) + + it('retires a tray asset and its inline reference despite late upload completion or Undo', async () => { + const { store, editor } = setup() + store.addAttachment({ ...asset, uploading: true, ref: '' }) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('and @source') + await userEvent.keyboard('{Enter}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + ) + await userEvent.click( + within(screen.getByTestId('composer-asset-section')).getByRole('button', { + name: 'Remove source.png' + }) + ) + store.updateAttachment(asset.id, { uploading: false, ref: asset.ref }) + await waitFor(() => + expect( + screen.queryByTestId('composer-asset-section') + ).not.toBeInTheDocument() + ) + await userEvent.click(editor) + await userEvent.keyboard('{Control>}z{/Control}') + expect(store.attachments).toEqual([]) + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + }) + + it('shows nodes only inline without adding an asset', async () => { + const { store } = setup() + store.setNodes([{ id: '361', title: 'Save Image' }]) + await screen.findByTestId('node-reference-chip') + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect( + screen.queryByTestId('composer-asset-section') + ).not.toBeInTheDocument() + expect(screen.getByRole('textbox')).toHaveTextContent('Save Image #361') + }) +}) diff --git a/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts b/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts index d736c5ce6ab..fcc48be254f 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts @@ -45,59 +45,28 @@ function renderComposer() { describe('inline node and asset references', () => { beforeEach(() => vi.useRealTimers()) - it('keeps upper-row removal and Undo synchronized with inline references', async () => { + it('keeps inline node deletion and Undo synchronized with selected context', async () => { const { store, selected, editor, send } = renderComposer() await userEvent.type(editor, 'Use ') selected.value = [{ id: '12', title: 'KSampler' }] - await waitFor(() => expect(editor.textContent).toBe('Use KSampler #12 ')) - await userEvent.keyboard('with ') + await screen.findByTestId('node-reference-chip') + await userEvent.keyboard('{Backspace}{Backspace}') + expect(store.nodes).toEqual([]) + expect(editor.textContent).toBe('Use ') + await userEvent.keyboard('{Control>}z{/Control}') + await screen.findByTestId('node-reference-chip') + expect(store.nodes).toEqual([{ id: '12', title: 'KSampler' }]) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + await userEvent.keyboard(' with ') store.addAttachment({ id: 'image', name: 'source.png', ref: 'uploaded.png' }) - await waitFor(() => - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') - ) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #12 reference' }) - ) - await waitFor(() => - expect( - screen.queryByTestId('composer-node-section') - ).not.toBeInTheDocument() - ) - expect(editor.textContent).toBe('Use with source.png ') - expect(store.nodes).toEqual([]) - editor.focus() - await userEvent.keyboard('{Control>}z{/Control}') - await waitFor(() => - expect(screen.getByTestId('composer-node-section')).toHaveTextContent( - 'KSampler' - ) - ) - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') - - await userEvent.click( - within(screen.getByTestId('composer-asset-section')).getByRole('button', { - name: 'Remove' - }) - ) - await waitFor(() => - expect( - screen.queryByTestId('composer-asset-section') - ).not.toBeInTheDocument() - ) - expect(editor.textContent).toBe('Use KSampler #12 with ') - editor.focus() - await userEvent.keyboard('{Control>}z{/Control}') - await waitFor(() => - expect(screen.getByTestId('composer-asset-section')).toHaveTextContent( - 'source.png' - ) - ) - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') + store.referenceAttachment('image') + await screen.findByTestId('asset-reference-chip') await userEvent.click(screen.getByRole('button', { name: 'Send' })) expect(send).toHaveBeenCalledWith( 'Use @[Node: KSampler #12] with @[Image: source.png]', @@ -114,10 +83,11 @@ describe('inline node and asset references', () => { ref: '', uploading: true }) + store.referenceAttachment('image') await screen.findByTestId('asset-reference-chip') await userEvent.click( within(screen.getByTestId('composer-asset-section')).getByRole('button', { - name: 'Remove' + name: 'Remove pending.png' }) ) store.updateAttachment('image', { ref: 'complete.png', uploading: false }) @@ -131,15 +101,8 @@ describe('inline node and asset references', () => { ).not.toBeInTheDocument() editor.focus() await userEvent.keyboard('{Control>}z{/Control}') - await screen.findByTestId('asset-reference-chip') - expect(store.attachments).toEqual([ - { - id: 'image', - name: 'pending.png', - ref: 'complete.png', - uploading: false - } - ]) + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect(store.attachments).toEqual([]) }) it('deletes a node from the sentence without restaging the selected canvas node', async () => { @@ -166,6 +129,7 @@ describe('inline node and asset references', () => { selected.value = [{ id: '12', title: 'KSampler' }] await screen.findByTestId('node-reference-chip') store.addAttachment({ id: 'image', name: 'source.png', ref: 'source.png' }) + store.referenceAttachment('image') await screen.findByTestId('asset-reference-chip') await user.keyboard('{Control>}a{/Control}') const clipboard = await user.copy() diff --git a/src/workbench/extensions/agent/components/agent/Composer.test.ts b/src/workbench/extensions/agent/components/agent/Composer.test.ts index b37bcafd6f5..b975e1a4f20 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.test.ts @@ -7,11 +7,12 @@ import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' import { render, screen, waitFor, within } from '@testing-library/vue' import userEvent from '@testing-library/user-event' import { assert, beforeEach, describe, expect, it, vi } from 'vitest' -import { defineComponent, h, nextTick, ref } from 'vue' -import type { DirectiveBinding } from 'vue' +import { defineComponent, h, nextTick, ref, shallowRef } from 'vue' +import type { DirectiveBinding, ShallowRef } from 'vue' import type { ComponentProps } from 'vue-component-type-helpers' import { i18n } from '@/i18n' +import { LGraph, LGraphNode } from '@/lib/litegraph/src/litegraph' import { consultEscapeOverride } from '@/platform/keybindings/escapeOverride' import { useTelemetry } from '@/platform/telemetry' import { useToastStore } from '@/platform/updates/common/toastStore' @@ -202,16 +203,38 @@ describe('Composer', () => { ).toBeTruthy() }) - it('hides the empty-composer hint once typing begins', async () => { - mount() - const box = screen.getByRole('textbox') + it.for([ + { tray: 'empty', attachments: [] }, + { + tray: 'with an asset', + attachments: [{ id: 'asset', name: 'cat.png', ref: 'cat.png' }] + } + ])( + 'hides the hint while typing and restores it after clearing (tray=$tray)', + async ({ attachments }) => { + const store = useAgentComposerStore() + store.replaceDraft({ text: '', workflowReferences: [], attachments }) + const { emitted } = mount() + const box = screen.getByRole('textbox') + expect( + screen.getByRole('button', { name: 'mention nodes' }) + ).toBeVisible() - await userEvent.click(box) - await userEvent.paste('hello') + await userEvent.click(box) + await userEvent.paste('hello') - expect(useAgentComposerStore().draft).toBe('hello') - expect(screen.queryByRole('button', { name: 'mention nodes' })).toBeNull() - }) + expect(store.draft).toBe('hello') + expect(screen.queryByRole('button', { name: 'mention nodes' })).toBeNull() + + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(store.draft).toBe('') + await userEvent.click( + screen.getByRole('button', { name: 'mention nodes' }) + ) + expect(emitted().selectNodes).toHaveLength(1) + expect(store.attachments).toEqual(attachments) + } + ) it('enters graph selection mode from the empty-composer hint', async () => { const getMentionNodes = vi.fn(() => []) @@ -910,6 +933,102 @@ describe('Composer', () => { ]) }) + it('loads current nodes when entering or re-entering the Nodes submenu', async () => { + const nodes = ref([]) + mount({ getMentionNodes: () => nodes.value }) + await openReferenceRoot() + + nodes.value = [NODES[0]] + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + + await userEvent.click(screen.getByRole('menuitem', { name: 'Back' })) + nodes.value = [NODES[2]] + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().draft).toBe('@') + }) + + it('filters current nodes after the graph changes with the submenu open', async () => { + const nodes = ref([NODES[0]]) + const { emitted } = mount({ getMentionNodes: () => nodes.value }) + await openReferenceSection('Nodes') + + nodes.value = [NODES[2]] + await userEvent.click(screen.getByRole('textbox')) + await userEvent.paste('vae') + + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().draft).toBe('@vae') + await userEvent.keyboard('{Enter}') + expect(emitted().mentionPick).toEqual([[NODES[2]]]) + expect(emitted().send).toBeUndefined() + }) + + it.for<{ + event: string + act: (graph: ShallowRef, node: LGraphNode) => void + labels: string[] + }>([ + { + event: 'adding a node', + act: (graph) => graph.value.add(new LGraphNode('Alpha')), + labels: ['Back', 'Alpha', 'KSampler'] + }, + { + event: 'renaming a node', + act: (_graph, node) => { + node.title = 'Alpha' + }, + labels: ['Back', 'Alpha'] + }, + { + event: 'removing a node', + act: (graph, node) => graph.value.remove(node), + labels: ['Back'] + }, + { + event: 'replacing the viewed graph', + act: (graph) => { + const replacement = new LGraph() + replacement.add(new LGraphNode('VAE Decode')) + graph.value = replacement + }, + labels: ['Back', 'VAE Decode'] + } + ])( + 'updates the open Nodes submenu after $event without typing', + async ({ act, labels }) => { + const graph = shallowRef(new LGraph()) + const node = new LGraphNode('KSampler') + graph.value.add(node) + const { emitted } = mount({ + getMentionNodes: () => + graph.value.nodes.map((item) => ({ + id: String(item.id), + title: item.title + })) + }) + await openReferenceSection('Nodes') + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + + act(graph, node) + + await waitFor(() => + expect( + within(screen.getByRole('menu', { name: 'Add to prompt' })) + .getAllByRole('menuitem') + .map((item) => item.textContent.trim()) + ).toEqual(labels) + ) + expect(useAgentComposerStore().draft).toBe('@') + expect(emitted().mentionPick).toBeUndefined() + expect(emitted().send).toBeUndefined() + } + ) + // Re-picking a staged node is a no-op, so it drops out of the list. it('hides nodes already in the basket', async () => { mount({ @@ -926,11 +1045,11 @@ describe('Composer', () => { expect(labels).toEqual(['Back', 'KSampler #7', 'VAE Decode']) }) - // The staged node is only hidden from the picker, not from the duplicate - // check - its chip must still show the id that tells it apart from the - // same-titled node still in the graph. it('keeps disambiguating a staged node against its graph twin', async () => { const twin = { id: '11', title: NODES[0].title } + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([NODES[0]]) mount({ getMentionNodes: () => [...NODES, twin], selectionTags: [NODES[0]] @@ -938,7 +1057,12 @@ describe('Composer', () => { await openReferenceSection('Nodes') - expect(screen.getByText(`#${NODES[0].id}`)).toBeInTheDocument() + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler #5' + ) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() }) it('keeps type-to-filter behavior inside the selected reference type', async () => { @@ -1334,7 +1458,10 @@ describe('Composer', () => { it.for(['pointer', 'Enter', 'Space'])( 'opens a staged workflow with %s without consuming the draft', async (interaction) => { - useAgentComposerStore().setText('Keep this prompt') + const store = useAgentComposerStore() + store.setText('Keep this prompt') + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) const { emitted } = mount({ workflowReferences: [ { id: 'wf-1', name: 'Water world', textOffset: 0 } @@ -1351,9 +1478,10 @@ describe('Composer', () => { expect(emitted().openReferenceWorkflow).toEqual([['wf-1', 'Water world']]) expect(emitted().removeWorkflowReference).toBeUndefined() expect(emitted().send).toBeUndefined() - expect(useAgentComposerStore().draft).toBe('Keep this prompt') - expect(screen.getByTestId('composer-node-section')).toHaveTextContent( - 'KSampler' + expect(store.draft).toBe('Keep this prompt ') + expect(store.nodes).toEqual([{ id: '5', title: 'KSampler' }]) + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler #5' ) } ) @@ -1517,20 +1645,20 @@ describe('Composer', () => { expect(useAgentComposerStore().draft).toBe('tex') }) - it('keeps selected nodes in a dedicated section above the inline prompt', () => { + it('keeps selected nodes inline alongside workflow references', () => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) mount({ - selectionTags: [{ id: '5', title: 'KSampler' }], + selectionTags: store.nodes, workflowReferences: [{ id: 'wf-1', name: 'Water world', textOffset: 0 }] }) - - const nodeSection = screen.getByTestId('composer-node-section') - const inlineInput = screen.getByTestId('composer-inline-input') - - expect(nodeSection).toHaveClass('border-b', 'p-3') - expect(nodeSection).toHaveTextContent('KSampler') - expect(nodeSection).not.toContainElement(screen.getByRole('textbox')) - expect(inlineInput).not.toHaveTextContent('KSampler') - expect(inlineInput).toHaveTextContent('Water world') + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect(screen.getByRole('textbox')).toHaveTextContent( + 'Water worldKSampler #5' + ) }) it('keeps added assets in a separate padded section above the prompt', async () => { @@ -1583,9 +1711,11 @@ describe('Composer', () => { screen.getByRole('button', { name: 'Open Water world' }) ) expect(onOpenReferenceWorkflow).toHaveBeenCalledWith('wf-1', 'Water world') - expect(screen.getByTestId('composer-asset-section')).toHaveTextContent( - 'cat.png' - ) + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: 'cat.png' + }) + ).toBeVisible() }) it('keeps the empty prompt hint visible when only nodes are selected', () => { @@ -1645,81 +1775,51 @@ describe('Composer', () => { expect(getMentionNodes).not.toHaveBeenCalled() }) - it('hides the id on a uniquely named selection chip', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - expect(screen.getByText('KSampler')).toBeInTheDocument() - expect(screen.queryByText('#5')).not.toBeInTheDocument() - }) - - it('emits removeTag when a selection chip is removed', async () => { - const { emitted } = mount({ - selectionTags: [{ id: '5', title: 'KSampler' }] - }) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #5 reference' }) - ) - - expect(emitted().removeTag).toEqual([['5']]) - }) - - it('builds the remove tooltip for a selection chip', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - const removeButton = screen.getByRole('button', { - name: 'Remove KSampler #5 reference' - }) - expect(tooltipBindings.get(removeButton)).toMatchObject({ value: 'Remove' }) - }) - - it('renders a selection chip label as non-interactive context', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - expect(screen.getByText('KSampler')).toBeVisible() - expect( - screen.queryByRole('button', { name: 'Show KSampler #5 on canvas' }) - ).toBeNull() - }) - - // The remove button sits outside the focus trigger; removing a chip must not - // also fly the canvas to the node being removed. - it('removes a selection chip without focusing its node', async () => { - const { emitted } = mount({ - selectionTags: [{ id: '5', title: 'KSampler' }] - }) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #5 reference' }) - ) - - expect(emitted().removeTag).toEqual([['5']]) - expect(emitted().focusTag).toBeUndefined() - }) - - it('shows the id on a lone selection chip with a graph title twin', () => { - const selected = { id: '5', title: 'KSampler' } - mount({ - selectionTags: [selected], - getMentionNodes: () => [selected, { id: '7', title: 'KSampler' }] - }) - - expect(screen.getByText('KSampler')).toBeInTheDocument() - expect(screen.getByText('#5')).toBeInTheDocument() - expect(screen.queryByText('#7')).not.toBeInTheDocument() - }) - - it('shows ids when selection chips have duplicate titles', () => { - mount({ - selectionTags: [ + it.for([ + { + label: 'unique title', + nodes: [{ id: '5', title: 'KSampler' }], + labels: ['KSampler #5'] + }, + { + label: 'duplicate titles', + nodes: [ { id: '5', title: 'KSampler' }, { id: '7', title: 'KSampler' } - ] - }) + ], + labels: ['KSampler #5', 'KSampler #7'] + } + ])( + 'identifies inline nodes with $label by their IDs', + ({ nodes, labels }) => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes(nodes) + mount({ selectionTags: store.nodes }) + expect( + screen + .getAllByTestId('node-reference-chip') + .map((chip) => chip.textContent) + ).toEqual(labels) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Show KSampler #5 on canvas' }) + ).toBeNull() + } + ) - expect(screen.getAllByText('KSampler')).toHaveLength(2) - expect(screen.getByText('#5')).toBeInTheDocument() - expect(screen.getByText('#7')).toBeInTheDocument() + it('removes an inline node without focusing its canvas node', async () => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) + const { emitted } = mount({ selectionTags: store.nodes }) + await userEvent.click(screen.getByRole('textbox')) + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(emitted().removeTag).toEqual([['5']]) + expect(emitted().focusTag).toBeUndefined() + expect(store.nodes).toEqual([]) }) it('renders an attachment preview and removes it from the composer', async () => { @@ -1741,7 +1841,9 @@ describe('Composer', () => { 'https://example.com/cat.png' ) - await userEvent.click(screen.getByRole('button', { name: 'Remove' })) + await userEvent.click( + screen.getByRole('button', { name: 'Remove cat.png' }) + ) expect(screen.queryByRole('img', { name: 'cat.png' })).toBeNull() }) diff --git a/src/workbench/extensions/agent/components/agent/Composer.vue b/src/workbench/extensions/agent/components/agent/Composer.vue index 8700d0c28af..0b937261235 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.vue +++ b/src/workbench/extensions/agent/components/agent/Composer.vue @@ -1,13 +1,6 @@ + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts new file mode 100644 index 00000000000..307b3ea14fb --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts @@ -0,0 +1,29 @@ +import { render, screen } from '@testing-library/vue' +import { assert, describe, expect, it, vi } from 'vitest' + +import { i18n } from '@/i18n' + +import AssetMediaPreview from './AssetMediaPreview.vue' + +describe('retained asset media previews', () => { + it.for(['video', 'audio'] as const)( + 'stops and resets %s when closed before unmounting', + async (kind) => { + const { rerender } = render(AssetMediaPreview, { + props: { name: 'Media', mediaUrl: '/media', kind, active: true }, + global: { plugins: [i18n] } + }) + const player = screen.getByLabelText('Media', { selector: kind }) + assert.instanceOf(player, HTMLMediaElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await rerender({ active: false }) + expect(player).toBeInTheDocument() + expect(pause).toHaveBeenCalledOnce() + expect(player.currentTime).toBe(0) + await rerender({ active: true }) + expect(screen.getByLabelText('Media', { selector: kind })).toBe(player) + expect(player.currentTime).toBe(0) + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue new file mode 100644 index 00000000000..7df00f0285b --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue @@ -0,0 +1,74 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts new file mode 100644 index 00000000000..ffd754fe187 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts @@ -0,0 +1,44 @@ +import { fireEvent, render, screen } from '@testing-library/vue' +import { describe, expect, it } from 'vitest' + +import { i18n } from '@/i18n' + +import AssetThumbnail from './AssetThumbnail.vue' + +describe('asset poster loading', () => { + it.for(['tray', 'menu', 'inline'] as const)( + 'falls back after a failed poster and retries a replacement URL in the %s', + async (variant) => { + const { rerender } = render(AssetThumbnail, { + props: { + name: 'My video', + mediaKind: 'video', + previewUrl: '/expired.png', + variant + }, + global: { plugins: [i18n] } + }) + const poster = screen.getByAltText(variant === 'tray' ? 'My video' : '') + await fireEvent.error(poster) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + expect( + screen.queryByAltText(variant === 'tray' ? 'My video' : '') + ).not.toBeInTheDocument() + await rerender({ previewUrl: '/replacement.png' }) + const replacement = screen.getByAltText( + variant === 'tray' ? 'My video' : '' + ) + expect(replacement).toHaveAttribute('src', '/replacement.png') + await fireEvent.error(poster) + expect(replacement).toBeInTheDocument() + await fireEvent.error(replacement) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + await fireEvent.error(poster) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + await rerender({ previewUrl: '/expired.png' }) + expect( + screen.getByAltText(variant === 'tray' ? 'My video' : '') + ).toHaveAttribute('src', '/expired.png') + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue new file mode 100644 index 00000000000..af2c4eff2ab --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue @@ -0,0 +1,73 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts new file mode 100644 index 00000000000..710e51fe0bd --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts @@ -0,0 +1,170 @@ +import userEvent from '@testing-library/user-event' +import { render, screen } from '@testing-library/vue' +import { assert, describe, expect, it, onTestFinished, vi } from 'vitest' +import { nextTick } from 'vue' + +import { i18n } from '@/i18n' + +import type { ComposerAttachment } from '../../../types/composerAttachment' +import AssetTray from './AssetTray.vue' + +const images: ComposerAttachment[] = [ + { id: 'cat', name: 'cat.png', ref: 'cat.png', previewUrl: '/cat.png' }, + { id: 'dog', name: 'dog.png', ref: 'dog.png', previewUrl: '/dog.png' } +] + +function renderTray(attachments = images) { + const view = render(AssetTray, { + props: { attachments }, + global: { plugins: [i18n], directives: { tooltip: () => {} } } + }) + onTestFinished(async () => { + view.unmount() + await nextTick() + }) + return { + ...view, + user: userEvent.setup({ advanceTimers: vi.advanceTimersByTimeAsync }) + } +} + +describe('asset tray previews', () => { + it('replaces a clicked preview when another asset is hovered', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.click(first) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + + await user.hover(second) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(first).toHaveAttribute('aria-expanded', 'false') + + await user.unhover(second) + await vi.advanceTimersByTimeAsync(150) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it('keeps the newly clicked preview open after the previous hover settles', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.hover(first) + await vi.advanceTimersByTimeAsync(250) + const preview = screen.getByRole('dialog', { name: 'cat.png' }) + expect(preview).toBeVisible() + await user.hover(screen.getByRole('region', { name: 'cat.png' })) + await vi.advanceTimersByTimeAsync(150) + + await user.click(second) + await vi.advanceTimersByTimeAsync(250) + await user.unhover(second) + await vi.advanceTimersByTimeAsync(150) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(preview).not.toBeInTheDocument() + + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + expect(second).toHaveFocus() + await user.hover(first) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + }) + + it('switches keyboard activation and returns focus to the current trigger on Escape', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + first.focus() + await user.keyboard('{Enter}') + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + second.focus() + await user.keyboard(' ') + + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + expect(second).toHaveFocus() + }) + + it('switches previews on touch activation without depending on hover', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.pointer([ + { keys: '[TouchA>]', target: first }, + { keys: '[/TouchA]', target: first } + ]) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await user.pointer([ + { keys: '[TouchA>]', target: second }, + { keys: '[/TouchA]', target: second } + ]) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + }) + + it('does not reopen a removed active preview when the same attachment returns', async () => { + const { user, rerender } = renderTray() + await user.click(screen.getByRole('button', { name: 'Preview cat.png' })) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await rerender({ attachments: [images[1]] }) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + + await rerender({ attachments: images }) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + }) + + it('keeps typing focus and text while switching hover previews', async () => { + const { user } = renderTray() + const promptView = render({ template: '' }) + onTestFinished(promptView.unmount) + const prompt = screen.getByRole('textbox', { name: 'Prompt' }) + await user.type(prompt, 'Keep typing') + await user.hover(screen.getByRole('button', { name: 'Preview cat.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(prompt).toHaveFocus() + await user.keyboard(' here') + expect(prompt).toHaveValue('Keep typing here') + }) + + it.for([ + { name: 'song.mp3', mediaKind: 'audio', selector: 'audio' }, + { name: 'clip.mp4', mediaKind: 'video', selector: 'video' } + ] as const)( + 'stops $mediaKind playback when a hovered asset replaces its interactive preview', + async ({ name, mediaKind, selector }) => { + const { user } = renderTray([ + { id: 'media', name, ref: name, mediaUrl: '/media', mediaKind }, + images[1] + ]) + await user.click(screen.getByRole('button', { name: `Preview ${name}` })) + const player = screen.getByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(player.currentTime).toBe(0) + expect(pause).toHaveBeenCalled() + expect(player).not.toBeInTheDocument() + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue b/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue new file mode 100644 index 00000000000..1860beecc76 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue @@ -0,0 +1,137 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts index 6180dc23192..5aad2a9e65f 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts @@ -1,6 +1,15 @@ -import { render, screen } from '@testing-library/vue' -import { describe, expect, it } from 'vitest' +import userEvent from '@testing-library/user-event' +import { + fireEvent, + render, + screen, + waitFor, + within +} from '@testing-library/vue' +import { assert, describe, expect, it, onTestFinished, vi } from 'vitest' +import { nextTick } from 'vue' +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' import { i18n } from '@/i18n' import AttachmentChip from './AttachmentChip.vue' @@ -8,12 +17,19 @@ import AttachmentChip from './AttachmentChip.vue' function renderChip(props: { name: string previewUrl?: string + mediaUrl?: string + mediaKind?: MediaKind uploading?: boolean }) { - return render(AttachmentChip, { + const rendered = render(AttachmentChip, { props, global: { plugins: [i18n] } }) + onTestFinished(async () => { + rendered.unmount() + await nextTick() + }) + return rendered } function iconMarker(container: Element): string { @@ -25,6 +41,173 @@ function iconMarker(container: Element): string { } describe('AttachmentChip', () => { + it.for([ + { name: 'cat.png', previewUrl: 'blob:cat', key: '{Enter}' }, + { name: 'notes.txt', previewUrl: undefined, key: ' ' } + ])( + 'discloses $name from the keyboard and restores focus on Escape', + async ({ key, ...props }) => { + const user = userEvent.setup() + renderChip(props) + expect(screen.queryByText(props.name)).not.toBeInTheDocument() + await user.tab() + const trigger = screen.getByRole('button', { + name: `Preview ${props.name}` + }) + expect(trigger).toHaveFocus() + await user.keyboard(key) + const preview = await screen.findByRole('dialog', { name: props.name }) + expect(within(preview).getByText(props.name)).toBeVisible() + await user.keyboard('{Escape}') + await waitFor(() => expect(preview).not.toBeInTheDocument()) + expect(trigger).toHaveFocus() + } + ) + + it('names the removal action and keeps it separate from preview activation', async () => { + const user = userEvent.setup() + const { emitted } = renderChip({ name: 'notes.txt' }) + await user.click(screen.getByRole('button', { name: 'Remove notes.txt' })) + expect(emitted().remove).toHaveLength(1) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it.for([ + { name: 'clip.mp4', selector: 'video', key: '{Enter}' }, + { name: 'song.mp3', selector: 'audio', key: ' ' } + ])( + 'opens $name playback from the keyboard, focuses native controls and restores focus on Escape', + async ({ name, selector, key }) => { + const user = userEvent.setup() + renderChip({ name, mediaUrl: '/playable-media' }) + const trigger = screen.getByRole('button', { name: `Preview ${name}` }) + await user.tab() + expect(trigger).toHaveFocus() + await user.keyboard(key) + expect(await screen.findByRole('dialog', { name })).toBeVisible() + await waitFor(() => + expect(trigger).toHaveAttribute('aria-expanded', 'true') + ) + const player = screen.getByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + await waitFor(() => expect(player).toHaveFocus()) + expect(player).toHaveAttribute('controls') + expect(player).not.toHaveAttribute('tabindex', '-1') + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.keyboard('{Escape}') + await waitFor(() => expect(player).not.toBeInTheDocument()) + await waitFor(() => expect(trigger).toHaveFocus()) + expect(trigger).toHaveAttribute('aria-expanded', 'false') + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + await user.keyboard(key) + expect(await screen.findByLabelText(name, { selector })).toHaveAttribute( + 'src', + '/playable-media' + ) + } + ) + + it('keeps removal independent of opening media playback', async () => { + const user = userEvent.setup() + const { emitted } = renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + await user.click(screen.getByRole('button', { name: 'Remove clip.mp4' })) + expect(emitted().remove).toHaveLength(1) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it('promotes a hovered preview to keyboard playback and keeps it open after the pointer leaves', async () => { + const user = userEvent.setup() + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + expect(trigger).toHaveFocus() + await user.hover(trigger) + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + expect(trigger).toHaveFocus() + await user.keyboard('{Enter}') + expect(player).toHaveFocus() + await user.unhover(trigger) + await user.pointer({ target: document.body }) + expect(screen.getByRole('dialog', { name: 'clip.mp4' })).toBeVisible() + await user.keyboard('{Escape}') + await waitFor(() => + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + ) + expect(trigger).toHaveFocus() + }) + + it('preserves typing focus when a hover preview opens and closes', async () => { + const user = userEvent.setup() + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const promptView = render({ template: '' }) + onTestFinished(promptView.unmount) + const prompt = screen.getByRole('textbox', { name: 'Prompt' }) + await user.click(prompt) + await user.type(prompt, 'Keep typing') + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.hover(trigger) + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + expect(prompt).toHaveFocus() + await user.unhover(trigger) + await user.pointer({ target: document.body }) + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(prompt).toHaveFocus() + expect(prompt).toHaveValue('Keep typing') + }) + + it('keeps keyboard playback focused when its source is replaced and clears an earlier load error', async () => { + const user = userEvent.setup() + const { rerender } = renderChip({ name: 'clip.mp4', mediaUrl: 'blob:clip' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + await user.keyboard('{Enter}') + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + assert.instanceOf(player, HTMLVideoElement) + await waitFor(() => expect(player).toHaveFocus()) + await fireEvent.error(player) + expect(screen.getByText('Video failed to load')).toBeVisible() + const pause = vi.spyOn(player, 'pause') + await rerender({ mediaUrl: '/uploaded.mp4' }) + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveFocus() + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveAttribute('src', '/uploaded.mp4') + expect(screen.queryByText('Video failed to load')).not.toBeInTheDocument() + expect(pause).toHaveBeenCalledOnce() + await user.keyboard('{Escape}') + await waitFor(() => + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + ) + expect(trigger).toHaveFocus() + }) + + it.for([ + { name: 'cat.png', previewUrl: 'blob:cat' }, + { name: 'song.mp3', previewUrl: undefined } + ])( + 'shows $name only on hover while retaining an accessible tray item', + async (props) => { + const user = userEvent.setup() + renderChip(props) + expect(screen.queryByText(props.name)).not.toBeInTheDocument() + await user.hover( + screen.getByRole('button', { name: `Preview ${props.name}` }) + ) + const preview = await screen.findByRole('dialog', { name: props.name }) + expect(within(preview).getByText(props.name)).toBeVisible() + } + ) + it('renders an image preview only for image files', () => { renderChip({ name: 'cat.png', previewUrl: 'blob:x' }) expect(screen.getByAltText('cat.png')).toBeInTheDocument() @@ -37,7 +220,8 @@ describe('AttachmentChip', () => { name: 'song.mp3', previewUrl: 'https://x/thumb.png' }) - expect(screen.queryByRole('img')).not.toBeInTheDocument() + expect(screen.queryByAltText('song.mp3')).not.toBeInTheDocument() + expect(screen.getByRole('img', { name: 'Audio' })).toBeInTheDocument() expect(iconMarker(container)).toContain('lucide--music') }) @@ -52,6 +236,175 @@ describe('AttachmentChip', () => { expect(iconMarker(container)).toContain(icon) }) + it('shows a video poster while keeping the playable file separate', async () => { + const user = userEvent.setup() + renderChip({ + name: 'My clip', + mediaKind: 'video', + previewUrl: '/poster.png', + mediaUrl: '/clip.mp4' + }) + expect(screen.getByRole('img', { name: 'My clip' })).toHaveAttribute( + 'src', + '/poster.png' + ) + await user.hover(screen.getByRole('button', { name: 'Preview My clip' })) + const player = await screen.findByLabelText('My clip', { + selector: 'video' + }) + expect(player).toHaveAttribute('src', '/clip.mp4') + expect(player).toHaveAttribute('poster', '/poster.png') + expect(player).toHaveAttribute('controls') + expect(player).not.toHaveAttribute('autoplay') + }) + + it('does not decode another video while waiting for the shared poster', () => { + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + expect( + screen.queryByLabelText('clip.mp4', { selector: 'video' }) + ).not.toBeInTheDocument() + expect(screen.getByRole('img', { name: 'Video' })).toBeInTheDocument() + }) + + it.for([ + { name: 'clip.mp4', mediaKind: 'video', selector: 'video' }, + { name: 'song.mp3', mediaKind: 'audio', selector: 'audio' } + ] as const)( + 'keeps $mediaKind controls usable inside hover preview and stops on close', + async ({ name, mediaKind, selector }) => { + const user = userEvent.setup() + renderChip({ + name, + mediaKind, + mediaUrl: '/playable-media', + previewUrl: '/poster.png' + }) + const trigger = screen.getByRole('button', { name: `Preview ${name}` }) + await user.hover(trigger) + const player = await screen.findByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + expect(player).toHaveAttribute('controls') + expect(player).toHaveAttribute('src', '/playable-media') + expect(player).not.toHaveAttribute('autoplay') + await user.unhover(trigger) + await user.hover(player) + expect(screen.getByRole('region', { name })).toBeVisible() + expect(player).toBeInTheDocument() + await fireEvent.play(player) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.keyboard('{Escape}') + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + } + ) + + it('keeps hover usable after keyboard and hovered-content dismissals', async () => { + vi.useFakeTimers() + onTestFinished(() => { + vi.useRealTimers() + }) + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTimeAsync }) + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + await user.keyboard('{Enter}') + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveFocus() + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + const first = screen.getByLabelText('clip.mp4', { selector: 'video' }) + await user.hover(first) + await vi.advanceTimersByTimeAsync(150) + await user.keyboard('{Escape}') + await vi.advanceTimersByTimeAsync(150) + expect(first).not.toBeInTheDocument() + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + const second = screen.getByLabelText('clip.mp4', { selector: 'video' }) + await user.unhover(trigger) + await user.pointer({ target: document.body }) + await vi.advanceTimersByTimeAsync(150) + expect(second).not.toBeInTheDocument() + // Escape also cancels a pointer hover whose delayed opening is still pending. + await user.hover(trigger) + await user.keyboard('{Enter}') + await user.keyboard('{Escape}') + await vi.advanceTimersByTimeAsync(250) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.unhover(trigger) + await vi.advanceTimersByTimeAsync(150) + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'clip.mp4' })).toBeVisible() + }) + + it('stops audio after the pointer leaves the preview', async () => { + const user = userEvent.setup() + renderChip({ name: 'song.mp3', mediaUrl: '/song.mp3' }) + const trigger = screen.getByRole('button', { name: 'Preview song.mp3' }) + await user.hover(trigger) + const player = await screen.findByLabelText('song.mp3', { + selector: 'audio' + }) + assert.instanceOf(player, HTMLAudioElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.unhover(trigger) + await user.pointer({ + target: document.body, + coords: { clientX: 500, clientY: 500 } + }) + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + }) + + it('stops an audio preview on replacement and removal', async () => { + const user = userEvent.setup() + const { rerender, unmount } = renderChip({ + name: 'song.mp3', + mediaUrl: 'blob:song' + }) + await user.hover(screen.getByRole('button', { name: 'Preview song.mp3' })) + const first = await screen.findByLabelText('song.mp3', { + selector: 'audio' + }) + assert.instanceOf(first, HTMLAudioElement) + const pauseFirst = vi.spyOn(first, 'pause') + await rerender({ name: 'song.mp3', mediaUrl: '/stored-song.mp3' }) + const replacement = screen.getByLabelText('song.mp3', { selector: 'audio' }) + assert.instanceOf(replacement, HTMLAudioElement) + expect(replacement).toHaveAttribute('src', '/stored-song.mp3') + expect(pauseFirst).toHaveBeenCalledOnce() + unmount() + expect(pauseFirst).toHaveBeenCalledTimes(2) + }) + + it.for([ + { name: 'clip.mp4', selector: 'video', message: 'Video failed to load' }, + { name: 'song.mp3', selector: 'audio', message: 'Audio failed to load' } + ])( + 'reports an unplayable $name without losing the asset name', + async ({ name, selector, message }) => { + const user = userEvent.setup() + renderChip({ + name, + mediaUrl: '/unsupported-media', + previewUrl: '/poster.png' + }) + await user.hover(screen.getByRole('button', { name: `Preview ${name}` })) + const player = await screen.findByLabelText(name, { selector }) + await fireEvent.error(player) + expect(screen.getByText(message)).toBeVisible() + expect(screen.getByText(name)).toBeVisible() + } + ) + it('shows the spinner while uploading', () => { renderChip({ name: 'cat.png', uploading: true }) expect( diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue index 08a88c5469f..9ff5dbc4612 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue @@ -1,60 +1,82 @@ diff --git a/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue b/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue new file mode 100644 index 00000000000..323d68d572c --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue @@ -0,0 +1,155 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue b/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue new file mode 100644 index 00000000000..d24429696ae --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue @@ -0,0 +1,55 @@ + + diff --git a/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts new file mode 100644 index 00000000000..284b4f98906 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts @@ -0,0 +1,55 @@ +import { render, screen } from '@testing-library/vue' +import { describe, expect, it } from 'vitest' + +import { i18n } from '@/i18n' + +import type { ComposerAttachment } from '../../../types/composerAttachment' +import InlineAssetReference from './InlineAssetReference.vue' + +function renderReference( + props: Pick +) { + return render(InlineAssetReference, { + props: { ...props, removeLabel: `Remove ${props.name} reference` }, + global: { plugins: [i18n] } + }) +} + +describe('inline asset media', () => { + it.for([ + { name: 'cat.png', mediaKind: 'image' }, + { name: 'My video', mediaKind: 'video' } + ] as const)( + 'shows the image/poster for $mediaKind', + ({ name, mediaKind }) => { + renderReference({ + name, + mediaKind, + previewUrl: '/poster.png' + }) + expect(screen.getByAltText('')).toHaveAttribute('src', '/poster.png') + expect(screen.getByText(name)).toBeVisible() + } + ) + + it('uses a video icon while awaiting the shared poster', () => { + renderReference({ + name: 'My video', + mediaKind: 'video' + }) + expect(screen.getByRole('img', { name: 'Video' })).toBeInTheDocument() + }) + + it.for([ + { name: 'recording', mediaKind: 'audio', label: 'Audio' }, + { name: 'song.mp3', mediaKind: undefined, label: 'Audio' }, + { name: 'My video', mediaKind: 'video', label: 'Video' } + ] as const)( + 'identifies $name by its media type when no thumbnail is available', + ({ name, mediaKind, label }) => { + renderReference({ name, mediaKind }) + expect(screen.getByRole('img', { name: label })).toBeInTheDocument() + expect(screen.getByText(name)).toBeVisible() + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue new file mode 100644 index 00000000000..a19d0efa243 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue @@ -0,0 +1,64 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue b/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue index c9258ddd5ab..5a5f1387799 100644 --- a/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue +++ b/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue @@ -6,17 +6,18 @@ import { closeHistory, history, redo, undo } from '@tiptap/pm/history' import { keymap } from '@tiptap/pm/keymap' import { EditorState, TextSelection } from '@tiptap/pm/state' import { Decoration, DecorationSet, EditorView } from '@tiptap/pm/view' -import { onBeforeUnmount, onMounted, useTemplateRef, watch } from 'vue' +import { + getCurrentInstance, + h, + onBeforeUnmount, + onMounted, + render, + useTemplateRef, + watch +} from 'vue' import { default as DOMPurify } from 'dompurify' import { useI18n } from 'vue-i18n' -import { buttonVariants } from '@comfyorg/design-system/button.variants' -import { - tagRemoveButtonVariants, - tagVariants -} from '@comfyorg/design-system/tag.variants' -import { cn } from '@comfyorg/tailwind-utils' - import type { ComposerPrompt } from '../../../types/composerPrompt' import { composerReferenceKey, @@ -31,6 +32,13 @@ import { import { selectedNodeKey } from '../../../composables/agent/useCanvasSelection' import type { PromptEditor } from '../../../types/promptEditor' import type { WorkflowReferenceMetadata } from '../../../types/workflowReference' +import InlineAssetReference from './InlineAssetReference.vue' +import { + inlineReferenceChipClass, + inlineReferenceRemoveAnchorClass, + inlineReferenceRemoveBadgeClass, + inlineReferenceRemoveButtonClass +} from './inlineReferenceChipStyles' import { inlinePromptSchema, promptDocument, @@ -69,10 +77,13 @@ const emit = defineEmits<{ openReferenceWorkflow: [id: string, name: string] removeWorkflowReference: [id: string] removeNodeReference: [id: string] + highlightAssets: [ids: string[]] }>() const { t } = useI18n() +const appContext = getCurrentInstance()?.appContext const host = useTemplateRef('host') let view: EditorView | undefined +const activeAssetReferences = new Map() const insertions = new Set<{ from: number; to: number }>() const plugins = [ history(), @@ -174,21 +185,93 @@ function insertPastedContent( ) } -function passiveReferenceView(node: Node, iconClass: string) { +function assetReferenceView( + node: Node, + editor: EditorView, + getPos: () => number | undefined +) { const dom = document.createElement('span') const reference = promptNodeReference(node, 0) - if (!reference) return { dom } + if (reference?.kind !== 'asset') return { dom } + dom.contentEditable = 'false' + dom.className = + 'inline-flex rounded-sm align-middle ring-offset-base-background [&.ProseMirror-selectednode]:ring-2 [&.ProseMirror-selectednode]:ring-base-foreground [&.ProseMirror-selectednode]:ring-offset-1' + const vnode = h(InlineAssetReference, { + name: reference.attachment.name, + previewUrl: reference.attachment.previewUrl, + mediaKind: reference.attachment.mediaKind, + removeLabel: t('agent.removeAssetReference', { + name: reference.attachment.name + }), + onHighlight: (active: boolean) => { + if (active) activeAssetReferences.set(dom, reference.attachment.id) + else if (!activeAssetReferences.delete(dom)) return + emit('highlightAssets', [...new Set(activeAssetReferences.values())]) + }, + onRemove: () => { + const position = getPos() + if (position === undefined) return + editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() + } + }) + vnode.appContext = appContext ?? null + render(vnode, dom) + return { + dom, + stopEvent: () => true, + ignoreMutation: () => true, + destroy: () => render(null, dom) + } +} + +function nodeReferenceView( + node: Node, + editor: EditorView, + getPos: () => number | undefined +) { + const dom = document.createElement('span') + const reference = promptNodeReference(node, 0) + if (reference?.kind !== 'node') return { dom } dom.contentEditable = 'false' dom.dataset.testid = `${reference.kind}-reference-chip` - dom.className = cn(tagVariants(), 'align-middle') + dom.className = inlineReferenceChipClass + dom.tabIndex = 0 const icon = document.createElement('span') - icon.className = `${iconClass} size-3 shrink-0` + icon.className = 'icon-[comfy--node] size-3 shrink-0' icon.setAttribute('aria-hidden', 'true') const label = document.createElement('span') label.className = 'min-w-0 max-w-56 truncate' - label.textContent = composerReferenceName(reference) + label.textContent = reference.node.title dom.append(icon, label) - return { dom, ignoreMutation: () => true } + const id = document.createElement('span') + id.className = 'shrink-0 text-muted-foreground' + id.textContent = ` #${reference.node.id}` + const remove = document.createElement('button') + remove.type = 'button' + remove.setAttribute( + 'aria-label', + t('agent.removeNodeLabel', { node: composerReferenceName(reference) }) + ) + const removeAnchor = document.createElement('span') + removeAnchor.className = inlineReferenceRemoveAnchorClass + remove.className = inlineReferenceRemoveButtonClass + const badge = document.createElement('span') + badge.className = inlineReferenceRemoveBadgeClass + const cross = document.createElement('span') + cross.className = 'icon-[lucide--x] size-2' + cross.setAttribute('aria-hidden', 'true') + badge.append(cross) + remove.append(badge) + remove.onclick = () => { + const position = getPos() + if (position === undefined) return + editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() + } + removeAnchor.append(remove) + dom.append(id, removeAnchor) + return { dom, stopEvent: () => true, ignoreMutation: () => true } } onMounted(() => { @@ -317,8 +400,6 @@ onMounted(() => { const files = attachableClipboardFiles(clipboard) const text = clipboard.getData('text/plain') const attachmentsOnly = clipboard.files.length > 0 && text === '' - // Attaching rewrites the prompt through the store, so the document edit - // has to land first or the editor overwrites the staged attachment. if (!attachmentsOnly) insertPastedContent(editor, clipboard, slice, text) if (files.length > 0) emit('attachFiles', files) return true @@ -328,8 +409,8 @@ onMounted(() => { referenceClipboardText(node) ), nodeViews: { - node: (node) => passiveReferenceView(node, 'icon-[comfy--node]'), - asset: (node) => passiveReferenceView(node, 'icon-[lucide--paperclip]'), + node: (node, editor, getPos) => nodeReferenceView(node, editor, getPos), + asset: assetReferenceView, workflow(node, editor, getPos) { const id: unknown = node.attrs.id const name: unknown = node.attrs.name @@ -337,10 +418,7 @@ onMounted(() => { if (typeof id !== 'string' || typeof name !== 'string') return { dom } dom.contentEditable = 'false' dom.dataset.testid = 'workflow-reference-chip' - dom.className = cn( - tagVariants({ interactive: true, removable: true }), - 'group/workflow align-middle' - ) + dom.className = inlineReferenceChipClass const open = document.createElement('button') open.type = 'button' open.tabIndex = 0 @@ -372,21 +450,16 @@ onMounted(() => { if (!unavailable) emit('openReferenceWorkflow', id, name) } const removeAnchor = document.createElement('span') - removeAnchor.className = 'relative inline-block h-4 w-0 align-middle' + removeAnchor.className = inlineReferenceRemoveAnchorClass const remove = document.createElement('button') remove.type = 'button' remove.setAttribute( 'aria-label', t('agent.removeWorkflowReference', { name }) ) - remove.className = cn( - buttonVariants({ variant: 'textonly', size: 'icon-sm' }), - tagRemoveButtonVariants(), - 'pointer-events-none absolute -top-2 -right-2 z-10 flex size-5 cursor-pointer items-center justify-center rounded-full p-0 text-base-foreground opacity-0 transition-opacity group-focus-within/workflow:pointer-events-auto group-focus-within/workflow:opacity-100 group-hover/workflow:pointer-events-auto group-hover/workflow:opacity-100 touch:pointer-events-auto touch:opacity-100' - ) + remove.className = inlineReferenceRemoveButtonClass const badge = document.createElement('span') - badge.className = - 'flex size-3 items-center justify-center rounded-full bg-base-background ring-1 ring-border-default hover:bg-secondary-background-hover' + badge.className = inlineReferenceRemoveBadgeClass const cross = document.createElement('span') cross.className = 'icon-[lucide--x] size-2' badge.append(cross) @@ -395,6 +468,7 @@ onMounted(() => { const position = getPos() if (position === undefined) return editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() } removeAnchor.append(remove) dom.append(open, removeAnchor) diff --git a/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue b/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue new file mode 100644 index 00000000000..dd05a61ca51 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue @@ -0,0 +1,101 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts index 53185944a5c..63889619692 100644 --- a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts +++ b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts @@ -72,6 +72,31 @@ describe('inline prompt', () => { } ) + it.for(['video', 'audio'] as const)( + 'retains $mediaKind preview metadata when round-tripping inline assets', + (mediaKind) => { + const draft: ComposerPrompt = { + text: 'Inspect ', + references: [ + { + kind: 'asset', + textOffset: 8, + attachment: { + id: 'media', + name: 'Renamed media', + ref: 'stored-file', + mediaKind, + mediaUrl: 'blob:media', + previewUrl: '/poster.png', + uploading: true + } + } + ] + } + expect(promptDraft(promptDocument(draft))).toEqual(draft) + } + ) + it('restores references before, between and after text, including adjacent tokens', () => { const draft: ComposerPrompt = { text: 'before 😀\nafter', diff --git a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts index b9c285f0fa0..c1b8d1567c1 100644 --- a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts +++ b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts @@ -1,6 +1,7 @@ import { Schema } from '@tiptap/pm/model' import type { Node } from '@tiptap/pm/model' +import { zMediaKindSchema } from '@/platform/assets/schemas/mediaAssetSchema' import { isNodeLocatorId } from '@/types/nodeIdentification' import type { ComposerInsertionPoint, @@ -68,6 +69,8 @@ export const inlinePromptSchema = new Schema({ name: {}, ref: {}, previewUrl: { default: null }, + mediaUrl: { default: null }, + mediaKind: { default: null }, uploading: { default: false } }, toDOM: (node) => [ @@ -117,6 +120,30 @@ export function promptDocument(prompt: ComposerPrompt): Node { return inlinePromptSchema.nodes.doc.create(null, content) } +function assetNodeReference( + node: Node, + id: string, + name: string, + textOffset: number +): ComposerReference | undefined { + const { ref, previewUrl, mediaUrl, uploading } = node.attrs + const mediaKind = zMediaKindSchema.safeParse(node.attrs.mediaKind).data + if (typeof ref !== 'string') return + return { + kind: 'asset', + textOffset, + attachment: { + id, + name, + ref, + ...(typeof previewUrl === 'string' ? { previewUrl } : {}), + ...(typeof mediaUrl === 'string' ? { mediaUrl } : {}), + ...(mediaKind ? { mediaKind } : {}), + ...(uploading === true ? { uploading: true } : {}) + } + } +} + export function promptNodeReference( node: Node, textOffset: number @@ -145,21 +172,8 @@ export function promptNodeReference( } } } - if (node.type.name === 'asset') { - const { ref, previewUrl, uploading } = node.attrs - if (typeof ref !== 'string') return - return { - kind: 'asset', - textOffset, - attachment: { - id, - name, - ref, - ...(typeof previewUrl === 'string' ? { previewUrl } : {}), - ...(uploading === true ? { uploading: true } : {}) - } - } - } + if (node.type.name === 'asset') + return assetNodeReference(node, id, name, textOffset) } export function promptDraft(doc: Node): ComposerPrompt { diff --git a/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts b/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts new file mode 100644 index 00000000000..18217a4ac31 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts @@ -0,0 +1,20 @@ +import { buttonVariants } from '@comfyorg/design-system/button.variants' +import { + tagRemoveButtonVariants, + tagVariants +} from '@comfyorg/design-system/tag.variants' +import { cn } from '@comfyorg/tailwind-utils' + +export const inlineReferenceChipClass = cn( + tagVariants({ interactive: true, removable: true }), + 'group/inline-reference h-4 p-0 align-middle' +) +export const inlineReferenceRemoveAnchorClass = + 'relative inline-block h-4 w-0 align-middle' +export const inlineReferenceRemoveButtonClass = cn( + buttonVariants({ variant: 'textonly', size: 'icon-sm' }), + tagRemoveButtonVariants(), + 'pointer-events-none absolute -top-2 -right-2 z-10 flex size-5 cursor-pointer items-center justify-center rounded-full p-0 text-base-foreground opacity-0 transition-opacity group-focus-within/inline-reference:pointer-events-auto group-focus-within/inline-reference:opacity-100 group-hover/inline-reference:pointer-events-auto group-hover/inline-reference:opacity-100 touch:pointer-events-auto touch:opacity-100' +) +export const inlineReferenceRemoveBadgeClass = + 'flex size-3 items-center justify-center rounded-full bg-base-background ring-1 ring-border-default hover:bg-secondary-background-hover' diff --git a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts index 4e0dc088059..bfbf6753295 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts @@ -11,7 +11,7 @@ import type { SubmissionMeta } from './useAgentDraftSubmission' import { useAgentDraftSubmission } from './useAgentDraftSubmission' import type { SelectedNode } from './useCanvasSelection' import { useCanvasSelection } from './useCanvasSelection' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' type ComposerStore = ReturnType type Send = ( @@ -195,6 +195,37 @@ describe('Agent draft submission', () => { expect(revoke).not.toHaveBeenCalled() }) + it('retains media through a pending send and failed recovery, releasing it on success', async () => { + const { composer, submit, send, pending } = setup() + const revoke = vi.spyOn(URL, 'revokeObjectURL') + composer.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: 'clip.mp4', + mediaKind: 'video', + mediaUrl: 'blob:video' + }) + composer.referenceAttachment('video') + const sending = submit() + composer.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + pending.resolve(false) + await sending + await nextTick() + expect(composer.attachments).toContainEqual({ + id: 'video', + name: 'clip.mp4', + ref: 'clip.mp4', + mediaKind: 'video', + mediaUrl: 'blob:video' + }) + expect(revoke).not.toHaveBeenCalled() + send.mockResolvedValue(true) + await submit() + expect(composer.attachments).toEqual([]) + expect(revoke.mock.calls).toEqual([['blob:cat'], ['blob:video']]) + }) + it('does not recover over an edit that was subsequently cleared', async () => { const { composer, selection, submit, pending } = setup() const sending = submit() @@ -209,6 +240,42 @@ describe('Agent draft submission', () => { expect(selection.staged.value).toEqual([]) }) + it('restores repeated mentions and unmentioned assets with their original inline positions after failure', async () => { + const { composer, submit, send } = setup() + composer.referenceAttachment('upload-1') + composer.referenceAttachment('upload-1') + composer.addAttachment({ + id: 'upload-2', + name: 'background.png', + ref: 'background.png' + }) + const prompt = composer.prompt + const attachments = [...composer.attachments] + send.mockResolvedValue(false) + + await submit() + + expect(composer.prompt).toEqual(prompt) + expect(composer.attachments).toEqual(attachments) + expect( + composer.prompt.references.filter((item) => item.kind === 'asset') + ).toHaveLength(2) + }) + + it('keeps a newer tray attachment instead of restoring a failed submission', async () => { + const { composer, submit, pending } = setup() + const sending = submit() + composer.addAttachment({ id: 'new', name: 'new.png', ref: 'new.png' }) + pending.resolve(false) + + await sending + + expect(composer.prompt).toEqual({ text: '', references: [] }) + expect(composer.attachments).toEqual([ + { id: 'new', name: 'new.png', ref: 'new.png' } + ]) + }) + it.for(['pending', 'failed'])( 'keeps edits made while the panel is closed when the submission is %s', async (phase) => { diff --git a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts index e22d55f0334..ab998b069be 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts @@ -9,7 +9,7 @@ import type { WorkflowReference } from '../../types/workflowReference' import type { AgentStarterPromptSource } from '../../utils/starterPrompts' import type { SelectedNode, useCanvasSelection } from './useCanvasSelection' import { selectedNodeKey } from './useCanvasSelection' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' interface UseAgentDraftSubmissionOptions { canSubmit: () => boolean @@ -58,22 +58,25 @@ export function useAgentDraftSubmission( const snapshot = composer.takeFailedSubmission() if (!snapshot || selection.staged.value.length > 0) return - composer.restorePrompt({ - text: snapshot.prompt.text, - references: snapshot.prompt.references.filter((reference) => { - if (reference.kind === 'workflow') - return reference.id !== options.editableWorkflowId() - if (reference.kind === 'node') - return ( - options.target() === snapshot.target && - snapshot.nodes.some( - (node) => - selectedNodeKey(node) === selectedNodeKey(reference.node) + composer.restorePrompt( + { + text: snapshot.prompt.text, + references: snapshot.prompt.references.filter((reference) => { + if (reference.kind === 'workflow') + return reference.id !== options.editableWorkflowId() + if (reference.kind === 'node') + return ( + options.target() === snapshot.target && + snapshot.nodes.some( + (node) => + selectedNodeKey(node) === selectedNodeKey(reference.node) + ) ) - ) - return true - }) - }) + return true + }) + }, + snapshot.attachments + ) if (options.target() === snapshot.target) selection.replace(snapshot.nodes) } diff --git a/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts b/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts index 02f111cca44..5492f1e677f 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts @@ -11,12 +11,14 @@ import type { import { transitionMentionPicker } from './mentionPickerState' import type { SelectedNode } from './useCanvasSelection' import { selectedNodeKey } from './useCanvasSelection' +import type { ComposerAttachment } from '../../types/composerAttachment' interface MentionPickerOptions { draft: () => string editor: () => PromptEditor | null selectionTags: () => SelectedNode[] workflows: () => WorkflowReferenceOption[] + assets: () => ComposerAttachment[] nodeReferenceDisabledReason: () => string | undefined workflowSelecting: () => boolean getMentionNodes: () => SelectedNode[] @@ -26,21 +28,19 @@ interface MentionPickerOptions { to: number ) => Promise pickNode: (node: SelectedNode) => void + pickAsset: (asset: ComposerAttachment) => void requestWorkflows: () => void } export function useAgentMentionPicker(options: MentionPickerOptions) { const { t } = useI18n() - const graphNodes = ref([]) - function loadMentionNodes(): void { - if (options.nodeReferenceDisabledReason()) { - graphNodes.value = [] - return - } - graphNodes.value = [...options.getMentionNodes()].sort((a, b) => - a.title.localeCompare(b.title) - ) - } + const graphNodes = computed(() => + options.nodeReferenceDisabledReason() + ? [] + : [...options.getMentionNodes()].sort((a, b) => + a.title.localeCompare(b.title) + ) + ) const mention = ref({ status: 'closed' }) const mentionSection = computed(() => @@ -57,6 +57,7 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { | { kind: 'section'; id: 'nodes' | 'workflows'; label: string } | { kind: 'back'; id: 'back'; label: string } | { kind: 'node'; id: string; label: string; node: SelectedNode } + | { kind: 'asset'; id: string; label: string; asset: ComposerAttachment } | { kind: 'workflow' id: string @@ -78,9 +79,20 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { { kind: 'section', id: 'nodes', label: t('agent.nodes') }, { kind: 'section', id: 'workflows', label: t('agent.workflows') } ] - return sections.filter(({ label }) => - label.toLowerCase().includes(search) - ) + return [ + ...options + .assets() + .filter((asset) => asset.name.toLowerCase().includes(search)) + .map( + (asset): MentionMatch => ({ + kind: 'asset', + id: asset.id, + label: asset.name, + asset + }) + ), + ...sections.filter(({ label }) => label.toLowerCase().includes(search)) + ] } const back: MentionMatch = { kind: 'back', id: 'back', label: t('g.back') } @@ -149,15 +161,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { } const graphDupes = computed(() => duplicatedTitles(graphNodes.value)) - const tagDupes = computed(() => duplicatedTitles(options.selectionTags())) - - watch( - () => options.selectionTags(), - (tags) => { - if (tags.length) loadMentionNodes() - }, - { immediate: true } - ) function dispatchMention(event: MentionPickerEvent): void { mention.value = transitionMentionPicker(mention.value, event) @@ -187,7 +190,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { dispatchMention({ type: 'closed' }) return } - if (mention.value.status === 'closed') loadMentionNodes() dispatchMention({ type: 'queryChanged', start: at, @@ -207,11 +209,7 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { watch( () => options.nodeReferenceDisabledReason(), (reason) => { - if (!reason) { - if (mention.value.status === 'open') loadMentionNodes() - return - } - graphNodes.value = [] + if (!reason) return if ( mention.value.status === 'open' && mention.value.section === 'nodes' @@ -225,15 +223,25 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { { flush: 'sync' } ) + function selectMentionSection( + section: Exclude, + from: number, + to: number + ): void { + options.editor()?.replaceText(from, to, '') + dispatchMention({ type: 'sectionSelected', section }) + if (section === 'workflows') options.requestWorkflows() + } + async function pickMention(match: MentionMatch): Promise { const state = mention.value if (state.status === 'closed' || isMentionDisabled(match)) return if (match.kind === 'section') { - options - .editor() - ?.replaceText(state.start + 1, state.start + 1 + state.query.length, '') - dispatchMention({ type: 'sectionSelected', section: match.id }) - if (match.id === 'workflows') options.requestWorkflows() + selectMentionSection( + match.id, + state.start + 1, + state.start + 1 + state.query.length + ) return } if (match.kind === 'back') { @@ -254,7 +262,8 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { options .editor() ?.replaceText(state.start, state.start + 1 + state.query.length, '') - options.pickNode(match.node) + if (match.kind === 'asset') options.pickAsset(match.asset) + else options.pickNode(match.node) dispatchMention({ type: 'closed' }) options.editor()?.focus() } @@ -314,7 +323,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { mentionVisible, mentionHasResults, graphDupes, - tagDupes, syncMention, pickMention, isNodeReferenceDisabled, diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts index 3ac080aa746..ba6b1feecfd 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts @@ -5,7 +5,8 @@ import { AgentApiError, AgentResponseUnreadableError } from '../../services/agent/agentRestClient' -import type { ComposerAttachment } from './useComposer' +import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' +import type { ComposerAttachment } from '../../types/composerAttachment' import { MAX_ATTACHMENT_BYTES, useAttachment } from './useAttachment' vi.mock(import('@/platform/telemetry/reportError')) @@ -20,7 +21,10 @@ function chipRegistry() { const chips: ComposerAttachment[] = [] return { chips, - stage: (attachment: ComposerAttachment) => chips.push(attachment), + stage: (attachment: ComposerAttachment) => { + chips.push(attachment) + return true + }, update: (id: string, patch: Partial) => { const index = chips.findIndex((chip) => chip.id === id) if (index >= 0) chips[index] = { ...chips[index], ...patch } @@ -33,9 +37,175 @@ function chipRegistry() { } describe('useAttachment', () => { - it('adds previews for picked images but not picked videos', async () => { - // A video object URL in an renders as a broken thumbnail, so only - // images get a previewUrl. + it('deduplicates a file within a batch and across pending drops', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + const batch = addFiles([file, file, file]) + expect(onDuplicate).toHaveBeenCalledExactlyOnceWith(['cat.png', 'cat.png']) + const repeated = addFiles([ + new File(['image'], 'cat.png', { lastModified: 1 }) + ]) + expect(store.attachments).toHaveLength(1) + await expect(repeated).resolves.toBe(false) + await expect(batch).resolves.toBe(true) + expect(upload).toHaveBeenCalledOnce() + expect(onDuplicate).toHaveBeenLastCalledWith(['cat.png']) + expect(onDuplicate).toHaveBeenCalledTimes(2) + }) + + it('ignores a settled duplicate but allows a removed file to be added again', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + await addFiles([file]) + expect(onDuplicate).not.toHaveBeenCalled() + const original = store.attachments[0] + store.referenceAttachment(original.id) + store.referenceAttachment(original.id) + await expect(addFiles([file])).resolves.toBe(false) + expect(store.attachments).toEqual([original]) + expect(store.prompt.references).toHaveLength(2) + store.removeAttachment(original.id) + await expect(addFiles([file])).resolves.toBe(true) + expect(store.attachments).toHaveLength(1) + expect(store.attachments[0].id).not.toBe(original.id) + expect(upload).toHaveBeenCalledTimes(2) + expect(onDuplicate).toHaveBeenCalledExactlyOnceWith(['cat.png']) + }) + + it.for([ + { body: 'different size', lastModified: 1 }, + { body: 'image', lastModified: 2 } + ])( + 'keeps same-named files with distinct source metadata: %j', + async (source) => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const { addFiles } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + await addFiles([ + new File(['image'], 'cat.png', { lastModified: 1 }), + new File([source.body], 'cat.png', { + lastModified: source.lastModified + }) + ]) + expect(store.attachments.map(({ name }) => name)).toEqual([ + 'cat.png', + 'cat.png' + ]) + expect(upload).toHaveBeenCalledTimes(2) + } + ) + + it('allows a failed file upload to be retried', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + upload.mockRejectedValueOnce(new Error('Upload failed')) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + await expect(addFiles([file])).resolves.toBe(false) + await expect(addFiles([file])).resolves.toBe(true) + expect(store.attachments).toHaveLength(1) + expect(upload).toHaveBeenCalledTimes(2) + expect(onDuplicate).not.toHaveBeenCalled() + }) + + it('keeps a fresh re-addition after a cancelled upload settles late', async () => { + const store = useAgentComposerStore() + let resolveOldUpload: (result: { ref: string }) => void = () => {} + const oldUpload = new Promise<{ ref: string }>((resolve) => { + resolveOldUpload = resolve + }) + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + upload.mockReturnValueOnce(oldUpload) + const { addFiles, cancelUpload } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + const pending = addFiles([file]) + const oldId = store.attachments[0].id + cancelUpload(oldId) + await expect(addFiles([file])).resolves.toBe(true) + const replacement = store.attachments[0] + expect(replacement.id).not.toBe(oldId) + resolveOldUpload({ ref: 'obsolete.png' }) + await pending + expect(store.attachments).toEqual([replacement]) + expect(replacement.ref).toBe('cat.png') + }) + + it('keeps same-named deferred files from distinct source URIs', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const resolve = async () => new File(['image'], 'cat.png') + const { addDeferredFile } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + await Promise.all([ + addDeferredFile('cat.png', resolve, 'uri:/first'), + addDeferredFile('cat.png', resolve, 'uri:/second') + ]) + expect(store.attachments).toHaveLength(2) + expect(upload).toHaveBeenCalledTimes(2) + }) + + it('deduplicates a deferred source while loading and after upload', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const resolve = vi.fn(async () => new File(['image'], 'cat.png')) + const { addDeferredFile } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const pending = addDeferredFile('cat.png', resolve, 'uri:/source') + const repeated = addDeferredFile('cat.png', resolve, 'uri:/source') + expect(store.attachments).toHaveLength(1) + await expect(repeated).resolves.toBe('duplicate') + await expect(pending).resolves.toBe('uploaded') + await expect( + addDeferredFile('cat.png', resolve, 'uri:/source') + ).resolves.toBe('duplicate') + expect(resolve).toHaveBeenCalledOnce() + expect(upload).toHaveBeenCalledOnce() + }) + + it('keeps image preview URLs separate from picked video sources', async () => { const upload = vi.fn(async (file: File) => ({ ref: file.name })) const registry = chipRegistry() const { addFiles } = useAttachment({ upload, ...registry }) @@ -52,6 +222,45 @@ describe('useAttachment', () => { expect(upload).toHaveBeenCalledTimes(2) }) + it.for([ + { name: 'clip.mp4', type: 'video/mp4', mediaKind: 'video' }, + { name: 'recording', type: 'audio/mpeg', mediaKind: 'audio' } + ])( + 'previews picked $mediaKind while uploading and then uses its server file', + async ({ name, type, mediaKind }) => { + let resolveUpload: (result: { + ref: string + url: string + }) => void = () => {} + const registry = chipRegistry() + const { addFiles } = useAttachment({ + ...registry, + upload: () => + new Promise((resolve) => { + resolveUpload = resolve + }) + }) + const pending = addFiles([new File(['media'], name, { type })]) + expect(registry.chips[0]).toMatchObject({ + mediaKind, + mediaUrl: expect.stringMatching(/^blob:/), + uploading: true + }) + expect(registry.chips[0].previewUrl).toBeUndefined() + resolveUpload({ + ref: 'stored-media', + url: '/api/view?filename=stored-media&type=input' + }) + await pending + expect(registry.chips[0]).toMatchObject({ + mediaKind, + mediaUrl: '/api/view?filename=stored-media&type=input', + uploading: false + }) + expect(registry.chips[0].previewUrl).toBeUndefined() + } + ) + it('rejects files over 20MB before staging or uploading', async () => { const upload = vi.fn() const onError = vi.fn() diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts index 0d02428142c..c0212143a6e 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts @@ -2,7 +2,7 @@ import { assert, expect, it, onTestFinished, vi } from 'vitest' import { createAgentRestClient } from '../../services/agent/agentRestClient' import { useAttachment } from './useAttachment' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' it('keeps a 4 MiB upload alive past 60 seconds and aborts it at 94 seconds', async () => { vi.useFakeTimers() @@ -29,7 +29,10 @@ it('keeps a 4 MiB upload alive past 60 seconds and aborts it at 94 seconds', asy const result = await client.uploadImage(file, file.name, uploadSignal) return { ref: result.name ?? file.name } }, - stage: (chip) => chips.set(chip.id, chip), + stage: (chip) => { + chips.set(chip.id, chip) + return true + }, update: (id, patch) => { const chip = chips.get(id) assert.exists(chip) diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.ts index 19279da0a60..38d9d480895 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.ts @@ -1,12 +1,13 @@ import { i18n } from '@/i18n' import { reportError } from '@/platform/telemetry/reportError' -import { hasImageType } from '@/utils/eventUtils' -import { formatSize } from '@/utils/formatUtil' +import { hasAudioType, hasImageType, hasVideoType } from '@/utils/eventUtils' +import { formatSize, getMediaTypeFromFilename } from '@/utils/formatUtil' +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' import { AgentApiError, AgentResponseUnreadableError } from '../../services/agent/agentRestClient' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 const UPLOAD_HANDSHAKE_TIMEOUT_MS = 30 * 1000 @@ -29,13 +30,21 @@ interface UploadResult { url?: string } +type DeferredFileResult = + | 'uploaded' + | 'unsupported' + | 'cancelled' + | 'failed' + | 'duplicate' + export interface UseAttachmentOptions { upload: (file: File, signal: AbortSignal) => Promise uploadTimeoutMs?: number maxBytes?: (file: File) => number onError?: (message: string) => void onUploaded?: () => void - stage: (attachment: ComposerAttachment) => void + onDuplicate?: (names: string[]) => void + stage: (attachment: ComposerAttachment) => boolean update: (id: string, patch: Partial) => void remove: (id: string) => void } @@ -75,6 +84,36 @@ async function withDeadline( let stagedCount = 0 +function attachmentMediaKind(file: File): MediaKind { + if (hasImageType(file)) return 'image' + if (hasVideoType(file)) return 'video' + if (hasAudioType(file)) return 'audio' + return getMediaTypeFromFilename(file.name) +} + +function localPreview( + file: File, + kind: MediaKind +): Pick { + const playable = kind === 'video' || kind === 'audio' + const url = + kind === 'image' || playable ? URL.createObjectURL(file) : undefined + return { + previewUrl: kind === 'image' ? url : undefined, + mediaUrl: playable ? url : undefined + } +} + +function uploadedPreview( + kind: MediaKind, + url?: string +): Partial { + if (!url) return {} + if (kind === 'audio' || kind === 'video') return { mediaUrl: url } + if (kind === 'image') return { previewUrl: url } + return {} +} + export function useAttachment(options: UseAttachmentOptions) { const pending = new Set() const inFlight = new Map() @@ -82,10 +121,11 @@ export function useAttachment(options: UseAttachmentOptions) { const waiting: Array<() => void> = [] let activeUploads = 0 - function stage(name: string): string { + function stage(name: string, sourceKey?: string): string | undefined { const id = `upload-${++stagedCount}:${name}` + if (!options.stage({ id, name, ref: '', uploading: true, sourceKey })) + return undefined pending.add(id) - options.stage({ id, name, ref: '', uploading: true }) return id } @@ -161,9 +201,11 @@ export function useAttachment(options: UseAttachmentOptions) { else activeUploads += 1 try { if (cancelled.has(id)) return false + const mediaKind = attachmentMediaKind(file) options.update(id, { name: file.name, - previewUrl: hasImageType(file) ? URL.createObjectURL(file) : undefined + mediaKind, + ...localPreview(file, mediaKind) }) const controller = new AbortController() inFlight.set(id, controller) @@ -174,7 +216,7 @@ export function useAttachment(options: UseAttachmentOptions) { ) options.update(id, { ref: result.ref, - ...(result.url ? { previewUrl: result.url } : {}), + ...uploadedPreview(mediaKind, result.url), uploading: false }) return true @@ -201,11 +243,11 @@ export function useAttachment(options: UseAttachmentOptions) { for (const id of Array.from(pending)) cancelUpload(id) } - async function addDeferredFile( + async function uploadDeferredFile( + id: string, name: string, resolve: () => Promise - ): Promise<'uploaded' | 'unsupported' | 'cancelled' | 'failed'> { - const id = stage(name) + ): Promise { try { const file = await withDeadline(resolve(), DEFERRED_FETCH_TIMEOUT_MS) if (cancelled.has(id)) return 'cancelled' @@ -229,10 +271,30 @@ export function useAttachment(options: UseAttachmentOptions) { } } + async function addDeferredFile( + name: string, + resolve: () => Promise, + sourceKey?: string + ): Promise { + const id = stage(name, sourceKey) + if (!id) { + options.onDuplicate?.([name]) + return 'duplicate' + } + return uploadDeferredFile(id, name, resolve) + } + async function addFiles(files: Iterable): Promise { + const duplicates: string[] = [] const staged = [...files] .filter((file) => !isTooLarge(file)) - .map((file) => ({ file, id: stage(file.name) })) + .flatMap((file) => { + const sourceKey = `file:${JSON.stringify([file.name, file.size, file.lastModified, file.type])}` + const id = stage(file.name, sourceKey) + if (!id) duplicates.push(file.name) + return id ? [{ file, id }] : [] + }) + if (duplicates.length) options.onDuplicate?.(duplicates) let uploaded = 0 await Promise.all( staged.map(async ({ id, file }) => { diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts index 8972c35f5a4..b710fc9199b 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts @@ -4,7 +4,7 @@ import { useTelemetry } from '@/platform/telemetry' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import { useComposer } from './useComposer' vi.mock(import('@/platform/telemetry')) @@ -45,10 +45,8 @@ describe('useComposer', () => { composer.submit() - expect(onSend).toHaveBeenCalledWith('make a cat @[Image: cat.png]', [ - attachment - ]) - expect(composer.draft.value).toBe(' make a cat ') + expect(onSend).toHaveBeenCalledWith('make a cat', [attachment]) + expect(composer.draft.value).toBe(' make a cat ') expect(composer.attachments.value).toEqual([attachment]) }) @@ -105,7 +103,7 @@ describe('useComposer', () => { expect(composer.canSend.value).toBe(true) composer.submit() - expect(onSend).toHaveBeenCalledWith('@[Image: cat.png]', [ + expect(onSend).toHaveBeenCalledWith('', [ { id: 'a1', name: 'cat.png', ref: 'r' } ]) }) @@ -216,7 +214,7 @@ describe('useComposer', () => { ) }) - it('treats a reference-only draft as non-empty for prompt attribution', () => { + it('treats an attachment-only draft as non-empty for prompt attribution', () => { const { composer } = setup() composer.addAttachment({ id: 'a1', name: 'cat.png', ref: 'r' }) @@ -233,14 +231,14 @@ describe('useComposer', () => { first.addAttachment({ id: 'a1', name: 'cat.png', ref: 'r' }) const { composer: second, onSend } = setup() - expect(second.draft.value).toBe('still here ') + expect(second.draft.value).toBe('still here') expect(second.attachments.value.map((a) => a.id)).toEqual(['a1']) second.submit() - expect(onSend).toHaveBeenCalledWith('still here@[Image: cat.png]', [ + expect(onSend).toHaveBeenCalledWith('still here', [ { id: 'a1', name: 'cat.png', ref: 'r' } ]) - expect(first.draft.value).toBe('still here ') + expect(first.draft.value).toBe('still here') expect(first.attachments.value.map((attachment) => attachment.id)).toEqual([ 'a1' ]) diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.ts b/src/workbench/extensions/agent/composables/agent/useComposer.ts index 2c453a566cf..407c582fd84 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.ts @@ -1,3 +1,4 @@ +import type { ComposerAttachment } from '../../types/composerAttachment' import { storeToRefs } from 'pinia' import { computed, getCurrentScope, onScopeDispose } from 'vue' import { v4 as uuidv4 } from 'uuid' @@ -8,14 +9,6 @@ import { composerPromptForSend } from '../../utils/composerPrompt' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' -export interface ComposerAttachment { - id: string - name: string - ref: string - previewUrl?: string - uploading?: boolean -} - export interface UseComposerOptions { onSend: (text: string, attachments: ComposerAttachment[]) => void isRunning: () => boolean @@ -30,7 +23,9 @@ export function useComposer(options: UseComposerOptions) { const canSend = computed( () => - (draft.value.trim().length > 0 || prompt.value.references.length > 0) && + (draft.value.trim().length > 0 || + prompt.value.references.length > 0 || + attachments.value.length > 0) && !attachments.value.some((item) => item.uploading) ) @@ -58,7 +53,9 @@ export function useComposer(options: UseComposerOptions) { starterPrompt?: AgentStarterPromptAttribution ): void { const draftWasEmpty = - !draft.value.trim() && prompt.value.references.length === 0 + !draft.value.trim() && + prompt.value.references.length === 0 && + attachments.value.length === 0 store.setText(draft.value ? `${draft.value} ${text}` : text) if (!starterPrompt) { store.markSuggestedPrompt() @@ -92,6 +89,7 @@ export function useComposer(options: UseComposerOptions) { setText: store.setText, replacePrompt: store.replacePrompt, addAttachment: store.addAttachment, + referenceAttachment: store.referenceAttachment, updateAttachment: store.updateAttachment, removeAttachment: store.removeAttachment } diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts index ca497689daf..813c88c6f9d 100644 --- a/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts @@ -5,7 +5,41 @@ import { createMockLoadedWorkflow } from '@/utils/__tests__/litegraphTestUtils' import { useAgentComposerStore } from './agentComposerStore' describe('composer reference ownership', () => { - it('updates detached uploads without attaching them until an explicit Undo', () => { + it('releases invalidated snapshot previews while retaining the current tray and ignoring repeated invalidation and stale settlement', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + const retained = { + id: 'retained', + name: 'retained.png', + ref: 'retained.png', + previewUrl: 'blob:retained' + } + store.addAttachment({ + id: 'old', + name: 'old.png', + ref: 'old.png', + previewUrl: 'blob:old' + }) + store.addAttachment(retained) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + store.restorePrompt({ text: 'New draft', references: [] }, [retained]) + store.invalidateSubmission() + store.invalidateSubmission() + store.settleSubmission(id, false) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:old') + expect(store.attachments).toEqual([retained]) + expect(store.draft).toBe('New draft') + expect(store.takeFailedSubmission()).toBeUndefined() + expect(store.referenceAttachment('old')).toBe(false) + expect(store.referenceAttachment('retained')).toBe(true) + }) + + it('keeps every asset occurrence and updates all of them after upload', () => { const store = useAgentComposerStore() store.addAttachment({ id: 'image', @@ -13,10 +47,43 @@ describe('composer reference ownership', () => { ref: '', uploading: true }) + store.referenceAttachment('image') + store.referenceAttachment('image') + const snapshot = store.prompt + expect(snapshot.references).toHaveLength(2) + + store.updateAttachment('image', { ref: 'uploaded.png', uploading: false }) + store.applyEditorPrompt(snapshot) + expect(store.prompt.references).toEqual( + snapshot.references.map((reference) => ({ + ...reference, + attachment: { + id: 'image', + name: 'source.png', + ref: 'uploaded.png', + uploading: false + } + })) + ) + expect(store.attachments).toHaveLength(1) + }) + + it('updates a tray upload without reinserting its deleted inline mention', () => { + const store = useAgentComposerStore() + store.addAttachment({ + id: 'image', + name: 'source.png', + ref: '', + uploading: true + }) + store.referenceAttachment('image') const snapshot = store.prompt store.removeReference('asset:image') store.updateAttachment('image', { ref: 'uploaded.png', uploading: false }) - expect(store.attachments).toEqual([]) + expect(store.prompt.references).toEqual([]) + expect(store.attachments).toEqual([ + { id: 'image', name: 'source.png', ref: 'uploaded.png', uploading: false } + ]) store.applyEditorPrompt(snapshot) expect(store.attachments).toEqual([ { id: 'image', name: 'source.png', ref: 'uploaded.png', uploading: false } @@ -34,6 +101,7 @@ describe('composer reference ownership', () => { uploading: true, previewUrl: 'blob:source' }) + store.referenceAttachment('image') const snapshot = store.prompt store.removeAttachment('image') store.updateAttachment('image', { ref: 'too-late.png', uploading: false }) @@ -42,7 +110,7 @@ describe('composer reference ownership', () => { expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:source') }) - it('retains previews for Undo, then releases only unused previews on unmount', () => { + it('retains included previews after inline deletion and unmount', () => { const store = useAgentComposerStore() const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) store.addAttachment({ @@ -57,11 +125,83 @@ describe('composer reference ownership', () => { ref: 'second.png', previewUrl: 'blob:second' }) + store.referenceAttachment('first') store.removeReference('asset:first') expect(revoke).not.toHaveBeenCalled() store.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + expect(store.attachments.map(({ id }) => id)).toEqual(['first', 'second']) + store.removeAttachment('first') expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:first') - expect(store.attachments.map(({ id }) => id)).toEqual(['second']) + }) + + it('releases excluded previews when replacing the tray and rejects late updates', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'old', + name: 'old.png', + ref: 'old.png', + previewUrl: 'blob:old' + }) + store.referenceAttachment('old') + const oldPrompt = store.prompt + store.replaceDraft({ + text: 'New draft', + workflowReferences: [], + attachments: [ + { id: 'new', name: 'new.png', ref: 'new.png', previewUrl: 'blob:new' } + ] + }) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:old') + store.updateAttachment('old', { previewUrl: 'blob:late' }) + expect(revoke).toHaveBeenLastCalledWith('blob:late') + store.applyEditorPrompt({ ...oldPrompt, text: 'Keep typing' }) + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(store.attachments.map(({ id }) => id)).toEqual(['new']) + expect(revoke).not.toHaveBeenCalledWith('blob:new') + }) + + it('retains included media until replacement/removal and rejects late object URLs', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: '', + mediaUrl: 'blob:video' + }) + store.referenceAttachment('video') + store.removeReference('asset:video') + store.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + store.updateAttachment('video', { mediaUrl: '/clip.mp4', ref: 'clip.mp4' }) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:video') + store.updateAttachment('video', { mediaUrl: 'blob:replacement' }) + store.removeAttachment('video') + store.updateAttachment('video', { mediaUrl: 'blob:late' }) + expect(revoke.mock.calls).toEqual([ + ['blob:video'], + ['blob:replacement'], + ['blob:late'] + ]) + expect(store.attachments).toEqual([]) + }) + + it('releases a shared blob only after both its image and media uses end', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: '', + previewUrl: 'blob:shared', + mediaUrl: 'blob:shared' + }) + store.updateAttachment('video', { previewUrl: undefined }) + expect(revoke).not.toHaveBeenCalled() + store.removeAttachment('video') + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:shared') }) it('rejects node history from a different target even when node IDs collide', () => { diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts new file mode 100644 index 00000000000..d2dc8c126bc --- /dev/null +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts @@ -0,0 +1,315 @@ +import { assert, beforeEach, describe, expect, it, vi } from 'vitest' + +import { createMockLoadedWorkflow } from '@/utils/__tests__/litegraphTestUtils' + +import type { ComposerAttachment } from '../../types/composerAttachment' +import * as videoThumbnail from '../../utils/videoThumbnail' +import { useAgentComposerStore } from './agentComposerStore' + +const video: ComposerAttachment = { + id: 'clip', + name: 'My video', + ref: '/clip.mp4', + mediaKind: 'video', + mediaUrl: '/clip.mp4' +} + +type ComposerStore = ReturnType + +function pendingThumbnail() { + let resolve!: (value: string | undefined) => void + const promise = new Promise((complete) => { + resolve = complete + }) + const result = { promise, resolve } + vi.mocked(videoThumbnail.createVideoThumbnail).mockReturnValueOnce( + result.promise + ) + return result +} + +beforeEach(() => { + vi.spyOn(videoThumbnail, 'createVideoThumbnail').mockResolvedValue(undefined) + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) +}) + +describe('composer shared video thumbnail lifetime', () => { + it('cancels an invalidated send capture and rejects its late poster while a new draft remains usable', async () => { + const obsolete = pendingThumbnail() + const current = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:old-video' }) + const obsoleteSignal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[0][1] + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + store.addAttachment({ + ...video, + id: 'new-video', + mediaUrl: 'blob:new-video' + }) + const currentSignal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[1][1] + store.setText('New chat draft') + store.invalidateSubmission() + expect(obsoleteSignal.aborted).toBe(true) + expect(currentSignal.aborted).toBe(false) + expect(URL.revokeObjectURL).toHaveBeenCalledExactlyOnceWith( + 'blob:old-video' + ) + obsolete.resolve('blob:late-poster') + current.resolve('blob:current-poster') + await Promise.all([obsolete.promise, current.promise]) + store.settleSubmission(id, false) + expect(store.takeFailedSubmission()).toBeUndefined() + expect(store.draft).toBe('New chat draft') + expect(store.attachments).toMatchObject([ + { id: 'new-video', previewUrl: 'blob:current-poster' } + ]) + expect(store.referenceAttachment(video.id)).toBe(false) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:late-poster') + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith('blob:new-video') + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith('blob:current-poster') + }) + + it('releases the current poster completed during an invalidated send exactly once', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:video' }) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + result.resolve('blob:completed-poster') + await result.promise + store.invalidateSubmission() + store.invalidateSubmission() + store.settleSubmission(id, true) + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:video') + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:completed-poster') + expect(store.submission).toBeNull() + expect(store.attachments).toEqual([]) + }) + + it('keeps a snapshot asset and decoder retained by the current draft through invalidation and a repeated send', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:retained-video' }) + store.referenceAttachment(video.id) + const snapshot = { + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + } + const first = store.startSubmission(snapshot) + store.restorePrompt(snapshot.prompt, snapshot.attachments) + store.invalidateSubmission() + expect( + vi.mocked(videoThumbnail.createVideoThumbnail).mock.calls[0][1].aborted + ).toBe(false) + result.resolve('blob:retained-poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:retained-poster') + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + const second = store.startSubmission({ + ...snapshot, + attachments: store.attachments + }) + store.settleSubmission(first, false) + expect(store.submission?.id).toBe(second) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + store.settleSubmission(second, false) + const failed = store.takeFailedSubmission() + assert.exists(failed) + store.restorePrompt(failed.prompt, failed.attachments) + expect(store.attachments[0].previewUrl).toBe('blob:retained-poster') + expect(store.prompt.references).toHaveLength(1) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + }) + + it('generates once for the included asset and shares the result with every occurrence', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + store.referenceAttachment(video.id) + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { name: 'Renamed' }) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + result.resolve('blob:poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(store.prompt.references.map((reference) => reference.kind)).toEqual([ + 'asset', + 'asset' + ]) + expect( + store.prompt.references + .filter((reference) => reference.kind === 'asset') + .map((reference) => reference.attachment.previewUrl) + ).toEqual(['blob:poster', 'blob:poster']) + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { + mediaUrl: '/uploaded.mp4', + uploading: false + }) + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + store.removeReference(`asset:${video.id}`) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + store.removeAttachment(video.id) + expect(URL.revokeObjectURL).toHaveBeenCalledExactlyOnceWith('blob:poster') + }) + + it.for([ + { + event: 'upload metadata', + act: (store: ComposerStore) => { + store.addAttachment({ + id: video.id, + name: 'clip.mp4', + ref: '', + uploading: true + }) + expect(videoThumbnail.createVideoThumbnail).not.toHaveBeenCalled() + store.updateAttachment(video.id, { + mediaKind: 'video', + mediaUrl: 'blob:clip' + }) + } + }, + { + event: 'a restored draft', + act: (store: ComposerStore) => + store.replaceDraft({ + text: '', + workflowReferences: [], + attachments: [video] + }) + } + ])( + 'starts shared capture when the video source arrives through $event', + async ({ act }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + act(store) + result.resolve('blob:poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + } + ) + + it('uses a trusted poster immediately without decoding video', () => { + const store = useAgentComposerStore() + store.replaceDraft({ + text: '', + workflowReferences: [], + attachments: [{ ...video, previewUrl: '/server-poster.png' }] + }) + store.referenceAttachment(video.id) + expect(store.attachments[0].previewUrl).toBe('/server-poster.png') + expect(videoThumbnail.createVideoThumbnail).not.toHaveBeenCalled() + }) + + it.for([ + { + event: 'removing the asset', + act: (store: ComposerStore) => store.removeAttachment(video.id) + }, + { + event: 'replacing the video source', + act: (store: ComposerStore) => + store.updateAttachment(video.id, { mediaUrl: '/new.mp4' }) + }, + { + event: 'receiving a trusted poster', + act: (store: ComposerStore) => + store.updateAttachment(video.id, { previewUrl: '/server.png' }) + }, + { + event: 'disposing the composer', + act: (store: ComposerStore) => store.$dispose() + } + ])( + 'discards and releases an obsolete capture after $event', + async ({ act }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + const signal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[0][1] + result.resolve('blob:stale') + act(store) + expect(signal.aborted).toBe(true) + await result.promise + expect( + store.attachments.every((asset) => asset.previewUrl !== 'blob:stale') + ).toBe(true) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:stale') + } + ) + + it('does not repeatedly retry failed decoding when referencing or updating the same asset', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + result.resolve(undefined) + await result.promise + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { name: 'Renamed' }) + expect(store.attachments[0].previewUrl).toBeUndefined() + expect(store.prompt.references).toHaveLength(1) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + }) + + it.for([ + { + outcome: 'reclaims the poster after a successful send', + sent: true, + assertOutcome: (store: ComposerStore) => { + expect(store.attachments).toEqual([]) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:poster') + } + }, + { + outcome: 'retains the poster when recovering a failed send', + sent: false, + assertOutcome: (store: ComposerStore) => { + const failed = store.takeFailedSubmission() + assert.exists(failed) + store.restorePrompt(failed.prompt, failed.attachments) + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(store.prompt.references).toHaveLength(1) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + } + } + ])( + '$outcome when capture completes during submission', + async ({ sent, assertOutcome }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + store.referenceAttachment(video.id) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + result.resolve('blob:poster') + await result.promise + store.settleSubmission(id, sent) + assertOutcome(store) + } + ) +}) diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts index fca76c577f8..a14d31139b3 100644 --- a/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts @@ -1,10 +1,12 @@ import { defineStore } from 'pinia' -import { computed, ref, shallowRef } from 'vue' +import { computed, onScopeDispose, ref, shallowReactive, shallowRef } from 'vue' + +import { getMediaTypeFromFilename } from '@/utils/formatUtil' import type { AgentInputMethod } from '@/platform/telemetry/types' import type { ComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' -import type { ComposerAttachment } from '../../composables/agent/useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import type { SelectedNode } from '../../composables/agent/useCanvasSelection' import { selectedNodeKey } from '../../composables/agent/useCanvasSelection' import type { @@ -18,6 +20,7 @@ import type { WorkflowReference } from '../../types/workflowReference' import { insertComposerReference } from '../../utils/composerPrompt' +import { createVideoThumbnail } from '../../utils/videoThumbnail' import type { AgentStarterPromptSource } from '../../utils/starterPrompts' interface ComposerDraft extends PromptSnapshot { @@ -48,10 +51,13 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const draftState = shallowRef({ text: '', references: [] }) const prompt = computed(() => draftState.value) const draft = computed(() => prompt.value.text) + const attachmentIds = ref([]) + const assetsById = shallowReactive(new Map()) const attachments = computed(() => - prompt.value.references.flatMap((item) => - item.kind === 'asset' ? [item.attachment] : [] - ) + attachmentIds.value.flatMap((id) => { + const attachment = assetsById.get(id) + return attachment ? [attachment] : [] + }) ) const workflowReferences = computed(() => prompt.value.references.flatMap((item) => { @@ -78,9 +84,49 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { textOffset: 0, referenceIndex: 0 }) - // Detached assets stay available to the editor's Undo history until it unmounts. - const undoAssets = new Map() const retiredAssets = new Set() + const thumbnailJobs = new Map< + string, + { source: string; controller: AbortController } + >() + onScopeDispose(() => { + for (const job of thumbnailJobs.values()) job.controller.abort() + thumbnailJobs.clear() + for (const attachment of assetsById.values()) revokePreview(attachment) + }) + + function cancelThumbnail(id: string): void { + thumbnailJobs.get(id)?.controller.abort() + thumbnailJobs.delete(id) + } + + function ensureThumbnail(attachment: ComposerAttachment): void { + const { id, mediaUrl, previewUrl } = attachment + const kind = + attachment.mediaKind ?? getMediaTypeFromFilename(attachment.name) + if (kind !== 'video' || !mediaUrl || previewUrl) { + cancelThumbnail(id) + return + } + if (thumbnailJobs.get(id)?.source === mediaUrl) return + cancelThumbnail(id) + const job = { source: mediaUrl, controller: new AbortController() } + thumbnailJobs.set(id, job) + void createVideoThumbnail(mediaUrl, job.controller.signal).then((url) => { + if (!url) return + const current = assetsById.get(id) + if ( + thumbnailJobs.get(id) !== job || + !current || + current.previewUrl || + current.mediaUrl !== mediaUrl + ) { + URL.revokeObjectURL(url) + return + } + updateAttachment(id, { previewUrl: url }) + }) + } const submission = shallowRef<{ id: number phase: 'pending' | 'failed' @@ -124,13 +170,13 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const seen = new Set() const references = next.references.flatMap((item): ComposerReference[] => { const key = composerReferenceKey(item) - if (seen.has(key)) return [] + if (item.kind !== 'asset' && seen.has(key)) return [] seen.add(key) if (item.kind === 'node' && item.scope !== nodeScope.value) return [] if (item.kind !== 'asset') return [item] - if (retiredAssets.has(item.attachment.id)) return [] - const attachment = undoAssets.get(item.attachment.id) ?? item.attachment - undoAssets.set(attachment.id, attachment) + if (!attachmentIds.value.includes(item.attachment.id)) return [] + const attachment = assetsById.get(item.attachment.id) + if (!attachment) return [] return [{ ...item, attachment }] }) if ( @@ -171,31 +217,41 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { }) } - function restorePrompt(next: ComposerPrompt): void { + function restorePrompt( + next: ComposerPrompt, + included: ComposerAttachment[] = attachments.value + ): void { + for (const attachment of included) { + const current = assetsById.get(attachment.id) + const restored = { + ...attachment, + ...(current?.mediaUrl === attachment.mediaUrl && + current?.previewUrl && + !attachment.previewUrl + ? { previewUrl: current.previewUrl } + : {}) + } + if (current) revokePreview(current, restored) + assetsById.set(attachment.id, restored) + retiredAssets.delete(attachment.id) + ensureThumbnail(restored) + } + attachmentIds.value = included.map(({ id }) => id) resetPromptHistory() applyEditorPrompt(next) + releaseUnusedAssets() } function replaceDraft(next: ComposerDraft): void { - for (const attachment of next.attachments) { - undoAssets.set(attachment.id, { ...attachment }) - retiredAssets.delete(attachment.id) - } - restorePrompt({ - text: next.text, - references: [ - ...next.workflowReferences.map( - (item): ComposerReference => ({ ...item, kind: 'workflow' }) - ), - ...next.attachments.map( - (attachment): ComposerReference => ({ - kind: 'asset', - attachment: { ...attachment }, - textOffset: next.text.length - }) - ) - ].sort((a, b) => a.textOffset - b.textOffset) - }) + restorePrompt( + { + text: next.text, + references: next.workflowReferences + .map((item): ComposerReference => ({ ...item, kind: 'workflow' })) + .sort((a, b) => a.textOffset - b.textOffset) + }, + next.attachments + ) } function setWorkflowReferences(references: WorkflowReference[]): void { @@ -294,9 +350,23 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { } function addAttachment(attachment: ComposerAttachment): boolean { - if (undoAssets.has(attachment.id) || retiredAssets.has(attachment.id)) + if (assetsById.has(attachment.id) || retiredAssets.has(attachment.id)) + return false + if ( + attachment.sourceKey && + attachments.value.some((item) => item.sourceKey === attachment.sourceKey) + ) return false - undoAssets.set(attachment.id, { ...attachment }) + assetsById.set(attachment.id, { ...attachment }) + attachmentIds.value = [...attachmentIds.value, attachment.id] + ++revision + ensureThumbnail(attachment) + return true + } + + function referenceAttachment(id: string): boolean { + const attachment = assetsById.get(id) + if (!attachment || !attachmentIds.value.includes(id)) return false const inserted = insertComposerReference( prompt.value, { kind: 'asset', attachment: { ...attachment }, textOffset: 0 }, @@ -307,44 +377,55 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { return true } - function revokePreview(attachment: ComposerAttachment): void { - if (attachment.previewUrl?.startsWith('blob:')) - URL.revokeObjectURL(attachment.previewUrl) + function revokePreview( + attachment: Pick, + retained?: ComposerAttachment + ): void { + const retainedUrls = new Set([retained?.previewUrl, retained?.mediaUrl]) + for (const url of new Set([attachment.previewUrl, attachment.mediaUrl])) + if (url?.startsWith('blob:') && !retainedUrls.has(url)) + URL.revokeObjectURL(url) } function updateAttachment( id: string, patch: Partial ): void { - const previous = undoAssets.get(id) + const previous = assetsById.get(id) if (!previous) { - if (patch.previewUrl?.startsWith('blob:')) - URL.revokeObjectURL(patch.previewUrl) + revokePreview(patch) return } - if ( - patch.previewUrl !== undefined && - patch.previewUrl !== previous.previewUrl - ) - revokePreview(previous) const attachment = { ...previous, ...patch, id } - undoAssets.set(id, attachment) - if (!attachments.value.some((item) => item.id === id)) return - updateDraft({ - text: draft.value, - references: prompt.value.references.map((item) => - item.kind === 'asset' && item.attachment.id === id - ? { ...item, attachment } - : item + if (attachment.mediaUrl !== previous.mediaUrl) cancelThumbnail(id) + revokePreview(previous, attachment) + assetsById.set(id, attachment) + ensureThumbnail(attachment) + if ( + prompt.value.references.some( + (item) => item.kind === 'asset' && item.attachment.id === id ) - }) + ) + updateDraft({ + text: draft.value, + references: prompt.value.references.map((item) => + item.kind === 'asset' && item.attachment.id === id + ? { ...item, attachment } + : item + ) + }) } function removeAttachment(id: string): void { - const attachment = undoAssets.get(id) + cancelThumbnail(id) + const attachment = assetsById.get(id) if (attachment) revokePreview(attachment) - undoAssets.delete(id) + assetsById.delete(id) retiredAssets.add(id) + if (attachmentIds.value.includes(id)) { + attachmentIds.value = attachmentIds.value.filter((item) => item !== id) + ++revision + } removeReference(`asset:${id}`) } @@ -352,10 +433,11 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const retained = new Set(attachments.value.map(({ id }) => id)) for (const attachment of submission.value?.snapshot.attachments ?? []) retained.add(attachment.id) - for (const [id, attachment] of undoAssets) + for (const [id, attachment] of assetsById) if (!retained.has(id)) { + cancelThumbnail(id) revokePreview(attachment) - undoAssets.delete(id) + assetsById.delete(id) retiredAssets.add(id) } } @@ -369,6 +451,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const chip = starterPrompt.value promptOrigin.value = 'typed' starterPrompt.value = null + attachmentIds.value = [] updateDraft({ text: '', references: [] }) insertionPoint.value = { textOffset: 0, referenceIndex: 0 } const id = ++nextSubmissionId @@ -405,6 +488,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { function invalidateSubmission(): void { submission.value = null + releaseUnusedAssets() } return { @@ -434,6 +518,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { setNodeScope, setNodes, addAttachment, + referenceAttachment, updateAttachment, removeAttachment, releaseUnusedAssets, diff --git a/src/workbench/extensions/agent/types/assetPreview.ts b/src/workbench/extensions/agent/types/assetPreview.ts new file mode 100644 index 00000000000..d67a6600c66 --- /dev/null +++ b/src/workbench/extensions/agent/types/assetPreview.ts @@ -0,0 +1 @@ +export type AssetPreviewMode = 'closed' | 'hover' | 'interactive' | 'dismissed' diff --git a/src/workbench/extensions/agent/types/composerAttachment.ts b/src/workbench/extensions/agent/types/composerAttachment.ts new file mode 100644 index 00000000000..be7cf14b1ee --- /dev/null +++ b/src/workbench/extensions/agent/types/composerAttachment.ts @@ -0,0 +1,12 @@ +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' + +export interface ComposerAttachment { + id: string + name: string + ref: string + sourceKey?: string + previewUrl?: string + mediaUrl?: string + mediaKind?: MediaKind + uploading?: boolean +} diff --git a/src/workbench/extensions/agent/types/composerPrompt.ts b/src/workbench/extensions/agent/types/composerPrompt.ts index 39355371cc2..d3b8280c0d0 100644 --- a/src/workbench/extensions/agent/types/composerPrompt.ts +++ b/src/workbench/extensions/agent/types/composerPrompt.ts @@ -1,6 +1,6 @@ import type { SelectedNode } from '../composables/agent/useCanvasSelection' import { selectedNodeKey } from '../composables/agent/useCanvasSelection' -import type { ComposerAttachment } from '../composables/agent/useComposer' +import type { ComposerAttachment } from './composerAttachment' import type { WorkflowReference } from './workflowReference' export type ComposerReference = diff --git a/src/workbench/extensions/agent/utils/videoThumbnail.test.ts b/src/workbench/extensions/agent/utils/videoThumbnail.test.ts new file mode 100644 index 00000000000..16f0a172b33 --- /dev/null +++ b/src/workbench/extensions/agent/utils/videoThumbnail.test.ts @@ -0,0 +1,107 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + createMockCanvasRenderingContext2D, + createTestCanvasElement +} from '@/utils/__tests__/canvasTestUtils' + +import { createVideoThumbnail } from './videoThumbnail' + +function mediaPlatform(canvas = document.createElement('canvas')) { + const video = document.createElement('video') + const createElement = vi.spyOn(document, 'createElement') + vi.when(createElement).calledWith('video').thenReturn(video) + vi.when(createElement).calledWith('canvas').thenReturn(canvas) + Object.defineProperties(video, { + videoWidth: { value: 1920 }, + videoHeight: { value: 1080 } + }) + vi.spyOn(video, 'pause').mockImplementation(() => {}) + vi.spyOn(video, 'load').mockImplementation(() => {}) + return { video, canvas } +} + +beforeEach(() => { + vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:poster') +}) + +describe('video thumbnail capture platform boundary', () => { + it('captures a bounded first frame without playback, then releases the decoder', async () => { + const context = createMockCanvasRenderingContext2D({ drawImage: vi.fn() }) + const { video, canvas } = mediaPlatform( + createTestCanvasElement({ ctx: context }) + ) + vi.spyOn(canvas, 'toBlob').mockImplementation((callback) => + callback(new Blob(['frame'], { type: 'image/jpeg' })) + ) + const result = createVideoThumbnail( + '/clip.mp4', + new AbortController().signal + ) + expect(video.src).toContain('/clip.mp4') + expect(video.crossOrigin).toBe('anonymous') + expect(video.muted).toBe(true) + expect(video.autoplay).toBe(false) + video.dispatchEvent(new Event('loadeddata')) + expect(await result).toBe('blob:poster') + expect(canvas.width).toBe(320) + expect(canvas.height).toBe(180) + expect(context.drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180) + expect(video).not.toHaveAttribute('src') + expect(video.pause).toHaveBeenCalled() + }) + + it('does not allocate a poster after cancellation during image encoding', async () => { + const { video, canvas } = mediaPlatform( + createTestCanvasElement({ + ctx: createMockCanvasRenderingContext2D({ drawImage: vi.fn() }) + }) + ) + let complete!: BlobCallback + vi.spyOn(canvas, 'toBlob').mockImplementation((callback) => { + complete = callback + }) + const controller = new AbortController() + const result = createVideoThumbnail('/clip.mp4', controller.signal) + video.dispatchEvent(new Event('loadeddata')) + controller.abort() + expect(await result).toBeUndefined() + complete(new Blob(['frame'], { type: 'image/jpeg' })) + expect(URL.createObjectURL).not.toHaveBeenCalled() + expect(video).not.toHaveAttribute('src') + }) + + it.for([ + { + event: 'a decoding error', + act: (video: HTMLVideoElement) => video.dispatchEvent(new Event('error')) + }, + { + event: 'a timeout', + act: () => vi.advanceTimersByTime(15_000) + }, + { + event: 'cancellation', + act: (_video: HTMLVideoElement, controller: AbortController) => + controller.abort() + }, + { + event: 'an unavailable canvas context', + act: (video: HTMLVideoElement) => + video.dispatchEvent(new Event('loadeddata')) + } + ])( + 'resolves to a fallback and releases the decoder after $event', + async ({ act }) => { + const { video } = mediaPlatform() + const controller = new AbortController() + const result = createVideoThumbnail('/clip.mp4', controller.signal) + act(video, controller) + expect(await result).toBeUndefined() + expect(video).not.toHaveAttribute('src') + expect(video.pause).toHaveBeenCalled() + video.dispatchEvent(new Event('loadeddata')) + expect(URL.createObjectURL).not.toHaveBeenCalled() + } + ) +}) diff --git a/src/workbench/extensions/agent/utils/videoThumbnail.ts b/src/workbench/extensions/agent/utils/videoThumbnail.ts new file mode 100644 index 00000000000..6b3c0a53a4f --- /dev/null +++ b/src/workbench/extensions/agent/utils/videoThumbnail.ts @@ -0,0 +1,59 @@ +export function createVideoThumbnail( + source: string, + signal: AbortSignal +): Promise { + if (signal.aborted) return Promise.resolve(undefined) + const video = document.createElement('video') + video.crossOrigin = 'anonymous' + video.preload = 'auto' + video.muted = true + video.playsInline = true + + return new Promise((resolve) => { + let settled = false + const finish = (url?: string) => { + if (settled) return + settled = true + clearTimeout(timer) + signal.removeEventListener('abort', cancel) + video.removeEventListener('error', cancel) + video.removeEventListener('loadeddata', capture) + video.pause() + video.removeAttribute('src') + video.load() + resolve(url) + } + const cancel = () => finish() + const capture = () => { + if (!video.videoWidth || !video.videoHeight) return finish() + try { + const canvas = document.createElement('canvas') + const scale = Math.min( + 320 / video.videoWidth, + 320 / video.videoHeight, + 1 + ) + canvas.width = Math.max(1, Math.round(video.videoWidth * scale)) + canvas.height = Math.max(1, Math.round(video.videoHeight * scale)) + const context = canvas.getContext('2d') + if (!context) return finish() + context.drawImage(video, 0, 0, canvas.width, canvas.height) + canvas.toBlob( + (blob) => { + if (!settled) finish(blob ? URL.createObjectURL(blob) : undefined) + }, + 'image/jpeg', + 0.8 + ) + } catch { + finish() + } + } + const timer = setTimeout(cancel, 15_000) + signal.addEventListener('abort', cancel, { once: true }) + video.addEventListener('error', cancel, { once: true }) + video.addEventListener('loadeddata', capture, { once: true }) + video.src = source + video.load() + }) +}