diff --git a/browser_tests/assets/silence.wav b/browser_tests/assets/silence.wav new file mode 100644 index 00000000000..555f153a976 Binary files /dev/null and b/browser_tests/assets/silence.wav differ diff --git a/browser_tests/fixtures/VueNodeHelpers.ts b/browser_tests/fixtures/VueNodeHelpers.ts index 47b109a43cb..3ea2a1aa678 100644 --- a/browser_tests/fixtures/VueNodeHelpers.ts +++ b/browser_tests/fixtures/VueNodeHelpers.ts @@ -180,6 +180,15 @@ export class VueNodeHelpers { await this.deleteSelected() } + /** Rename a node through the canvas title editor. */ + async renameNode(nodeId: string, newTitle: string): Promise { + const title = this.getNodeLocator(nodeId).getByTestId('node-title') + await title.dblclick({ delay: 5 }) + const input = title.getByTestId('node-title-input') + await input.fill(newTitle) + await input.press('Enter') + } + /** * Delete selected Vue nodes using Backspace key */ diff --git a/browser_tests/fixtures/agentPanelFixture.ts b/browser_tests/fixtures/agentPanelFixture.ts index 96eacbd447d..63975d51e6f 100644 --- a/browser_tests/fixtures/agentPanelFixture.ts +++ b/browser_tests/fixtures/agentPanelFixture.ts @@ -10,6 +10,7 @@ import type { import type { RemoteConfig } from '@/platform/remoteConfig/types' import type { UserDataFullInfo } from '@/platform/remote/comfyui/types' +import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema' import { AGENT_CONSENT_SETTING_ID } from '@/platform/settings/constants/agent' import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' import type { AgentTurnAccepted } from '@/workbench/extensions/agent/schemas/agentApiSchema' @@ -18,6 +19,8 @@ import { cloudAppFixture, waitForCloudApp } from '@e2e/fixtures/cloudAppFixture' import { mockBilling } from '@e2e/fixtures/utils/cloudBillingMocks' import { bootCloud, mockCloudBoot } from '@e2e/fixtures/utils/cloudBootMocks' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { nextFrame } from '@e2e/fixtures/utils/timing' +import type { WorkspaceStore } from '@e2e/types/globals' const APP_URL = process.env.PLAYWRIGHT_TEST_URL || 'http://localhost:8188' @@ -198,3 +201,22 @@ export async function bootAgentApp( await page.goto(APP_URL) await waitForCloudApp(page) } + +/** Replace the boot tab's graph while retaining its active workflow identity. */ +export async function loadIntoBootWorkflow( + page: Page, + json: ComfyWorkflowJSON +): Promise { + await page.waitForFunction(() => { + const workspace = window.app?.extensionManager as WorkspaceStore | undefined + return workspace?.workflow.activeWorkflow != null + }) + await page.evaluate(async (json) => { + const activeWorkflow = (window.app!.extensionManager as WorkspaceStore) + .workflow.activeWorkflow! + await window.app!.loadGraphData(json, true, true, activeWorkflow) + if (json.nodes.length > 0) + await window.app!.extensionManager.command.execute('Comfy.Canvas.FitView') + }, json) + await nextFrame(page) +} diff --git a/browser_tests/fixtures/components/AgentPanel.ts b/browser_tests/fixtures/components/AgentPanel.ts index c5ba5634633..d5cf3dc4d21 100644 --- a/browser_tests/fixtures/components/AgentPanel.ts +++ b/browser_tests/fixtures/components/AgentPanel.ts @@ -18,6 +18,7 @@ export class AgentPanel { public readonly workflowPicker: Locator public readonly fileInput: Locator public readonly composerAssetSection: Locator + public readonly scrollAssetsRight: Locator public readonly attachmentChips: Locator public readonly composer: Locator public readonly composerPromptArea: Locator @@ -55,6 +56,9 @@ export class AgentPanel { }) this.fileInput = this.root.getByTestId('agent-file-input') this.composerAssetSection = this.root.getByTestId('composer-asset-section') + this.scrollAssetsRight = this.root.getByRole('button', { + name: enMessages.g.scrollRight + }) this.attachmentChips = this.root.getByTestId('agent-attachment-chip') this.composer = this.root.getByRole('textbox', { name: /^Describe ideas/ }) this.composerPromptArea = this.root.getByTestId('composer-inline-input') @@ -63,9 +67,63 @@ export class AgentPanel { }) } + async scrollAssetsToEnd(): Promise { + const count = await this.attachmentChips.count() + for ( + let step = 0; + step < count && (await this.scrollAssetsRight.isEnabled()); + step++ + ) { + const target = await this.composerAssetSection.evaluate((element) => + Math.min( + element.scrollWidth - element.clientWidth, + element.scrollLeft + element.clientWidth + ) + ) + await this.scrollAssetsRight.click() + await expect + .poll(() => + this.composerAssetSection.evaluate((element) => element.scrollLeft) + ) + .toBeCloseTo(target, 0) + } + await expect(this.scrollAssetsRight).toBeDisabled() + } + + assetPreview(name: string): Locator { + return this.page.getByRole('dialog', { name, exact: true }) + } + + previewAssetButton(name: string): Locator { + return this.attachmentChip(name).getByRole('button', { + name: enMessages.agent.previewAsset.replace('{name}', name), + exact: true + }) + } + + async expectAttachmentFullyVisible(name: string): Promise { + const tray = this.root.getByRole('region', { + name: enMessages.assetBrowser.assets, + exact: true + }) + await expect + .poll(async () => { + const [card, viewport] = await Promise.all([ + this.attachmentChip(name).boundingBox(), + tray.boundingBox() + ]) + return ( + !!card && + !!viewport && + card.y >= viewport.y && + card.y + card.height <= viewport.y + viewport.height + ) + }) + .toBe(true) + } + /** - * The composer attachment carrying `name`. Matches on the chip's own - * attribute rather than its text, which truncates at `max-w-32`. + * The composer attachment carrying `name`. * * `name` is a filename and may legitimately contain a quote or backslash, so * it is escaped for the double-quoted CSS string rather than interpolated diff --git a/browser_tests/fixtures/data/assetFixtures.ts b/browser_tests/fixtures/data/assetFixtures.ts index c6fc819d6ea..41f1181cdf6 100644 --- a/browser_tests/fixtures/data/assetFixtures.ts +++ b/browser_tests/fixtures/data/assetFixtures.ts @@ -75,6 +75,17 @@ export const AGENT_VIDEO_ASSET: Asset = createOutputAsset({ last_access_time: '2026-09-18T00:00:00.000Z' }) +export const AGENT_AUDIO_ASSET: Asset = createOutputAsset({ + id: '33333333-3333-4333-a333-333333333333', + hash: 'blake3:3333333333333333333333333333333333333333333333333333333333333333', + name: 'agent_generated_audio.wav', + mime_type: 'audio/wav', + size: 24_044, + created_at: '2026-09-18T00:00:00.000Z', + updated_at: '2026-09-18T00:00:00.000Z', + last_access_time: '2026-09-18T00:00:00.000Z' +}) + export const STALE_TEMP_FILENAME = 'ComfyUI_temp_thsmm_00001_.png' export const STALE_TEMP_CARD_TEXT = STALE_TEMP_FILENAME.replace(/\.[^.]+$/, '') export const STALE_TEMP_ASSET: Asset = createOutputAsset({ diff --git a/browser_tests/fixtures/helpers/DragDropHelper.ts b/browser_tests/fixtures/helpers/DragDropHelper.ts index 06740fc8811..532f635b67a 100644 --- a/browser_tests/fixtures/helpers/DragDropHelper.ts +++ b/browser_tests/fixtures/helpers/DragDropHelper.ts @@ -1,7 +1,7 @@ -import { readFileSync } from 'fs' +import { readFileSync, statSync } from 'fs' import { basename } from 'path' -import type { Page } from '@playwright/test' +import type { Locator, Page } from '@playwright/test' import type { Position } from '@e2e/fixtures/types' import { getMimeType } from '@e2e/fixtures/utils/mimeTypeUtil' @@ -11,6 +11,28 @@ import { nextFrame } from '@e2e/fixtures/utils/timing' export class DragDropHelper { constructor(private readonly page: Page) {} + async dropFilesOn(target: Locator, filePaths: string[]): Promise { + const files = filePaths.map((filePath) => ({ + bytes: Array.from(readFileSync(filePath)), + name: basename(filePath), + type: getMimeType(basename(filePath)), + lastModified: statSync(filePath).mtimeMs + })) + const transfer = await this.page.evaluateHandle((files) => { + const transfer = new DataTransfer() + for (const { bytes, name, type, lastModified } of files) + transfer.items.add( + new File([new Uint8Array(bytes)], name, { type, lastModified }) + ) + return transfer + }, files) + try { + await target.dispatchEvent('drop', { dataTransfer: transfer }) + } finally { + await transfer.dispose() + } + } + async dragAndDropExternalResource( options: { fileName?: string diff --git a/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts b/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts new file mode 100644 index 00000000000..f5244341920 --- /dev/null +++ b/browser_tests/tests/agent/agentAssetPreviewPlayback.spec.ts @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict' + +import { expect } from '@playwright/test' + +import { + agentTest as test, + bootAgentApp +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { AssetsSidebarTab } from '@e2e/fixtures/components/SidebarTab' +import { + AGENT_AUDIO_ASSET, + AGENT_VIDEO_ASSET +} from '@e2e/fixtures/data/assetFixtures' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.describe('Asset preview playback', { tag: '@cloud' }, () => { + for (const { kind, asset, file, contentType } of [ + { + kind: 'video', + asset: AGENT_VIDEO_ASSET, + file: 'plain_video.mp4', + contentType: 'video/mp4' + }, + { + kind: 'audio', + asset: AGENT_AUDIO_ASSET, + file: 'silence.wav', + contentType: 'audio/wav' + } + ] as const) { + test(`stops ${kind} at close and reopens paused during the exit animation`, async ({ + page, + agentFlagEnabled + }) => { + await page.route(`**/api/assets/${asset.id}/content**`, (route) => + route.fulfill({ path: assetPath(file), contentType }) + ) + await bootAgentApp(page, agentFlagEnabled, { + assets: { assets: [asset], total: 1, has_more: false } + }) + const panel = new AgentPanel(page) + await panel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(panel.root) + await panel.composer.fill('Keep this draft') + const trigger = panel.previewAssetButton(asset.name) + await trigger.press('Enter') + const player = panel.assetPreview(asset.name).locator(kind) + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.readyState) + ) + .toBeGreaterThanOrEqual(2) + await player.press('Space') + await expect + .poll(() => player.evaluate((media: HTMLMediaElement) => media.paused)) + .toBe(false) + const playingElement = await player.elementHandle() + assert(playingElement) + try { + await page.keyboard.press('Escape') + expect( + await playingElement.evaluate( + (media) => media instanceof HTMLMediaElement && media.paused + ) + ).toBe(true) + await page.keyboard.press('Enter') + await expect(trigger).toHaveAttribute('aria-expanded', 'true') + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (media: HTMLMediaElement) => + media.paused && media.currentTime === 0 + ) + ) + .toBe(true) + await player.press('Space') + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.paused) + ) + .toBe(false) + const reopenedElement = await player.elementHandle() + assert(reopenedElement) + try { + await panel.composer.click() + expect( + await reopenedElement.evaluate( + (media) => media instanceof HTMLMediaElement && media.paused + ) + ).toBe(true) + } finally { + await reopenedElement.dispose() + } + await expect(trigger).toHaveAttribute('aria-expanded', 'false') + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + } finally { + await playingElement.dispose() + } + }) + } +}) diff --git a/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts b/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts new file mode 100644 index 00000000000..e81ba4bdb9b --- /dev/null +++ b/browser_tests/tests/agent/agentAssetPreviewSwitching.spec.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict' + +import { expect } from '@playwright/test' + +import { + agentTest as test, + bootAgentApp +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { AssetsSidebarTab } from '@e2e/fixtures/components/SidebarTab' +import { + AGENT_AUDIO_ASSET, + AGENT_VIDEO_ASSET +} from '@e2e/fixtures/data/assetFixtures' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.describe('Single asset tray preview', { tag: '@cloud' }, () => { + test.beforeEach(async ({ page, agentFlagEnabled }) => { + await page.route( + `**/api/assets/${AGENT_AUDIO_ASSET.id}/content**`, + (route) => + route.fulfill({ + path: assetPath('silence.wav'), + contentType: 'audio/wav' + }) + ) + await page.route( + `**/api/assets/${AGENT_VIDEO_ASSET.id}/content**`, + (route) => + route.fulfill({ + path: assetPath('plain_video.mp4'), + contentType: 'video/mp4' + }) + ) + await bootAgentApp(page, agentFlagEnabled, { + assets: { + assets: [AGENT_AUDIO_ASSET, AGENT_VIDEO_ASSET], + total: 2, + has_more: false + } + }) + const panel = new AgentPanel(page) + await panel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.waitForAssets(2) + await assets.getAssetCardByName('agent_generated_audio').dragTo(panel.root) + await expect(panel.attachmentChip(AGENT_AUDIO_ASSET.name)).toBeVisible() + await assets.getAssetCardByName('agent_generated_video').dragTo(panel.root) + await expect(panel.attachmentChips).toHaveCount(2) + await panel.composer.fill('Keep this draft') + }) + + test('replaces a clicked preview on hover and preserves keyboard switching and dismissal', async ({ + page + }) => { + const panel = new AgentPanel(page) + const audio = panel.previewAssetButton(AGENT_AUDIO_ASSET.name) + const video = panel.previewAssetButton(AGENT_VIDEO_ASSET.name) + await audio.click() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toBeVisible() + await video.hover() + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toBeVisible() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toHaveCount(0) + await expect(audio).toHaveAttribute('aria-expanded', 'false') + await panel.composer.hover() + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toHaveCount(0) + + await audio.press('Enter') + await expect( + panel.assetPreview(AGENT_AUDIO_ASSET.name).locator('audio') + ).toBeFocused() + await video.press('Space') + await expect( + panel.assetPreview(AGENT_VIDEO_ASSET.name).locator('video') + ).toBeFocused() + await expect(panel.assetPreview(AGENT_AUDIO_ASSET.name)).toHaveCount(0) + await page.keyboard.press('Escape') + await expect(panel.assetPreview(AGENT_VIDEO_ASSET.name)).toHaveCount(0) + await expect(video).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + }) + + for (const { kind, asset, nextAsset } of [ + { kind: 'audio', asset: AGENT_AUDIO_ASSET, nextAsset: AGENT_VIDEO_ASSET }, + { kind: 'video', asset: AGENT_VIDEO_ASSET, nextAsset: AGENT_AUDIO_ASSET } + ] as const) { + test(`stops native ${kind} playback when hover replaces it and reopens paused`, async ({ + page + }) => { + const panel = new AgentPanel(page) + const trigger = panel.previewAssetButton(asset.name) + await trigger.click() + const player = panel.assetPreview(asset.name).locator(kind) + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate((media: HTMLMediaElement) => media.readyState) + ) + .toBeGreaterThanOrEqual(2) + await player.press('Space') + await expect + .poll(() => player.evaluate((media: HTMLMediaElement) => media.paused)) + .toBe(false) + const playing = await player.elementHandle() + assert(playing) + try { + await panel.previewAssetButton(nextAsset.name).hover() + await expect(panel.assetPreview(nextAsset.name)).toBeVisible() + await expect + .poll(() => + playing.evaluate( + (media) => + media instanceof HTMLMediaElement && + media.paused && + media.currentTime === 0 + ) + ) + .toBe(true) + await expect(panel.assetPreview(asset.name)).toHaveCount(0) + await trigger.press('Enter') + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (media: HTMLMediaElement) => + media.paused && media.currentTime === 0 + ) + ) + .toBe(true) + await expect(panel.assetPreview(nextAsset.name)).toHaveCount(0) + await page.keyboard.press('Escape') + await expect(trigger).toBeFocused() + await expect(panel.assetPreview(asset.name)).toHaveCount(0) + } finally { + await playing.dispose() + } + }) + } +}) diff --git a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts b/browser_tests/tests/agent/agentAttachJsonFile.spec.ts index a9e5605627e..456529ec19c 100644 --- a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts +++ b/browser_tests/tests/agent/agentAttachJsonFile.spec.ts @@ -41,13 +41,18 @@ test.describe( // that was reported (the OS picker hid .json files) and the actual fix // (AGENT_ATTACH_ACCEPT now lists .json/application/json). await fileInput.setInputFiles(assetPath('default.json')) - await expect(assetSection).toContainText('default.json') + const attachment = assetSection.getByRole('group', { + name: 'default.json', + exact: true + }) + await expect(attachment).toBeVisible() await expect( assetSection.locator(`[aria-label="${enMessages.agent.uploading}"]`) ).toHaveCount(0) - await assetSection - .getByRole('button', { name: enMessages.agent.remove, exact: true }) + await attachment.hover() + await attachment + .getByRole('button', { name: 'Remove default.json', exact: true }) .click() await expect(assetSection).toHaveCount(0) }) diff --git a/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts b/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts index ca3bba0a9c8..ce8f96cb1ea 100644 --- a/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts +++ b/browser_tests/tests/agent/agentAttachmentHistoryPersistence.spec.ts @@ -192,9 +192,12 @@ for (const scenario of [ const composer = panel.getByRole('textbox', { name: /^Describe ideas/ }) await dropAssets(page, composer, scenario.assets, 'input') for (const filename of filenames) { - await expect(panel.getByTestId('composer-asset-section')).toContainText( - filename - ) + await expect( + panel.getByTestId('composer-asset-section').getByRole('group', { + name: filename, + exact: true + }) + ).toBeVisible() } await composer.pressSequentially('compare these attachments') diff --git a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts index f59fee1e7e4..a71501b0416 100644 --- a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts +++ b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts @@ -40,7 +40,9 @@ test.describe('Agent attachment cancellation', { tag: '@cloud' }, () => { }) await uploadReceived await expect( - panel.getByTestId('composer-asset-section').getByText('pending.mp4') + panel + .getByTestId('composer-asset-section') + .getByRole('group', { name: 'pending.mp4' }) ).toBeVisible() await expect(send).toBeDisabled() @@ -51,9 +53,9 @@ test.describe('Agent attachment cancellation', { tag: '@cloud' }, () => { await openButton.click() await expect(composer).toHaveText('Keep this draft') - await expect(panel.getByText('pending.mp4', { exact: true })).toHaveCount( - 0 - ) + await expect( + panel.getByRole('group', { name: 'pending.mp4' }) + ).toHaveCount(0) await expect(send).toBeEnabled() } finally { releaseUpload() diff --git a/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts new file mode 100644 index 00000000000..53fd750fbc0 --- /dev/null +++ b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts @@ -0,0 +1,80 @@ +import type { UploadImageResponse } from '@comfyorg/ingest-types' +import { expect } from '@playwright/test' + +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' +import { agentTest as test } from '@e2e/tests/agent/agentPanelMocks' + +const ATTACHMENT_COUNT = 36 + +// Regression: https://linear.app/comfyorg/issue/FE-3202 +test.describe('Agent composer bulk attachments', { tag: '@cloud' }, () => { + test('keeps a visible send path and a reachable chip list after attaching dozens of images', async ({ + agentPanel, + comfyPage, + postedMessages + }) => { + const page = comfyPage.page + // Each attachment ref is the upload response name, so a shared name would + // make the POST assertion below pass on one ref echoed 36 times. Unique + // names are what let it distinguish that from 36 refs actually travelling. + const uploadedRefs: string[] = [] + await page.route('**/api/upload/image', (route) => { + const name = `reference-${uploadedRefs.length + 1}.png` + const uploaded: UploadImageResponse = { + name, + subfolder: '', + type: 'input' + } + uploadedRefs.push(name) + return route.fulfill(jsonRoute(uploaded)) + }) + + await agentPanel.open() + await agentPanel.selectWorkflow() + await agentPanel.composer.fill('Compare these references') + await agentPanel.fileInput.setInputFiles( + Array.from({ length: ATTACHMENT_COUNT }, (_, index) => ({ + name: `reference-${index + 1}.png`, + mimeType: 'image/png', + buffer: Buffer.from(`reference-${index + 1}`) + })) + ) + + await expect(agentPanel.attachmentChips).toHaveCount(ATTACHMENT_COUNT) + await expect(agentPanel.attachmentChips.last()).not.toBeInViewport({ + ratio: 1 + }) + await agentPanel.scrollAssetsToEnd() + await expect(agentPanel.attachmentChips.last()).toBeInViewport({ ratio: 1 }) + await expect(agentPanel.sendButton).toBeEnabled() + // ratio: 1 rather than the default ratio: 0 — a Send button clipped down to + // a sliver is the FE-3202 symptom, and it satisfies ratio: 0. + await expect(agentPanel.sendButton).toBeInViewport({ ratio: 1 }) + + await agentPanel.composerAssetSection.hover() + await page.mouse.wheel(-10000, 0) + await expect(agentPanel.attachmentChips.first()).toBeInViewport({ + ratio: 1 + }) + await expect(agentPanel.attachmentChips.last()).not.toBeInViewport({ + ratio: 1 + }) + await page.mouse.wheel(10000, 0) + await expect( + agentPanel.attachmentChip(`reference-${ATTACHMENT_COUNT}.png`) + ).toBeInViewport({ ratio: 1 }) + await expect(agentPanel.sendButton).toBeInViewport({ ratio: 1 }) + + // Click, not Enter: the keyboard path posts even when the button has been + // pushed out of reach, so only a real click proves Send is usable. + await agentPanel.sendButton.click() + await expect.poll(() => postedMessages.length).toBe(1) + + expect(uploadedRefs).toHaveLength(ATTACHMENT_COUNT) + const postedAttachments: string[] = JSON.parse( + postedMessages[0] + ).attachments + expect(postedAttachments).toHaveLength(ATTACHMENT_COUNT) + expect(postedAttachments).toEqual(expect.arrayContaining(uploadedRefs)) + }) +}) diff --git a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts index 26cf330b2a2..6d51327f22a 100644 --- a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts +++ b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts @@ -1,3 +1,5 @@ +import assert from 'node:assert/strict' + import { expect } from '@playwright/test' import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' @@ -36,12 +38,16 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { ) await bootAgentApp(page, agentFlagEnabled, { - assets: { assets: [AGENT_VIDEO_ASSET], total: 1, has_more: false } + assets: { + assets: [{ ...AGENT_VIDEO_ASSET, display_name: null }], + total: 1, + has_more: false + } }) }) for (const view of ['grid', 'list'] as const) { - test(`attaches an asset dragged onto the composer from ${view} view`, async ({ + test(`re-attaches a library asset without a display name from ${view} view`, async ({ page }) => { const agentPanel = new AgentPanel(page) @@ -53,6 +59,7 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { if (view === 'list') { await assets.openSettingsMenu() await assets.listViewOption.click() + await assets.closeSettingsMenu() } const source = @@ -68,9 +75,171 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { // PM-1157/PM-1158 failure, and it would pass a bare count assertion. await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() await expect(agentPanel.attachmentChips).toHaveCount(1) + await source.dragTo(agentPanel.root) + await expect(agentPanel.attachmentChips).toHaveCount(1) + const duplicateNotice = page.getByRole('alert').filter({ + hasText: `${ASSET_NAME} is already in the asset tray` + }) + await expect(duplicateNotice).toHaveCount(1) + await duplicateNotice.getByRole('button', { name: 'Close' }).click() + await agentPanel.composer.fill('Keep this draft @agent_generated') + await agentPanel.root + .getByRole('menuitem', { name: ASSET_NAME, exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) + await agentPanel.attachmentChip(ASSET_NAME).hover() + await agentPanel + .attachmentChip(ASSET_NAME) + .getByRole('button', { name: `Remove ${ASSET_NAME}`, exact: true }) + .click() + await expect(agentPanel.attachmentChips).toHaveCount(0) + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await expect(agentPanel.composer).toContainText('Keep this draft') + await source.dragTo(agentPanel.root) + await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() + await expect(agentPanel.attachmentChips).toHaveCount(1) + await agentPanel.composer.press('End') + await agentPanel.composer.pressSequentially('@agent_generated') + await agentPanel.root + .getByRole('menuitem', { name: ASSET_NAME, exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) }) } + test('reuses a decoded video poster and supports native keyboard playback with cleanup', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(agentPanel.root) + const poster = agentPanel + .attachmentChip(ASSET_NAME) + .getByRole('img', { name: ASSET_NAME, exact: true }) + await expect(poster).toHaveAttribute('src', /^blob:/) + await expect + .poll(() => + poster.evaluate( + (image: HTMLImageElement) => image.complete && image.naturalWidth > 0 + ) + ) + .toBe(true) + const posterUrl = await poster.getAttribute('src') + assert(posterUrl) + await agentPanel.composer.fill('Animate @agent_generated') + const suggestion = agentPanel.root.getByRole('menuitem', { + name: ASSET_NAME, + exact: true + }) + await expect(suggestion.locator('img')).toHaveAttribute('src', posterUrl) + await suggestion.click() + await expect( + agentPanel.composer.getByTestId('asset-reference-chip').locator('img') + ).toHaveAttribute('src', posterUrl) + await agentPanel.composer.hover() + await expect(agentPanel.assetPreview(ASSET_NAME)).toHaveCount(0) + await agentPanel.composer.pressSequentially(' with motion') + await expect(agentPanel.composer).toBeFocused() + await agentPanel.composerAssetSection.focus() + await agentPanel.composerAssetSection.press('Tab') + const trigger = agentPanel.previewAssetButton(ASSET_NAME) + await expect(trigger).toBeFocused() + await trigger.press('Enter') + const preview = agentPanel.assetPreview(ASSET_NAME) + const player = preview.locator('video') + await expect(preview).toBeVisible() + await expect(player).toBeFocused() + await expect(player).toHaveAttribute('controls', '') + await expect(player).not.toHaveAttribute('autoplay') + await expect(player).toHaveAttribute('poster', posterUrl) + await expect + .poll(() => + player.evaluate((video: HTMLVideoElement) => video.readyState) + ) + .toBeGreaterThanOrEqual(2) + await expect + .poll(() => player.evaluate((video: HTMLVideoElement) => video.paused)) + .toBe(true) + await player.press('Space') + await expect + .poll(() => player.evaluate((video: HTMLVideoElement) => video.paused)) + .toBe(false) + await expect + .poll(() => + player.evaluate((video: HTMLVideoElement) => video.currentTime) + ) + .toBeGreaterThan(0) + const playingElement = await player.elementHandle() + assert(playingElement) + await player.press('Escape') + await expect(preview).toHaveCount(0) + await expect(trigger).toBeFocused() + await expect + .poll(() => + playingElement.evaluate( + (video) => + video instanceof HTMLVideoElement && + video.paused && + !video.isConnected + ) + ) + .toBe(true) + await playingElement.dispose() + await trigger.press('Space') + await expect(player).toBeVisible() + await expect(player).toBeFocused() + await expect + .poll(() => + player.evaluate( + (video: HTMLVideoElement) => video.paused && video.currentTime === 0 + ) + ) + .toBe(true) + await player.press('Escape') + await expect(preview).toHaveCount(0) + await expect(agentPanel.composer).toContainText(' with motion') + }) + + test.describe('a short viewport', () => { + test.use({ viewport: { width: 1024, height: 520 } }) + + test('keeps the complete asset card visible as the prompt grows', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + await agentPanel.open() + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(agentPanel.root) + await expect(agentPanel.attachmentChip(ASSET_NAME)).toBeVisible() + await agentPanel.expectAttachmentFullyVisible(ASSET_NAME) + const original = await agentPanel.attachmentChip(ASSET_NAME).boundingBox() + assert(original) + await agentPanel.composer.fill( + 'Keep this line and the attached video.\n'.repeat(40) + ) + await agentPanel.expectAttachmentFullyVisible(ASSET_NAME) + await expect + .poll( + async () => + (await agentPanel.attachmentChip(ASSET_NAME).boundingBox())?.height + ) + .toBe(original.height) + await expect(agentPanel.attachmentChips).toHaveCount(1) + await expect(agentPanel.composer).toContainText( + 'Keep this line and the attached video.' + ) + }) + }) + // A filename is user data and may contain a quote or a backslash, which would // break the attribute selector the chip is matched by. Exercised end to end // rather than unit-tested, so a regression in the escaping shows up as the diff --git a/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts b/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts index 27d4ac49398..c5d61db5362 100644 --- a/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts +++ b/browser_tests/tests/agent/agentComposerPlaceholderReset.spec.ts @@ -60,9 +60,10 @@ test.describe( const removeButton = panel.getByRole('button', { name: `Remove KSampler #${node.id} reference` }) - await expect(removeButton).toBeVisible() - await expect(placeholder).toHaveCount(0) + await expect(placeholder).toBeHidden() + await panel.getByTestId('node-reference-chip').hover() + await expect(removeButton).toBeVisible() await removeButton.click() await expect(removeButton).toHaveCount(0) diff --git a/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts b/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts new file mode 100644 index 00000000000..dd273091746 --- /dev/null +++ b/browser_tests/tests/agent/agentComposerRepeatedFileDrop.spec.ts @@ -0,0 +1,101 @@ +import { expect } from '@playwright/test' + +import { bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { inlineReferencesTest as test } from '@e2e/fixtures/agentInlineReferencesFixture' +import { DragDropHelper } from '@e2e/fixtures/helpers/DragDropHelper' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test( + 'keeps repeated file drops in one tray item while pending and after uploading', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.composer.fill('Keep this draft') + const drops = new DragDropHelper(page) + const file = assetPath('test_upload_image.png') + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect( + panel.root.getByRole('status', { name: 'Uploading' }) + ).toBeVisible() + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + const notice = page + .getByRole('alert') + .filter({ hasText: 'test_upload_image.png is already in the asset tray' }) + await expect(notice).toHaveCount(1) + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + await notice.getByRole('button', { name: 'Close' }).click() + assetUpload.finish() + await expect( + panel.root.getByRole('status', { name: 'Uploading' }) + ).toHaveCount(0) + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect(notice).toHaveCount(1) + await notice.getByRole('button', { name: 'Close' }).click() + await panel.composer.press('End') + await panel.composer.pressSequentially(' @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .click() + await panel.composer.pressSequentially(' @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .click() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(2) + await panel.attachmentChip('test_upload_image.png').hover() + await panel + .attachmentChip('test_upload_image.png') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + .click() + await expect(panel.attachmentChips).toHaveCount(0) + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await expect(panel.composer).toContainText('Keep this draft') + } +) + +test( + 'shows one plural duplicate notice while continuing a mixed pending batch', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.composer.fill('Keep this draft') + const drops = new DragDropHelper(page) + const file = assetPath('test_upload_image.png') + await drops.dropFilesOn(panel.root, [file]) + await expect(panel.attachmentChips).toHaveCount(1) + await drops.dropFilesOn(panel.root, [file, file, assetPath('silence.wav')]) + await expect(panel.attachmentChips).toHaveCount(2) + await expect(panel.attachmentChip('test_upload_image.png')).toBeVisible() + await expect(panel.attachmentChip('silence.wav')).toBeVisible() + await expect( + page.getByRole('alert').filter({ hasText: 'Skipped 2 duplicate assets' }) + ).toHaveCount(1) + await expect( + panel.root.getByRole('status', { name: 'Uploading', exact: true }) + ).toHaveCount(2) + await expect(panel.composer).toBeFocused() + await expect(panel.composer).toContainText('Keep this draft') + assetUpload.finish() + await expect( + panel.root.getByRole('status', { name: 'Uploading', exact: true }) + ).toHaveCount(0) + await expect(panel.attachmentChips).toHaveCount(2) + } +) diff --git a/browser_tests/tests/agent/agentComposerTouch.spec.ts b/browser_tests/tests/agent/agentComposerTouch.spec.ts new file mode 100644 index 00000000000..67806570a3f --- /dev/null +++ b/browser_tests/tests/agent/agentComposerTouch.spec.ts @@ -0,0 +1,67 @@ +import { expect } from '@playwright/test' + +import { bootAgentApp } from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { inlineReferencesTest as test } from '@e2e/fixtures/agentInlineReferencesFixture' +import { assetPath } from '@e2e/fixtures/utils/paths' + +test.use({ hasTouch: true, viewport: { width: 1024, height: 768 } }) + +test( + 'discloses filenames and removes a staged asset without hover', + { tag: '@cloud' }, + async ({ page, assetUpload }) => { + await bootAgentApp(page, true) + const panel = new AgentPanel(page) + await panel.open() + await panel.fileInput.setInputFiles(assetPath('test_upload_image.png')) + await panel.fileInput.setInputFiles({ + name: 'notes.txt', + mimeType: 'text/plain', + buffer: Buffer.from('Notes') + }) + assetUpload.finish() + await expect(panel.attachmentChips).toHaveCount(2) + await panel.composer.fill('Keep this draft @test_upload') + await panel.root + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) + .tap() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(1) + await panel + .attachmentChip('notes.txt') + .getByRole('button', { name: 'Preview notes.txt', exact: true }) + .tap() + const preview = page.getByRole('dialog', { name: 'notes.txt', exact: true }) + await expect(preview.getByText('notes.txt', { exact: true })).toBeVisible() + await panel.previewAssetButton('test_upload_image.png').tap() + await expect(panel.assetPreview('test_upload_image.png')).toBeVisible() + await expect(preview).toHaveCount(0) + await panel.previewAssetButton('notes.txt').tap() + await expect(preview).toBeVisible() + await expect(panel.assetPreview('test_upload_image.png')).toHaveCount(0) + await panel.composer.tap() + await expect(preview).toHaveCount(0) + const remove = panel + .attachmentChip('test_upload_image.png') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + await expect + .poll(() => + remove.evaluate((element) => + element.checkVisibility({ checkOpacity: true }) + ) + ) + .toBe(true) + await remove.tap() + await expect(panel.attachmentChip('test_upload_image.png')).toHaveCount(0) + await expect(panel.attachmentChip('notes.txt')).toBeVisible() + await expect( + panel.composer.getByTestId('asset-reference-chip') + ).toHaveCount(0) + await expect(panel.composer).toContainText('Keep this draft') + } +) diff --git a/browser_tests/tests/agent/agentInlineReferences.spec.ts b/browser_tests/tests/agent/agentInlineReferences.spec.ts index 4d97dfbe7f8..994b5fbdbfe 100644 --- a/browser_tests/tests/agent/agentInlineReferences.spec.ts +++ b/browser_tests/tests/agent/agentInlineReferences.spec.ts @@ -16,7 +16,7 @@ test.use({ }) test( - 'keeps inline node and asset removal synchronized with the upper rows and Undo', + 'keeps asset attachments in the tray while inline references follow prompt editing and Undo', { tag: ['@cloud', '@ui'] }, async ({ page, workflowSelection, assetUpload }) => { await page.locator('#comfy-file-input').setInputFiles({ @@ -44,39 +44,50 @@ test( await panel .getByTestId('agent-file-input') .setInputFiles(assetPath('test_upload_image.png')) - const text = 'Adjust Color balance #12 to match test_upload_image.png ' - await expect(editor).toHaveText(text) + const tray = panel.getByTestId('composer-asset-section') + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await expect(tray).not.toContainText('test_upload_image.png') + await expect(editor).toHaveText('Adjust Color balance #12 to match ') + await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await expect( panel.getByRole('button', { name: enMessages.agent.send, exact: true }) ).toBeDisabled() + await editor.pressSequentially('@test_upload') await panel - .getByTestId('composer-asset-section') - .getByRole('button', { name: enMessages.agent.remove, exact: true }) + .getByRole('menuitem', { name: 'test_upload_image.png', exact: true }) .click() - assetUpload.finish() - await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) - await editor.press('ControlOrMeta+z') + const text = 'Adjust Color balance #12 to match test_upload_image.png ' await expect(editor).toHaveText(text) + assetUpload.finish() await expect( panel.getByRole('button', { name: enMessages.agent.send, exact: true }) ).toBeEnabled() + await editor.press('Backspace') + await editor.press('Backspace') + await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await editor.press('ControlOrMeta+z') + await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(1) + await editor.getByTestId('node-reference-chip').hover() await panel .getByRole('button', { name: 'Remove Color balance #12 reference' }) .click() - await expect(editor).toHaveText('Adjust to match test_upload_image.png ') + await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await editor.press('ControlOrMeta+z') - await expect(editor).toHaveText(text) - await expect(panel.getByTestId('composer-node-section')).toHaveText( - 'Color balance' - ) + await expect(editor.getByTestId('node-reference-chip')).toHaveCount(1) + await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) await editor.press('ControlOrMeta+a') await editor.press('ControlOrMeta+c') await expect .poll(() => page.evaluate(() => navigator.clipboard.readText())) .toBe( - 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png] ' + 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png]' ) await editor.press('ArrowRight') await page @@ -94,15 +105,37 @@ test( await expect.poll(() => workflowSelection.savedPaths.length).toBe(2) workflowSelection.finishSave(true) await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) + await editor.pressSequentially(' @') + await panel + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + panel.getByText(enMessages.agent.noNodesToReference, { exact: true }) + ).toBeVisible() + await expect( + panel.getByRole('menuitem', { name: /Color balance/ }) + ).toHaveCount(0) + await editor.press('Escape') await editor.press('ControlOrMeta+a') await editor.press('Backspace') await editor.press('ControlOrMeta+v') await expect(editor).toHaveText( - 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png] ' + 'Adjust @[Node: Color balance #12] to match @[Image: test_upload_image.png]' ) await expect(editor.getByTestId('node-reference-chip')).toHaveCount(0) await expect(editor.getByTestId('asset-reference-chip')).toHaveCount(0) await expect(panel.getByTestId('composer-node-section')).toHaveCount(0) + await expect( + tray.getByRole('group', { name: 'test_upload_image.png' }) + ).toBeVisible() + await tray.getByRole('group', { name: 'test_upload_image.png' }).hover() + await panel + .getByTestId('composer-asset-section') + .getByRole('button', { + name: 'Remove test_upload_image.png', + exact: true + }) + .click() await expect(panel.getByTestId('composer-asset-section')).toHaveCount(0) await expect( panel.getByRole('button', { name: enMessages.agent.switchWorkflow }) diff --git a/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts b/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts new file mode 100644 index 00000000000..e90e55583f4 --- /dev/null +++ b/browser_tests/tests/agent/agentNodeMentionFreshness.spec.ts @@ -0,0 +1,134 @@ +import { expect } from '@playwright/test' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { zComfyWorkflow } from '@/platform/workflow/validation/schemas/workflowSchema' +import type { ComfyNodeDef } from '@/schemas/nodeDefSchema' +import workflow from '@e2e/assets/nodes/single_ksampler.json' with { type: 'json' } +import { + agentTest as test, + bootAgentApp, + loadIntoBootWorkflow, + mockWorkflowPersistence +} from '@e2e/fixtures/agentPanelFixture' +import { AgentPanel } from '@e2e/fixtures/components/AgentPanel' +import { Topbar } from '@e2e/fixtures/components/Topbar' +import { nextFrame } from '@e2e/fixtures/utils/timing' +import { VueNodeHelpers } from '@e2e/fixtures/VueNodeHelpers' + +const kSamplerNodeDef: ComfyNodeDef = { + name: 'KSampler', + display_name: 'KSampler', + description: '', + category: 'agent replay', + python_module: 'browser_tests.agent_replay', + output_node: false, + input: { + required: { + model: ['MODEL', {}], + seed: ['INT', { default: 0, min: 0, max: Number.MAX_SAFE_INTEGER }], + steps: ['INT', { default: 20, min: 1, max: 10_000 }], + cfg: ['FLOAT', { default: 8, min: 0, max: 100, step: 0.1 }], + sampler_name: [['euler'], {}], + scheduler: [['normal'], {}], + positive: ['CONDITIONING', {}], + negative: ['CONDITIONING', {}], + latent_image: ['LATENT', {}], + denoise: ['FLOAT', { default: 1, min: 0, max: 1, step: 0.01 }] + } + }, + output: ['LATENT'], + output_name: ['LATENT'], + output_is_list: [false] +} + +test.describe( + 'Node mention freshness (FE-3220)', + { tag: ['@cloud', '@vue-nodes'] }, + () => { + test.beforeEach(async ({ page, agentFlagEnabled }) => { + await bootAgentApp(page, agentFlagEnabled, { + settings: { 'Comfy.VueNodes.Enabled': true }, + objectInfo: { KSampler: kSamplerNodeDef }, + beforeNavigate: (page) => + mockWorkflowPersistence(page, 'a81718a4-02ae-41e6-ae85-c33b7bb880f6') + }) + await loadIntoBootWorkflow(page, zComfyWorkflow.parse(workflow)) + const agentPanel = new AgentPanel(page) + await agentPanel.open() + await agentPanel.selectWorkflow() + }) + + test('refreshes nodes after renaming, deleting and undoing in the target workflow', async ({ + page + }) => { + const agentPanel = new AgentPanel(page) + const vueNodes = new VueNodeHelpers(page) + await agentPanel.composer.fill('Keep this draft @') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByRole('menuitem', { name: 'KSampler', exact: true }) + ).toBeVisible() + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await vueNodes.renameNode('3', 'Color grade') + await nextFrame(page) + await expect( + vueNodes.getNodeLocator('3').getByTestId('node-title') + ).toHaveText('Color grade') + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByRole('menuitem', { + name: 'Color grade', + exact: true + }) + ).toBeVisible() + await expect( + agentPanel.root.getByRole('menuitem', { name: 'KSampler', exact: true }) + ).toHaveCount(0) + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await vueNodes.deleteNode('3') + await nextFrame(page) + await expect(vueNodes.getNodeLocator('3')).toHaveCount(0) + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await expect( + agentPanel.root.getByText(enMessages.agent.noNodesToReference, { + exact: true + }) + ).toBeVisible() + await expect( + agentPanel.root.getByRole('menuitem', { + name: 'Color grade', + exact: true + }) + ).toHaveCount(0) + await agentPanel.composer.press('Escape') + await agentPanel.composer.press('Backspace') + await new Topbar(page).triggerTopbarCommand(['Edit', 'Undo']) + await nextFrame(page) + await expect( + vueNodes.getNodeLocator('3').getByTestId('node-title') + ).toHaveText('Color grade') + await expect(agentPanel.composer).toHaveText('Keep this draft ') + await agentPanel.composer.pressSequentially('@') + await agentPanel.root + .getByRole('menuitem', { name: enMessages.agent.nodes, exact: true }) + .click() + await agentPanel.root + .getByRole('menuitem', { name: 'Color grade', exact: true }) + .click() + await expect( + agentPanel.composer.getByTestId('node-reference-chip') + ).toHaveText('Color grade #3') + await expect(agentPanel.composer).toContainText('Keep this draft') + }) + } +) diff --git a/browser_tests/tests/agent/agentPanel.spec.ts b/browser_tests/tests/agent/agentPanel.spec.ts index b92dbbb13ad..04019db6c35 100644 --- a/browser_tests/tests/agent/agentPanel.spec.ts +++ b/browser_tests/tests/agent/agentPanel.spec.ts @@ -474,7 +474,9 @@ test.describe('In-App Agent panel', { tag: '@cloud' }, () => { }) expect((await uploadResponse).ok()).toBe(true) await expect( - panel.getByTestId('composer-asset-section').getByText('movie.mp4') + panel + .getByTestId('composer-asset-section') + .getByRole('group', { name: 'movie.mp4' }) ).toBeVisible() await expect.poll(() => uploadCount).toBe(1) @@ -486,9 +488,9 @@ test.describe('In-App Agent panel', { tag: '@cloud' }, () => { await expect( page.getByText('too-large.mp4 is larger than 24 MB') ).toBeVisible() - await expect(panel.getByText('too-large.mp4', { exact: true })).toHaveCount( - 0 - ) + await expect( + panel.getByRole('group', { name: 'too-large.mp4' }) + ).toHaveCount(0) await expect.poll(() => uploadCount).toBe(1) }) diff --git a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md new file mode 100644 index 00000000000..203dbcd3ca6 --- /dev/null +++ b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md @@ -0,0 +1,53 @@ +# ADR-AGENT-COMPOSER-0040: Independent Assets and Prompt References + +Date: 2026-10-05 + +## Status + +Proposed + +## Context + +An asset added to the agent composer must be sent even without an inline +mention. Users may mention one asset repeatedly, delete or undo an occurrence, +and continue typing while an upload or send completes. Deriving attachments +from prompt references makes deletion remove the asset from the request and +makes repeated mentions duplicate it. + +## Decision + +The existing composer store owns the ordered included asset IDs and asset +metadata. Prompt references identify occurrences at text positions. Adding an +asset stages it; mentioning an included asset inserts an occurrence. Editor +history may restore occurrences only for assets still included in the tray. +Removing a tray item retires its identity and removes every occurrence; an +explicit new drop gets a fresh identity. + +Send snapshots retain both the included assets and the prompt. An untouched +failed draft restores both; newer user input wins. Local preview resources +remain owned by the store while included or held by a pending snapshot, and are +released when no longer retained. UI hover/focus highlighting and native scroll +measurements remain local to components and never change the draft. + +Playable media URLs and image poster URLs are distinct metadata. The store owns +one cancellable video-thumbnail job per asset/source and publishes its result to +the tray and all prompt occurrences. UI surfaces render that poster; only tray +hover mounts a player. A source change, removal or disposal cancels obsolete +capture. Pending snapshots retain media resources and failed recovery preserves +a poster completed for the same source while the send was pending. + +## Alternatives considered + +- Continue deriving attachments from references: cannot express an unmentioned + included asset or repeated references without changing send semantics. +- Keep a separate tray in a component: loses assets across remount and makes + upload completion, Undo and failed-send recovery depend on mounted UI. +- Create another persistent asset store: duplicates the existing composer's + submission lifecycle without a separate sharing requirement. + +## Consequences + +Attachment metadata is resolved through one owner when applying editor history. +Removing an inline occurrence keeps the included file. Send payloads contain +one entry per included asset while preserving every inline mention. Callers +stage and reference assets through distinct store commands. diff --git a/docs/adr/README.md b/docs/adr/README.md index 51f72fc3b05..6a2eedfce3f 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -11,6 +11,7 @@ An Architecture Decision Record captures an important architectural decision mad | ADR | Title | Status | Date | | ---------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------- | ---------- | | [AGENT-BINDING-0035](AGENT-BINDING-0035-document-identity-gates-persisted-workflow-tab-bindings.md) | Document Identity Gates Persisted Workflow Tab Bindings | Proposed | 2026-09-19 | +| [AGENT-COMPOSER-0040](AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md) | Independent Assets and Prompt References | Proposed | 2026-10-05 | | [AGENT-CONTEXT-0028](AGENT-CONTEXT-0028-separate-workflow-references-from-editor-tabs.md) | Separate Workflow References from Editor Tabs | Proposed | 2026-09-07 | | [AGENT-TARGET-0037](AGENT-TARGET-0037-commit-workflow-target-on-send-or-node-reference.md) | Commit Workflow Target on Explicit User Intent | Proposed | 2026-09-24 | | [ASSETS-DRAG-DROP-0035](ASSETS-DRAG-DROP-0035-asset-card-drag-uses-file-url.md) | Asset Card Drag-and-Drop Uses the File URL, Never the Preview URL | Proposed | 2026-09-19 | diff --git a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts index e1aac975d99..ec5e00ae789 100644 --- a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts +++ b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts @@ -238,7 +238,9 @@ describe('AssetsSidebarListView', () => { display_name: 'Clip', attachment_ref: 'clip.mp4', media_kind: 'video', - preview_url: undefined + preview_url: undefined, + media_url: + 'http://localhost:3000/api/view?filename=clip.mp4&type=output&subfolder=' }), MIME_ASSET_INFO ) diff --git a/src/locales/en/main.json b/src/locales/en/main.json index a69ed605559..2f45ff43e2b 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5191,6 +5191,7 @@ } }, "agent": { + "previewAsset": "Preview {name}", "title": "Comfy Agent", "loadFailed": "The agent panel failed to load.", "entryButton": "Agent", @@ -5278,6 +5279,7 @@ "attachmentBatchLimit": "{count} file was not added because the upload queue limit is {limit}. | {count} files were not added because the upload queue limit is {limit}.", "attachmentsTooLarge": "{count} file exceeds the upload size limit. | {count} files exceed the upload size limit.", "attachmentUploadFailed": "{name} could not be uploaded", + "attachmentsAlreadyAdded": "{name} is already in the asset tray | Skipped {count} duplicate assets", "mention": "Mention a node", "changeWorkflowForChat": "Change the workflow that the agent can edit", "chatOptions": "Chat options", @@ -5324,7 +5326,11 @@ "focusNode": "Show on canvas", "focusNodeLabel": "Show {node} on canvas", "removeNodeLabel": "Remove {node} reference", + "removeAssetReference": "Remove {name} reference", + "removeAsset": "Remove {name}", + "previewAsset": "Preview {name}", "uploading": "Uploading", + "uploadingAttachments": "Uploading {count} attachment | Uploading {count} attachments", "noNodesToMention": "No nodes in this workflow", "friend": "there", "sendFailed": "Message failed to send", diff --git a/src/platform/assets/schemas/mediaAssetSchema.ts b/src/platform/assets/schemas/mediaAssetSchema.ts index 843147c781d..889ca2902dc 100644 --- a/src/platform/assets/schemas/mediaAssetSchema.ts +++ b/src/platform/assets/schemas/mediaAssetSchema.ts @@ -5,7 +5,7 @@ import { zResultItem } from '@/schemas/apiSchema' import { assetItemSchema } from './assetSchema' -const zMediaKindSchema = z.enum([ +export const zMediaKindSchema = z.enum([ 'video', 'audio', 'image', @@ -58,7 +58,8 @@ const zDraggedAssetInfo = zResultItem.and( z.object({ attachment_ref: z.string().min(1).optional(), media_kind: zMediaKindSchema.optional(), - preview_url: z.string().url().optional() + preview_url: z.string().url().optional(), + media_url: z.string().url().optional() }) ) diff --git a/src/platform/assets/utils/assetDragUtil.test.ts b/src/platform/assets/utils/assetDragUtil.test.ts new file mode 100644 index 00000000000..e445b34908e --- /dev/null +++ b/src/platform/assets/utils/assetDragUtil.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { api } from '@/scripts/api' +import { getDroppedAsset } from '@/utils/eventUtils' + +import { startAssetDrag } from './assetDragUtil' + +const mockFlags = vi.hoisted(() => ({ assetsEnabled: false })) + +vi.mock(import('@/composables/useFeatureFlags'), () => ({ + useFeatureFlags: () => ({ flags: mockFlags }) +})) +vi.mock(import('@/scripts/api')) + +function drag(asset: AssetItem) { + const dataTransfer = new DataTransfer() + const event = new DragEvent('dragstart', { cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return getDroppedAsset(dataTransfer) +} + +describe('asset drag media sources', () => { + it.for([ + { + label: 'server poster', + assetsEnabled: true, + preview_id: 'poster-id', + thumbnail_url: undefined, + preview_url: undefined, + poster: 'http://localhost:8188/api/assets/poster-id/content', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'distinct thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '/view?filename=poster.png', + preview_url: '/view?filename=clip.mp4', + poster: 'http://localhost:8188/api/view?filename=poster.png', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'absolute CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: 'https://cdn.example/thumb.webp?signature=poster', + preview_url: '/view?filename=clip.mp4', + poster: 'https://cdn.example/thumb.webp?signature=poster', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'protocol-relative CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '//cdn.example/thumb.webp', + preview_url: '/view?filename=clip.mp4', + poster: 'http://cdn.example/thumb.webp', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'history video without a poster', + assetsEnabled: false, + preview_id: undefined, + thumbnail_url: '/view?filename=clip.mp4', + preview_url: '/view?filename=clip.mp4', + poster: undefined, + media: 'http://localhost:3000/view?filename=clip.mp4' + }, + { + label: 'assets API video without a poster', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: undefined, + preview_url: undefined, + poster: undefined, + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + } + ])( + 'preserves distinct file/image sources for $label', + ({ + assetsEnabled, + preview_id, + thumbnail_url, + preview_url, + poster, + media + }) => { + mockFlags.assetsEnabled = assetsEnabled + vi.mocked(api.apiURL).mockImplementation( + (path) => `http://localhost:8188/api${path}` + ) + const asset = { + id: 'video-id', + name: 'clip.mp4', + display_name: 'My clip', + tags: ['input'], + created_at: '2026-10-05T00:00:00Z', + updated_at: '2026-10-05T00:00:00Z', + preview_id, + thumbnail_url, + preview_url + } satisfies AssetItem + expect(drag(asset)).toMatchObject({ + name: 'My clip', + kind: 'video', + mediaUrl: media, + previewUrl: poster + }) + } + ) +}) diff --git a/src/platform/assets/utils/assetDragUtil.ts b/src/platform/assets/utils/assetDragUtil.ts index 9487c359402..4a4623dfcfc 100644 --- a/src/platform/assets/utils/assetDragUtil.ts +++ b/src/platform/assets/utils/assetDragUtil.ts @@ -9,6 +9,7 @@ * row that starts a drag carrying nothing. */ import { getMediaTypeFromFilename } from '@/utils/formatUtil' +import { api } from '@/scripts/api' import { getAssetType } from '../composables/media/assetMappers' import { getOutputAssetMetadata } from '../schemas/assetMetadataSchema' @@ -18,6 +19,39 @@ import { getAssetUrlFilename } from './assetMetadataUtils' import { resolvePreviewUrl } from './assetPreviewUtil' import { getAssetFileUrl } from './assetUrlUtil' +function resolveThumbnailUrl(url: string): URL | null { + const source = + URL.canParse(url) || url.startsWith('//') ? url : api.apiURL(url) + return URL.parse(source, location.href) +} + +function assetMediaSources(asset: AssetItem) { + const mediaKind = getMediaTypeFromFilename(asset.name) + const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) + const playable = mediaKind === 'video' || mediaKind === 'audio' + const mediaUrl = playable + ? URL.parse( + getAssetFileUrl(asset, { disposition: 'inline' }), + location.href + ) + : undefined + const posterUrl = asset.preview_id + ? previewUrl + : asset.thumbnail_url && asset.thumbnail_url !== asset.preview_url + ? resolveThumbnailUrl(asset.thumbnail_url) + : undefined + return { + media_kind: mediaKind, + preview_url: + mediaKind === 'image' + ? previewUrl?.toString() + : mediaKind === 'video' + ? posterUrl?.toString() + : undefined, + media_url: mediaUrl?.toString() + } +} + /** * Start a native drag carrying `asset`. * @@ -43,9 +77,6 @@ export function startAssetDrag( if (!dataTransfer) return const output = getOutputAssetMetadata(asset.user_metadata)?.allOutputs?.[0] - const mediaKind = getMediaTypeFromFilename(asset.name) - const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) - const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) const assetInfo = { ...(output?.filename ? { @@ -57,14 +88,14 @@ export function startAssetDrag( : { filename: asset.name, type: getAssetType(asset.tags), - display_name: asset.display_name + display_name: asset.display_name ?? undefined }), attachment_ref: getAssetUrlFilename(asset), - media_kind: mediaKind, - preview_url: mediaKind === 'image' ? previewUrl?.toString() : undefined + ...assetMediaSources(asset) } dataTransfer.setData(MIME_ASSET_INFO, JSON.stringify(assetInfo)) + const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) if (!fileUrl) return dataTransfer.setData('text/uri-list', fileUrl.toString()) diff --git a/src/platform/keybindings/keybindingService.escape.test.ts b/src/platform/keybindings/keybindingService.escape.test.ts index 589bf490a6e..c90c3fad0cb 100644 --- a/src/platform/keybindings/keybindingService.escape.test.ts +++ b/src/platform/keybindings/keybindingService.escape.test.ts @@ -134,19 +134,21 @@ describe('keybindingService - Escape key handling', () => { expect(mockCommandExecute).not.toHaveBeenCalled() }) - it('should leave Escape events from menus to the menu', async () => { - const menu = document.createElement('div') - menu.setAttribute('role', 'menu') - const menuItem = document.createElement('div') - menuItem.setAttribute('role', 'menuitemcheckbox') - menu.appendChild(menuItem) - - const event = createKeyboardEvent('Escape', { target: menuItem }) - await keybindingService.keybindHandler(event) + it.for(['menu', 'dialog'])( + 'should leave Escape events from role=%s to the overlay', + async (role) => { + const overlay = document.createElement('div') + overlay.setAttribute('role', role) + const target = document.createElement('div') + overlay.appendChild(target) + + const event = createKeyboardEvent('Escape', { target }) + await keybindingService.keybindHandler(event) - expect(event.preventDefault).not.toHaveBeenCalled() - expect(mockCommandExecute).not.toHaveBeenCalled() - }) + expect(event.preventDefault).not.toHaveBeenCalled() + expect(mockCommandExecute).not.toHaveBeenCalled() + } + ) it('does not throw when Escape fires with a non-Element target (e.g. document, in Safari when nothing has focus)', async () => { const event = createKeyboardEvent('Escape', { diff --git a/src/platform/keybindings/keybindingService.ts b/src/platform/keybindings/keybindingService.ts index a33d0574f22..09029c165ae 100644 --- a/src/platform/keybindings/keybindingService.ts +++ b/src/platform/keybindings/keybindingService.ts @@ -27,7 +27,7 @@ export function useKeybindingService() { if ( event.key === 'Escape' && target instanceof Element && - target.closest('[role="menu"]') + target.closest('[role="menu"], [role="dialog"]') ) { return } diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts index 41cebaa3ed6..43ecee6055b 100644 --- a/src/utils/eventUtils.ts +++ b/src/utils/eventUtils.ts @@ -8,6 +8,7 @@ export interface DroppedAsset { ref?: string kind?: MediaKind previewUrl?: string + mediaUrl?: string } class DroppedAssetFetchError extends Error { @@ -45,7 +46,8 @@ export function getDroppedAsset( uri, ref, kind: asset?.media_kind, - previewUrl: asset?.preview_url + previewUrl: asset?.preview_url, + mediaUrl: asset?.media_url } : undefined } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index 9e6482ecbc7..23b4847b630 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -38,12 +38,9 @@ import { useFreeUsePlacement } from './experiments/freeUsePlacement' setupInlinePromptEditorDom() import type { ComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' - -import type { - LGraph, - LGraphNode, - Subgraph -} from '@/lib/litegraph/src/litegraph' +import type { Subgraph } from '@/lib/litegraph/src/litegraph' +import { LGraph, LGraphCanvas, LGraphNode } from '@/lib/litegraph/src/litegraph' +import { createTestSubgraph } from '@/lib/litegraph/src/subgraph/__fixtures__/subgraphHelpers' import { toRootGraphId } from '@/types/graphScopeId' import { toNodeId } from '@/types/nodeId' @@ -58,7 +55,12 @@ import { useWorkflowTabActivityStore } from '@/stores/workflowTabActivityStore' import { useSidebarTabStore } from '@/stores/workspace/sidebarTabStore' import { useToastStore } from '@/platform/updates/common/toastStore' import { useAssetsStore } from '@/stores/assetsStore' -import { createTestDragAndScale } from '@/utils/__tests__/canvasTestUtils' +import { + createMockCanvasRenderingContext2D, + createTestDragAndScale +} from '@/utils/__tests__/canvasTestUtils' +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { startAssetDrag } from '@/platform/assets/utils/assetDragUtil' import { getFilenameDetails } from '@/utils/formatUtil' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' import { useWorkflowService } from '@/platform/workflow/core/services/workflowService' @@ -1155,6 +1157,39 @@ function fileOfSize(name: string, size: number, type: string): File { return file } +function assetPanelDrag( + displayName: string | null, + overrides: Partial = {} +) { + const asset = { + id: 'library-source', + name: 'library.png', + hash: 'stored-library.png', + display_name: displayName, + tags: ['input'], + created_at: '2026-10-03T00:00:00Z', + updated_at: '2026-10-03T00:00:00Z', + ...overrides + } satisfies AssetItem + const values = new Map() + const dataTransfer = { + get types() { + return [...values.keys()] + }, + getData: (type: string) => values.get(type) ?? '', + setData: (type: string, value: string) => { + values.set(type, value) + } + } as unknown as DataTransfer + const event = new Event('dragstart', { cancelable: true }) as DragEvent + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return { + types: Array.from(dataTransfer.types), + getData: (type: string) => dataTransfer.getData(type) + } +} + // DES-527 replaced the paperclip and @ buttons with a single + menu, so every // attach or node-mention gesture now starts by opening it. async function openAddMenu(): Promise { @@ -1204,6 +1239,7 @@ function setupNodeSelectionCanvas() { selectItems, deselect, deselectAll, + setDirty: vi.fn(), animateToBounds: vi.fn(), multi_select: false, allow_dragnodes: true, @@ -1229,6 +1265,46 @@ function setupNodeSelectionCanvas() { const SUBGRAPH_UUID = '12345678-1234-1234-1234-123456789abc' +function syncFakeSelection(): void { + canvasStore.updateSelectedItems() +} + +function setupOwnedSelectionCanvas() { + const rootGraph = new LGraph() + const subgraph = createTestSubgraph({ rootGraph, id: SUBGRAPH_UUID }) + rootGraph.subgraphs.set(subgraph.id, subgraph) + const rootNode = new LGraphNode('Root node') + rootNode.id = toNodeId(3) + rootGraph.add(rootNode) + const subgraphNode = new LGraphNode('KSampler') + subgraphNode.id = toNodeId(12) + subgraph.add(subgraphNode) + + const canvasElement = document.createElement('canvas') + canvasElement.getContext = vi + .fn() + .mockReturnValue(createMockCanvasRenderingContext2D()) + canvasElement.getBoundingClientRect = vi + .fn() + .mockReturnValue({ left: 0, top: 0, width: 800, height: 600 }) + const canvas = new LGraphCanvas(canvasElement, rootGraph, { + skip_render: true + }) + canvas.onSelectionChange = () => syncFakeSelection() + + appMock.graph.nodes = rootGraph.nodes + Object.assign(appMock.rootGraph, { subgraphs: rootGraph.subgraphs }) + appMock.canvas = canvas as unknown as typeof appMock.canvas + canvasStore.canvas = canvas + viewGraph(canvas, subgraph) + return { canvas, rootGraph, subgraph, rootNode, subgraphNode } +} + +function viewGraph(canvas: LGraphCanvas, graph: LGraph): void { + canvas.setGraph(graph) + canvasStore.currentGraph = graph +} + function nestSelectionCanvasInSubgraph( state: ReturnType ) { @@ -1359,6 +1435,228 @@ describe('AgentPanelRoot attach flow', () => { useAgentWorkflowTabBindingStore().bind('wf-42', tab.path) }) + it.for([ + { displayName: null, label: 'library.png' }, + { displayName: 'My library image', label: 'library.png' } + ])( + 'stages an assets-panel drop with display name $displayName from the real producer payload', + async ({ displayName, label }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep this draft') + await nextTick() + const target = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) + const data = assetPanelDrag(displayName) + + expect(dispatchDrag(target, 'dragover', data)).toBe(true) + expect(dispatchDrag(target, 'drop', data)).toBe(true) + expect( + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: label } + ) + ).toBeInTheDocument() + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: label, + ref: 'stored-library.png', + previewUrl: 'http://localhost:3000/api/assets/library-source/content' + }) + ]) + expect(store.prompt).toEqual({ text: 'Keep this draft', references: [] }) + expect(uploaded).toEqual([]) + } + ) + + it.for([ + { + filename: 'clip.mp4', + label: 'My clip', + expectedName: 'clip.mp4', + kind: 'video', + previewId: 'poster', + previewUrl: 'http://localhost:3000/api/assets/poster/content', + indicatorLabel: 'clip.mp4', + indicatorSource: 'http://localhost:3000/api/assets/poster/content' + }, + { + filename: 'song.mp3', + label: 'My recording', + expectedName: 'song.mp3', + kind: 'audio', + previewId: undefined, + previewUrl: undefined, + indicatorLabel: 'Audio', + indicatorSource: null + } + ])( + 'preserves playable $kind metadata from the assets panel through the chat tray', + async ({ + filename, + label, + expectedName, + kind, + previewId, + previewUrl, + indicatorLabel, + indicatorSource + }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep typing') + await nextTick() + const data = assetPanelDrag(label, { + name: filename, + hash: `stored-${filename}`, + preview_id: previewId + }) + expect(dispatchDrag(screen.getByRole('textbox'), 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: expectedName }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: expectedName, + mediaKind: kind, + previewUrl, + mediaUrl: + 'http://localhost:3000/api/assets/library-source/content?disposition=inline' + }) + ]) + expect( + within(trayItem) + .getByRole('img', { name: indicatorLabel }) + .getAttribute('src') + ).toBe(indicatorSource) + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(uploaded).toEqual([]) + } + ) + + it('deduplicates an assets-panel item that needs fetching to identify its media type', async () => { + const assetData = assetPanelDrag('Recording.mp3', { name: 'recording' }) + const source = 'http://localhost:3000/api/view?filename=recording' + const data = { + ...assetData, + getData: (type: string) => + type === 'text/uri-list' ? source : assetData.getData(type) + } + expect( + JSON.parse(data.getData('application/x-comfy-asset-info')) + ).toMatchObject({ media_kind: 'other' }) + const fetched: string[] = [] + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => { + if (String(input) === source) { + fetched.push(source) + return new Response(new Blob(['audio'], { type: 'audio/mpeg' })) + } + if (String(input).includes('/upload/')) + return json(200, { + name: 'recording.mp3', + subfolder: '', + type: 'input' + }) + if (String(input).includes('/assets')) + return json(200, { assets: [], total: 0, has_more: false }) + return json(200, agentThreadList()) + }) + ) + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + dispatchDrag(target, 'drop', data) + await vi.waitFor(() => expect(store.attachments[0]?.uploading).toBe(false)) + dispatchDrag(target, 'drop', data) + await nextTick() + expect(store.attachments).toHaveLength(1) + expect(fetched).toEqual([source]) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ + severity: 'info', + detail: 'recording is already in the asset tray' + }) + ]) + }) + + it('allows a removed library asset to be explicitly dropped again while retaining retirement of its old identity', async () => { + stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + const data = assetPanelDrag('library.png') + expect(dispatchDrag(target, 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: 'library.png' }) + const oldAttachment = store.attachments[0] + assert.exists(oldAttachment) + store.referenceAttachment(oldAttachment.id) + const oldPrompt = store.prompt + await userEvent.hover(trayItem) + await userEvent.click( + within(trayItem).getByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'library.png' }) + }) + ) + expect(store.attachments).toEqual([]) + + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + const newAttachment = store.attachments[0] + assert.exists(newAttachment) + expect(newAttachment.id).not.toBe(oldAttachment.id) + store.updateAttachment(oldAttachment.id, { ref: 'too-late.png' }) + store.applyEditorPrompt(oldPrompt) + expect(store.attachments).toEqual([newAttachment]) + expect(newAttachment.ref).toBe('stored-library.png') + expect(store.prompt.references).toEqual([]) + expect(useToastStore().messagesToAdd).toEqual([]) + expect(dispatchDrag(target, 'drop', data)).toBe(true) + expect(store.attachments).toHaveLength(1) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ + severity: 'info', + detail: 'library.png is already in the asset tray' + }) + ]) + }) + + it('allows the same library asset to be dropped into a new draft after sending', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => + init?.method === 'POST' + ? json(202, { thread_id: 'th-1', message_id: 'm-1' }) + : json(200, agentThreadList()) + ) + ) + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + const target = screen.getByRole('textbox') + const data = assetPanelDrag('library.png') + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + const previousId = store.attachments[0]?.id + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(store.attachments).toEqual([]) + + expect(dispatchDrag(target, 'drop', data)).toBe(true) + await screen.findByRole('group', { name: 'library.png' }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: 'library.png', + ref: 'stored-library.png' + }) + ]) + expect(store.attachments[0]?.id).not.toBe(previousId) + expect(store.prompt.references).toEqual([]) + }) + it('uploads a picked file, stages its ref, and forwards it on the next send', async () => { const messageBodies: unknown[] = [] const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { @@ -1392,8 +1690,9 @@ describe('AgentPanelRoot attach flow', () => { await userEvent.upload(input, file) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() @@ -1402,7 +1701,7 @@ describe('AgentPanelRoot attach flow', () => { expect(messageBodies).toHaveLength(1) expect(messageBodies[0]).toMatchObject({ - content: 'make it pop@[Image: cat.png]', + content: 'make it pop', attachments: ['uploaded_cat.png'] }) expect(telemetry.trackAgentMessageSent).toHaveBeenCalledWith({ @@ -1538,8 +1837,9 @@ describe('AgentPanelRoot attach flow', () => { ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'movie.mp4' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'movie.mp4' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['movie.mp4'])) @@ -1620,7 +1920,10 @@ describe('AgentPanelRoot attach flow', () => { detail: 'movie.mp4 is larger than 24 MB' }) ) - expect(screen.queryByText('movie.mp4')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'movie.mp4' }) + ).not.toBeInTheDocument() + expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() }) it('uses a larger server limit for non-video attachments', async () => { @@ -1633,8 +1936,9 @@ describe('AgentPanelRoot attach flow', () => { dispatchDrag(screen.getByRole('textbox'), 'drop', { files: [image] }) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'huge.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'huge.png' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['huge.png'])) @@ -1674,8 +1978,9 @@ describe('AgentPanelRoot attach flow', () => { ).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'movie.mp4' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'movie.mp4' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['movie.mp4'])) @@ -1705,8 +2010,9 @@ describe('AgentPanelRoot attach flow', () => { ).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - name + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: name } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual([name])) @@ -1847,7 +2153,7 @@ describe('AgentPanelRoot attach flow', () => { await vi.waitFor(() => expect(refresh).toHaveBeenCalledTimes(2)) }) - it('gives Undo a grace period before cancelling a removed upload', async () => { + it('cancels a removed tray upload after its grace period', async () => { const signals: AbortSignal[] = [] vi.stubGlobal( 'fetch', @@ -1874,7 +2180,7 @@ describe('AgentPanelRoot attach flow', () => { await vi.waitFor(() => expect(signals).toHaveLength(1)) const composer = useAgentComposerStore() const removeButton = await screen.findByRole('button', { - name: i18n.global.t('agent.remove') + name: i18n.global.t('agent.removeAsset', { name: 'cat.png' }) }) vi.useFakeTimers() try { @@ -1894,6 +2200,50 @@ describe('AgentPanelRoot attach flow', () => { } }) + it('keeps a removed tray upload retired after completion and stale editor history', async () => { + const signals: AbortSignal[] = [] + let finishUpload: (response: Response) => void = () => {} + const upload = new Promise((resolve) => { + finishUpload = resolve + }) + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + if (!String(input).includes('/upload/')) + return json(200, agentThreadList()) + if (init?.signal) signals.push(init.signal) + return upload + }) + ) + renderWithSelectedTarget() + await nextTick() + const refresh = vi + .spyOn(useAssetsStore().inputAssets, 'loadNew') + .mockResolvedValue(undefined) + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + files: [new File(['x'], 'cat.png', { type: 'image/png' })] + }) + await vi.waitFor(() => expect(signals).toHaveLength(1)) + const composer = useAgentComposerStore() + const prompt = composer.prompt + + await userEvent.click( + await screen.findByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'cat.png' }) + }) + ) + + expect(signals[0].aborted).toBe(false) + finishUpload( + json(200, { name: 'uploaded-cat.png', subfolder: '', type: 'input' }) + ) + await vi.waitFor(() => expect(refresh).toHaveBeenCalledOnce()) + expect(composer.attachments).toEqual([]) + composer.applyEditorPrompt(prompt) + expect(composer.attachments).toEqual([]) + }) + it('uses the server limit for audio rejection copy', async () => { getServerFeature.mockReturnValue(24 * 1024 * 1024) const uploaded = stubUploadFetch() @@ -1927,26 +2277,58 @@ describe('AgentPanelRoot attach flow', () => { ) }) - it('shows the asset drop target during a trusted drag and clears it on leave', async () => { - stubUploadFetch() - renderWithSelectedTarget() - await nextTick() - const target = screen.getByRole('textbox') - const data = { - types: ['application/x-comfy-asset-info', 'text/uri-list'] + it.for([ + { tray: 'empty', attachments: [] }, + { + tray: 'with an asset', + attachments: [{ id: 'kept', name: 'kept.png', ref: 'kept.png' }] } + ])( + 'shows panel-wide drop feedback across child transitions while preserving the composer (tray=$tray)', + async ({ attachments }) => { + stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.replaceDraft({ + text: 'Keep this draft', + workflowReferences: [], + attachments + }) + await nextTick() + const prompt = store.prompt + const header = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) + const composer = screen.getByTestId('composer-input-box') + const data = { + types: ['application/x-comfy-asset-info', 'text/uri-list'] + } + expect(screen.queryByRole('status')).not.toBeInTheDocument() - dispatchDrag(target, 'dragenter', data) - await nextTick() + dispatchDrag(header, 'dragenter', data) + await nextTick() - const dropTarget = screen.getByRole('status') - expect(dropTarget).toHaveTextContent('Drag and drop assets here') + const dropTarget = screen.getByRole('status') + expect(dropTarget).toHaveTextContent('Drag and drop assets here') + expect(composer).not.toContainElement(dropTarget) + expect(screen.getByRole('textbox')).toHaveTextContent('Keep this draft') + expect(store.prompt).toEqual(prompt) + expect(screen.queryAllByRole('group', { name: 'kept.png' })).toHaveLength( + attachments.length + ) - dispatchDrag(dropTarget, 'dragleave', data) - await nextTick() + dispatchDrag(composer, 'dragenter', data) + dispatchDrag(header, 'dragleave', data) + await nextTick() + expect(screen.getByRole('status')).toBe(dropTarget) - expect(screen.queryByRole('status')).not.toBeInTheDocument() - }) + dispatchDrag(composer, 'dragleave', data) + await nextTick() + expect(screen.queryByRole('status')).not.toBeInTheDocument() + expect(store.prompt).toEqual(prompt) + expect(store.attachments).toEqual(attachments) + } + ) it('rejects URI-only drags without showing or claiming the asset target', async () => { const uploaded = stubUploadFetch() @@ -2026,8 +2408,9 @@ describe('AgentPanelRoot attach flow', () => { expect(dropTarget).not.toBeInTheDocument() expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - filename + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: filename } ) ).toBeInTheDocument() @@ -2036,7 +2419,7 @@ describe('AgentPanelRoot attach flow', () => { expect(messageBodies).toHaveLength(1) expect(messageBodies[0]).toMatchObject({ - content: `describe this@[${mime === 'image/png' ? 'Image' : 'Video'}: ${filename}]`, + content: 'describe this', attachments: [`uploaded_${filename}`] }) } @@ -2085,13 +2468,15 @@ describe('AgentPanelRoot attach flow', () => { expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - filename + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: filename } ) ).toBeInTheDocument() expect( - within(screen.getByTestId('composer-asset-section')).getAllByText( - filename + within(screen.getByTestId('composer-asset-section')).getAllByRole( + 'group', + { name: filename } ) ).toHaveLength(1) expect( @@ -2133,8 +2518,9 @@ describe('AgentPanelRoot attach flow', () => { ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'model.obj' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'model.obj' } ) ).toBeInTheDocument() expect( @@ -2182,8 +2568,9 @@ describe('AgentPanelRoot attach flow', () => { expect(dispatchDrag(target, 'drop', dragData)).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'gen.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'gen.png' } ) ).toBeInTheDocument() expect( @@ -2247,23 +2634,77 @@ describe('AgentPanelRoot attach flow', () => { stubUploadFetch() renderWithSelectedTarget() await nextTick() - const target = screen.getByRole('textbox') + const target = screen.getByRole('button', { + name: i18n.global.t('agent.close') + }) const workflow = new File(['{}'], 'flow.json', { type: 'application/json' }) expect(dispatchDrag(target, 'drop', { files: [workflow] })).toBe(false) - expect(screen.queryByText('flow.json')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'flow.json' }) + ).not.toBeInTheDocument() const asset = new File(['x'], 'cat.png', { type: 'image/png' }) expect(dispatchDrag(target, 'drop', { files: [asset] })).toBe(true) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() }) + it.for([ + { copies: 1, notice: 'cat.png is already in the asset tray' }, + { copies: 2, notice: 'Skipped 2 duplicate assets' } + ])('reports $copies skipped files once', async ({ copies, notice }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + store.setText('Keep this draft') + const target = screen.getByRole('textbox') + await userEvent.click(target) + dispatchDrag(target, 'drop', { + files: [ + new File(['image'], 'cat.png', { type: 'image/png', lastModified: 1 }) + ] + }) + await vi.waitFor(() => expect(store.attachments[0]?.uploading).toBe(false)) + expect(useToastStore().messagesToAdd).toEqual([]) + dispatchDrag(target, 'drop', { + files: [ + ...Array.from( + { length: copies }, + () => + new File(['image'], 'cat.png', { + type: 'image/png', + lastModified: 1 + }) + ), + new File(['another image'], 'dog.png', { type: 'image/png' }) + ] + }) + await nextTick() + expect( + within(screen.getByTestId('composer-asset-section')).getAllByRole( + 'group', + { + name: 'cat.png' + } + ) + ).toHaveLength(1) + await vi.waitFor(() => expect(uploaded).toEqual(['cat.png', 'dog.png'])) + expect(useToastStore().messagesToAdd).toEqual([ + expect.objectContaining({ severity: 'info', detail: notice }) + ]) + expect(screen.getByRole('group', { name: 'dog.png' })).toBeInTheDocument() + expect(target).toHaveFocus() + expect(store.prompt).toEqual({ text: 'Keep this draft', references: [] }) + }) + it('attaches only the assets out of a mixed drop', async () => { const uploaded = stubUploadFetch() renderWithSelectedTarget() @@ -2280,13 +2721,142 @@ describe('AgentPanelRoot attach flow', () => { dispatchDrag(screen.getByRole('textbox'), 'drop', { files }) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['cat.png'])) }) + // Falsifiers for the committed-drop gates: an implementation that emits + // before the deferred fetch or upload settles, or ignores their results, + // fails these. + it.for(['failed', 'cancelled'] as const)( + 'emits no drop telemetry for a %s deferred asset', + async (outcome) => { + let resolveAsset: (response: Response) => void = () => {} + let rejectAsset: (error: Error) => void = () => {} + vi.stubGlobal( + 'fetch', + vi.fn((input: RequestInfo | URL) => { + const url = String(input) + if (url.includes('/api/view')) + return new Promise((resolve, reject) => { + resolveAsset = resolve + rejectAsset = reject + }) + if (url.endsWith('/api/upload/image')) + return Promise.resolve( + json(200, { + name: 'uploaded_gen.png', + subfolder: '', + type: 'input' + }) + ) + return Promise.resolve(json(200, agentThreadList())) + }) + ) + renderWithSelectedTarget() + await nextTick() + telemetry.trackAgentAttachButtonClicked.mockClear() + + const dragData = { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ filename: 'gen.png', type: 'input' }) + : 'http://localhost/api/view?filename=gen.png' + } + expect(dispatchDrag(screen.getByRole('textbox'), 'drop', dragData)).toBe( + true + ) + expect( + await screen.findByLabelText(i18n.global.t('agent.uploading')) + ).toBeInTheDocument() + + if (outcome === 'cancelled') { + await userEvent.click( + screen.getByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'gen.png' }) + }) + ) + resolveAsset(new Response(new Blob(['asset'], { type: 'image/png' }))) + } else { + rejectAsset(new Error('asset fetch failed')) + } + await vi.waitFor(() => + expect( + screen.queryByLabelText(i18n.global.t('agent.uploading')) + ).not.toBeInTheDocument() + ) + await nextTick() + + expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() + } + ) + + it.for([ + { files: ['bad.png'], expectedEvents: 0 }, + { files: ['bad.png', 'good.png'], expectedEvents: 1 } + ])( + 'emits $expectedEvents drop event(s) when uploads of $files are rejected except any good one', + async ({ files, expectedEvents }) => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (!url.includes('/upload/')) return json(200, agentThreadList()) + const body = init?.body + const file = body instanceof FormData ? body.get('image') : null + const name = file instanceof File ? file.name : '' + if (name.startsWith('bad')) return json(500, {}) + return json(200, { + name: `uploaded_${name}`, + subfolder: '', + type: 'input' + }) + }) + ) + renderWithSelectedTarget() + await nextTick() + telemetry.trackAgentAttachButtonClicked.mockClear() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + files: files.map((name) => new File(['x'], name, { type: 'image/png' })) + }) + + // The rejected upload's chip leaves; only a committed one stays settled. + await vi.waitFor(() => + expect( + screen.queryByRole('group', { name: 'bad.png' }) + ).not.toBeInTheDocument() + ) + await vi.waitFor(() => + expect( + screen.queryByLabelText(i18n.global.t('agent.uploading')) + ).not.toBeInTheDocument() + ) + await nextTick() + + if (expectedEvents === 0) { + expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() + } else { + expect( + within(screen.getByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'good.png' } + ) + ).toBeInTheDocument() + await vi.waitFor(() => + expect( + telemetry.trackAgentAttachButtonClicked + ).toHaveBeenCalledExactlyOnceWith() + ) + } + } + ) + it('shows an uploading chip and blocks send until the upload settles', async () => { let settleUpload: () => void = () => {} const fetchMock = vi.fn(async (url: string) => { @@ -2316,8 +2886,9 @@ describe('AgentPanelRoot attach flow', () => { ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() expect( @@ -2367,8 +2938,9 @@ describe('AgentPanelRoot attach flow', () => { screen.getByTestId('agent-file-input'), file ) - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) await sendFromComposer('second message') @@ -2412,15 +2984,18 @@ describe('AgentPanelRoot attach flow', () => { file ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() executionErrors.showErrorOverlay.mockClear() failUpload() await vi.waitFor(() => - expect(screen.queryByText('cat.png')).not.toBeInTheDocument() + expect( + screen.queryByRole('group', { name: 'cat.png' }) + ).not.toBeInTheDocument() ) expect(revoke).toHaveBeenCalledTimes(1) // A rejected file is the user's to fix; raising the server-error overlay @@ -2455,7 +3030,6 @@ describe('AgentPanelRoot attach flow', () => { }) it('releases a dismissed in-flight preview when its upload is cancelled', async () => { - vi.useFakeTimers() const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) vi.stubGlobal( 'fetch', @@ -2477,23 +3051,93 @@ describe('AgentPanelRoot attach flow', () => { file ) expect( - within(await screen.findByTestId('composer-asset-section')).getByText( - 'cat.png' + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'cat.png' } ) ).toBeInTheDocument() - const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + const user = userEvent.setup() await user.click( - screen.getByRole('button', { name: i18n.global.t('agent.remove') }) + screen.getByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'cat.png' }) + }) ) - expect(screen.queryByText('cat.png')).not.toBeInTheDocument() - expect(revoke).not.toHaveBeenCalled() - await vi.advanceTimersByTimeAsync(5_000) + expect( + screen.queryByRole('group', { name: 'cat.png' }) + ).not.toBeInTheDocument() expect(revoke).toHaveBeenCalledTimes(1) view.unmount() expect(revoke).toHaveBeenCalledTimes(1) revoke.mockRestore() - vi.useRealTimers() + }) + + it('keeps a durable tray preview through inline deletion, Undo and unmount', async () => { + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + if (url.endsWith('/api/upload/image')) { + return new Response( + JSON.stringify({ + name: 'uploaded_cat.png', + subfolder: '', + type: 'input' + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ) + } + return new Response('{"threads":[]}', { + status: 200, + headers: { 'Content-Type': 'application/json' } + }) + }) + ) + + const view = renderWithSelectedTarget() + + const file = new File(['x'], 'cat.png', { type: 'image/png' }) + await userEvent.upload( + screen.getByTestId('agent-file-input'), + file + ) + const assetSection = await screen.findByTestId('composer-asset-section') + expect( + within(assetSection).getByRole('group', { name: 'cat.png' }) + ).toBeInTheDocument() + await waitFor(() => + expect( + within(assetSection).getByRole('img', { name: 'cat.png' }) + ).toHaveAttribute('src', '/api/view?filename=uploaded_cat.png&type=input') + ) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:mock-url') + + const composer = useAgentComposerStore() + composer.referenceAttachment(composer.attachments[0].id) + await screen.findByTestId('asset-reference-chip') + await userEvent.click(screen.getByRole('textbox')) + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: 'cat.png' + }) + ).toBeVisible() + screen.getByRole('textbox').focus() + await userEvent.keyboard('{Control>}z{/Control}') + expect( + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'img', + { name: 'cat.png' } + ) + ).toHaveAttribute('src', '/api/view?filename=uploaded_cat.png&type=input') + expect(screen.getByTestId('asset-reference-chip')).toHaveTextContent( + 'cat.png' + ) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:mock-url') + view.unmount() + expect(revoke).toHaveBeenCalledTimes(1) + revoke.mockRestore() }) }) @@ -7046,21 +7690,105 @@ describe('AgentPanelRoot workflow binding', () => { it.for([ ...[ - 'untouched', - 'new-draft', - 'cleared-draft', - 'removed-reference', - 'removed-attachment', - 'new-chat', - 'history' - ].flatMap((nextAction) => - ['mounted', 'reopened'].map((panel) => ({ nextAction, panel })) + { + nextAction: 'untouched', + expectedDraft: '', + act: () => Promise.resolve() + }, + { + nextAction: 'new-draft', + expectedDraft: 'New input', + act: async (textbox: HTMLElement) => { + await userEvent.click(textbox) + await userEvent.paste('New input') + } + }, + { + nextAction: 'cleared-draft', + expectedDraft: '', + act: async (textbox: HTMLElement) => { + await userEvent.click(textbox) + await userEvent.paste('New input') + await userEvent.clear(textbox) + } + }, + { + nextAction: 'removed-reference', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { + name: i18n.global.t('agent.addToPrompt') + }) + ) + await userEvent.click( + screen.getByRole('menuitem', { name: 'Workflows' }) + ) + await userEvent.click( + await screen.findByRole('menuitem', { name: 'reference' }) + ) + await userEvent.click( + screen.getByRole('button', { name: 'Remove reference reference' }) + ) + } + }, + { + nextAction: 'removed-attachment', + expectedDraft: '', + act: async () => { + useAgentComposerStore().addAttachment({ + id: 'upload-2', + name: 'new.png', + ref: 'new.png' + }) + await userEvent.click( + await screen.findByRole('button', { + name: i18n.global.t('agent.removeAsset', { name: 'new.png' }) + }) + ) + } + }, + { + nextAction: 'new-chat', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { name: i18n.global.t('agent.newChat') }) + ) + } + }, + { + nextAction: 'history', + expectedDraft: '', + act: async () => { + await userEvent.click( + screen.getByRole('button', { + name: i18n.global.t('agent.showChatHistory') + }) + ) + await userEvent.click(await screen.findByText('Earlier chat')) + } + } + ].flatMap((scenario) => + ['mounted', 'reopened'].map((panel) => ({ ...scenario, panel })) ), - { nextAction: 'untouched', panel: 'closed' }, - { nextAction: 'stop', panel: 'reopened' } + { + nextAction: 'untouched', + panel: 'closed', + expectedDraft: '', + act: () => Promise.resolve() + }, + { + nextAction: 'stop', + panel: 'reopened', + expectedDraft: '', + act: async () => { + await userEvent.click(screen.getByRole('button', { name: 'Stop' })) + } + } ])( 'settles a submission without losing composer intent: $nextAction ($panel)', - async ({ nextAction, panel }) => { + async ({ nextAction, panel, expectedDraft, act }) => { setupWorkflowContext({ targetId: 'wf-42', references: [ @@ -7109,21 +7837,28 @@ describe('AgentPanelRoot workflow binding', () => { setupNodeSelectionCanvas() const first = renderWithSelectedTarget() useAgentPanelStore().isOpen = true - await openMentionPicker() - await userEvent.click(await screen.findByText('KSampler')) let textbox = screen.getByRole('textbox') - await userEvent.type(textbox, ' Compare @') - await userEvent.click(screen.getByRole('menuitem', { name: 'Workflows' })) - await userEvent.click( - await screen.findByRole('menuitem', { name: 'reference' }) - ) const composer = useAgentComposerStore() - composer.addAttachment({ - id: 'upload-1', - name: 'cat.png', - ref: 'uploaded_cat.png' + composer.replaceDraft({ + text: 'Compare Keep this draft ', + workflowReferences: [ + { id: 'wf-reference', name: 'reference', textOffset: 8 } + ], + attachments: [ + { + id: 'upload-1', + name: 'cat.png', + ref: 'uploaded_cat.png' + } + ] + }) + composer.setNodes([{ id: '12', title: 'KSampler' }]) + composer.setInsertionPoint({ + textOffset: composer.draft.length, + referenceIndex: composer.prompt.references.length }) - await userEvent.keyboard(' Keep this draft ') + composer.referenceAttachment('upload-1') + const originalDraft = composer.draft const originalReferences = [...composer.workflowReferences] await userEvent.click(screen.getByRole('button', { name: 'Send' })) await vi.waitFor(() => expect(bodies).toHaveLength(1)) @@ -7142,53 +7877,8 @@ describe('AgentPanelRoot workflow binding', () => { textbox = screen.getByRole('textbox') expect(screen.getByRole('button', { name: 'Stop' })).toBeVisible() } - if (nextAction === 'new-draft' || nextAction === 'cleared-draft') - await userEvent.type(textbox, 'New input') - if (nextAction === 'cleared-draft') await userEvent.clear(textbox) - if (nextAction === 'removed-reference') { - await userEvent.click( - screen.getByRole('button', { - name: i18n.global.t('agent.addToPrompt') - }) - ) - await userEvent.click( - screen.getByRole('menuitem', { name: 'Workflows' }) - ) - await userEvent.click( - await screen.findByRole('menuitem', { name: 'reference' }) - ) - await userEvent.click( - screen.getByRole('button', { name: 'Remove reference reference' }) - ) - } - if (nextAction === 'removed-attachment') { - composer.addAttachment({ - id: 'upload-2', - name: 'new.png', - ref: 'new.png' - }) - await userEvent.click( - await screen.findByRole('button', { - name: i18n.global.t('agent.remove') - }) - ) - } - if (nextAction === 'new-chat') - await userEvent.click( - screen.getByRole('button', { name: i18n.global.t('agent.newChat') }) - ) - if (nextAction === 'history') { - await userEvent.click( - screen.getByRole('button', { - name: i18n.global.t('agent.showChatHistory') - }) - ) - await userEvent.click(await screen.findByText('Earlier chat')) - } - if (nextAction === 'stop') { - await userEvent.click(screen.getByRole('button', { name: 'Stop' })) - expect(cancellations).toHaveLength(0) - } + await act(textbox) + expect(cancellations).toHaveLength(0) finishSend( nextAction === 'stop' ? json(202, ack('wf-42', 'm-stopped')) @@ -7211,9 +7901,7 @@ describe('AgentPanelRoot workflow binding', () => { expect(screen.queryByRole('button', { name: 'Stop' })).toBeNull() ) if (nextAction === 'untouched') { - expect(useAgentComposerStore().draft).toBe( - ' Compare Keep this draft ' - ) + expect(useAgentComposerStore().draft).toBe(originalDraft) expect(composer.workflowReferences).toEqual(originalReferences) expect(composer.attachments).toMatchObject([ { ref: 'uploaded_cat.png' } @@ -7229,7 +7917,7 @@ describe('AgentPanelRoot workflow binding', () => { expect(bodies[1]).toMatchObject({ workflow_id: 'wf-42', content: - '@[Node: KSampler #12] Compare [reference](workflow://wf-reference) @[Image: cat.png] Keep this draft', + '@[Node: KSampler #12] Compare [reference](workflow://wf-reference) Keep this draft @[Image: cat.png]', selection: { node_ids: ['12'] }, attachments: ['uploaded_cat.png'], workflow_references: [ @@ -7237,9 +7925,7 @@ describe('AgentPanelRoot workflow binding', () => { ] }) } else { - expect(useAgentComposerStore().draft).toBe( - nextAction === 'new-draft' ? 'New input' : '' - ) + expect(useAgentComposerStore().draft).toBe(expectedDraft) expect(composer.attachments).toEqual([]) expect( screen.queryByRole('button', { name: 'Open reference' }) @@ -7381,6 +8067,33 @@ describe('AgentPanelRoot workflow binding', () => { expect(canvasStore.selectedItems).toEqual([state.nodes[0]]) }) + it.for([1, 2])( + 'redraws the canvas after removing an inline node from %i selected nodes', + async (count) => { + makeTab() + mockMessagesEndpoint('wf-42') + const state = setupNodeSelectionCanvas() + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + await enterNodeSelectionMode() + const selected = [state.nodes[1], state.nodes[0]].slice(0, count) + state.selectItems(selected) + canvasStore.updateSelectedItems() + await nextTick() + state.canvas.setDirty.mockClear() + + await userEvent.click( + screen.getByRole('button', { name: 'Remove KSampler #12 reference' }) + ) + + expect( + screen.queryByRole('button', { name: 'Remove KSampler #12 reference' }) + ).toBeNull() + expect([...state.selectedItems]).toEqual(selected.slice(1)) + expect(state.canvas.setDirty).toHaveBeenCalledWith(true) + } + ) + it('does not expose a canvas-focus action on a reference chip', async () => { makeTab() setupNodeSelectionCanvas() @@ -7438,6 +8151,50 @@ describe('AgentPanelRoot workflow binding', () => { expect([...state.selectedItems]).toEqual([rootTwin]) }) + it('updates the open node picker when the viewed graph changes without editing the prompt', async () => { + makeTab('wf-42') + const state = setupOwnedSelectionCanvas() + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + await openMentionPicker() + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + const prompt = useAgentComposerStore().prompt + + viewGraph(state.canvas, state.rootGraph) + + expect( + await screen.findByRole('menuitem', { name: 'Root node' }) + ).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().prompt).toEqual(prompt) + expect(useAgentComposerStore().nodes).toEqual([]) + }) + + it('lists target nodes loaded after opening the reference menu', async () => { + makeTab('wf-42') + const state = setupNodeSelectionCanvas() + showRootGraph(state) + renderWithSelectedTarget() + useAgentPanelStore().isOpen = true + + const textbox = screen.getByRole('textbox') + await userEvent.click(textbox) + await userEvent.paste('Use @') + showRootGraph(state, state.nodes) + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + await userEvent.click(screen.getByRole('menuitem', { name: 'KSampler' })) + expect(useAgentComposerStore().nodes).toEqual([ + { id: '12', locatorId: '12', title: 'KSampler' } + ]) + await userEvent.paste('next') + expect(useAgentComposerStore().draft).toBe('Use next') + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler' + ) + }) + it('excludes referenced nodes from the mention picker', async () => { makeTab() setupNodeSelectionCanvas() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 243864318fe..780976096fe 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -10,7 +10,6 @@ import { nextTick, onBeforeUnmount, provide, - readonly, ref, watch } from 'vue' @@ -606,7 +605,9 @@ let nodeReferenceWorkflow = selectionTags.value.length : null function viewedGraphNodes() { - return app.canvas?.graph?.nodes ?? app.graph?.nodes ?? [] + return ( + (canvasStore.currentGraph ?? app.canvas?.graph ?? app.graph)?.nodes ?? [] + ) } function mentionableNodes(): SelectedNode[] { @@ -1419,11 +1420,9 @@ function onNewChat(): void { else agentPanelStore.startFollowingVisibleWorkflow() } -const panelRef = ref>() const fileInput = ref() const assetDragActive = ref(false) let assetDragDepth = 0 -provide('agentAssetDragActive', readonly(assetDragActive)) let selectingNodes = false let nodeSelectionCanvas: LGraphCanvas | undefined let restoreAllowDragNodes: boolean | undefined @@ -1576,12 +1575,25 @@ const attachment = useAttachment({ // must not raise the server-error overlay. onError: (message) => toast.add({ severity: 'warn', detail: message, life: 5000 }), + onDuplicate: notifyDuplicateAttachments, stage: composerStore.addAttachment, update: composerStore.updateAttachment, remove: composerStore.removeAttachment, isPresent: (id) => composerStore.attachments.some((item) => item.id === id) }) +function notifyDuplicateAttachments(names: string[]): void { + toast.add({ + severity: 'info', + detail: t( + 'agent.attachmentsAlreadyAdded', + { name: names[0], count: names.length }, + names.length + ), + life: 3500 + }) +} + const removedUploadCancellationTimers = new Map() watch( @@ -1654,6 +1666,7 @@ function onRemoveSelectionTag(id: string): void { removeSelectionTag(id) if (node) { canvasStore.canvas?.deselect(node) + canvasStore.canvas?.setDirty(true) } } @@ -1699,7 +1712,7 @@ function onPanelDragLeave(): void { if (assetDragDepth === 0) assetDragActive.value = false } -async function attachDroppedAsset(event: DragEvent): Promise { +async function attachDroppedAsset(event: DragEvent): Promise { const asset = event.dataTransfer && getDroppedAsset(event.dataTransfer) if (!asset) { toast.add({ @@ -1707,10 +1720,11 @@ async function attachDroppedAsset(event: DragEvent): Promise { detail: t('agent.assetNotAttachable'), life: 5000 }) - return + return false } const assetFilename = asset.filename ?? asset.name + const sourceKey = asset.ref ? `asset:${asset.ref}` : `uri:${asset.uri}` const safeRef = asset.ref && !asset.ref.startsWith('/') && @@ -1723,18 +1737,26 @@ async function attachDroppedAsset(event: DragEvent): Promise { asset.kind !== 'other' && getMediaTypeFromFilename(assetFilename) === asset.kind ) { - panelRef.value?.addAttachment({ - id: `asset:${asset.ref}`, + if (composerStore.attachments.some((item) => item.ref === asset.ref)) { + notifyDuplicateAttachments([assetFilename]) + return false + } + const added = composerStore.addAttachment({ + id: `asset:${crypto.randomUUID()}`, name: assetFilename, ref: asset.ref, - previewUrl: asset.previewUrl + sourceKey, + previewUrl: asset.previewUrl, + mediaUrl: asset.mediaUrl, + mediaKind: asset.kind }) - return + if (!added) notifyDuplicateAttachments([assetFilename]) + return added } - if (!isAgentAttachable(new File([], assetFilename))) { + if (!asset.uri && !isAgentAttachable(new File([], assetFilename))) { toast.add({ severity: 'warn', detail: t('agent.assetNotAttachable') }) - return + return false } const result = await attachment.addDeferredFile( @@ -1746,7 +1768,8 @@ async function attachDroppedAsset(event: DragEvent): Promise { maxBytes ) return file - } + }, + sourceKey ) if (result === 'unsupported') toast.add({ @@ -1754,19 +1777,22 @@ async function attachDroppedAsset(event: DragEvent): Promise { detail: t('agent.assetNotAttachable'), life: 5000 }) + return result === 'uploaded' } function onPanelDragOver(event: DragEvent): void { if (isAttachableDrag(event)) event.preventDefault() } -function onPanelDrop(event: DragEvent): void { +async function onPanelDrop(event: DragEvent): Promise { clearAssetDrag() // A dropped asset card carries a URI, not a File, so the claim must happen // before the async fetch resolves it into one. if ((event.dataTransfer?.files.length ?? 0) === 0 && isAssetDrag(event)) { event.preventDefault() - void attachDroppedAsset(event) + void attachDroppedAsset(event).then((attached) => { + if (attached) useTelemetry()?.trackAgentAttachButtonClicked() + }) return } // Anything the composer cannot attach still belongs to the graph loader, which @@ -1776,14 +1802,15 @@ function onPanelDrop(event: DragEvent): void { ) if (files.length === 0) return event.preventDefault() - void attachment.addFiles(files) + if (await attachment.addFiles(files)) + useTelemetry()?.trackAgentAttachButtonClicked() } +
+
() => + h(Composer, { + hasWorkflowTarget: true, + selectionTags: store.nodes, + onSend: send + }) + }) + const view = render(Host, { + global: { plugins: [i18n], directives: { tooltip: () => {} } } + }) + return { ...view, store, send, editor: screen.getByRole('textbox') } +} + +function trackResizes() { + const callbacks = new Map void>>() + vi.stubGlobal( + 'ResizeObserver', + class implements ResizeObserver { + private targets = new Set() + private notify: () => void + constructor(callback: ResizeObserverCallback) { + this.notify = () => callback([], this) + } + observe(target: Element) { + this.targets.add(target) + const listeners = callbacks.get(target) ?? new Set<() => void>() + listeners.add(this.notify) + callbacks.set(target, listeners) + } + unobserve(target: Element) { + callbacks.get(target)?.delete(this.notify) + this.targets.delete(target) + } + disconnect() { + for (const target of this.targets) this.unobserve(target) + } + } + ) + return (target: Element) => + callbacks.get(target)?.forEach((notify) => notify()) +} + +function setTrayGeometry( + tray: HTMLElement, + width: number, + contentWidth: number +) { + const dimensions = { width, contentWidth } + let left = 0 + Object.defineProperties(tray, { + clientWidth: { get: () => dimensions.width }, + scrollWidth: { get: () => dimensions.contentWidth }, + scrollLeft: { + get: () => { + left = Math.min( + left, + Math.max(0, dimensions.contentWidth - dimensions.width) + ) + return left + }, + set: (value: number) => { + left = value + } + }, + scrollTo: { + value: (options: ScrollToOptions) => { + tray.scrollLeft = options.left ?? tray.scrollLeft + tray.dispatchEvent(new Event('scroll')) + } + } + }) + return dimensions +} + +describe('composer asset tray', () => { + it('updates every inline video thumbnail without changing its references or opening a preview', async () => { + const user = userEvent.setup() + const { store } = setup() + store.addAttachment({ + id: 'video', + name: 'My video', + ref: '', + mediaKind: 'video', + mediaUrl: 'blob:video', + uploading: true + }) + store.referenceAttachment('video') + store.referenceAttachment('video') + const chips = await screen.findAllByTestId('asset-reference-chip') + expect( + chips.map((chip) => + within(chip) + .getByRole('img', { name: 'Video' }) + .getAttribute('aria-label') + ) + ).toEqual(['Video', 'Video']) + store.updateAttachment('video', { + ref: 'stored.mp4', + mediaUrl: '/stored.mp4', + previewUrl: '/poster.png', + uploading: false + }) + await waitFor(() => + expect( + screen + .getAllByTestId('asset-reference-chip') + .map((chip) => within(chip).getByAltText('').getAttribute('src')) + ).toEqual(['/poster.png', '/poster.png']) + ) + await user.hover(screen.getAllByTestId('asset-reference-chip')[0]) + expect(screen.getByRole('group', { name: 'My video' })).toHaveAttribute( + 'data-highlighted', + 'true' + ) + expect( + screen.queryByRole('region', { name: 'My video' }) + ).not.toBeInTheDocument() + expect(store.prompt.references).toHaveLength(2) + expect(store.attachments).toHaveLength(1) + }) + + it('shows the audio type of a renamed inline attachment', async () => { + const { store } = setup() + store.addAttachment({ + id: 'audio', + name: 'Recording', + ref: 'song.mp3', + mediaKind: 'audio', + mediaUrl: '/song.mp3' + }) + store.referenceAttachment('audio') + const chip = await screen.findByTestId('asset-reference-chip') + expect(within(chip).getByRole('img', { name: 'Audio' })).toBeInTheDocument() + }) + + it.for([ + { + name: 'My video', + mediaKind: 'video', + previewUrl: '/poster.png', + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => [ + within(chip).getByAltText(''), + within(trayItem).getByAltText('My video'), + within(menuItem).getByAltText('') + ], + sources: ['/poster.png', '/poster.png', '/poster.png'], + labels: [null, null, null] + }, + { + name: 'Recording', + mediaKind: 'audio', + previewUrl: undefined, + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => + [chip, trayItem, menuItem].map((surface) => + within(surface).getByRole('img', { name: 'Audio' }) + ), + sources: [null, null, null], + labels: ['Audio', 'Audio', 'Audio'] + } + ] as const)( + 'uses the same media indicator for $mediaKind in the tray, menu and inline', + async ({ name, mediaKind, previewUrl, indicators, sources, labels }) => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment({ + id: 'media', + name, + ref: '/file', + mediaKind, + previewUrl + }) + store.referenceAttachment('media') + const chip = await screen.findByTestId('asset-reference-chip') + await user.click(editor) + await user.keyboard('@') + const menuItem = await screen.findByRole('menuitem', { name }) + const trayItem = screen.getByRole('group', { name }) + const images = indicators({ chip, trayItem, menuItem }) + expect(images.map((image) => image.getAttribute('src'))).toEqual(sources) + expect(images.map((image) => image.getAttribute('aria-label'))).toEqual( + labels + ) + await user.click(menuItem) + expect(await screen.findAllByTestId('asset-reference-chip')).toHaveLength( + 2 + ) + expect(store.attachments).toHaveLength(1) + } + ) + + it('announces all pending uploads outside the scroll viewport and preserves the draft on Enter', async () => { + const user = userEvent.setup() + const { store, editor, send } = setup() + const status = screen.getByTestId('composer-upload-status') + expect(status).toBeEmptyDOMElement() + expect(status).toHaveAttribute('aria-live', 'polite') + expect(status).toHaveAttribute('aria-atomic', 'true') + store.addAttachment({ + id: 'first', + name: 'first.png', + ref: '', + uploading: true + }) + store.addAttachment({ + id: 'second', + name: 'second.png', + ref: '', + uploading: true + }) + await user.type(editor, 'Keep my draft') + expect(status).toHaveTextContent('Uploading 2 attachments') + expect( + within(screen.getByTestId('composer-asset-section')).queryByTestId( + 'composer-upload-status' + ) + ).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Send' })).toBeDisabled() + await user.keyboard('{Enter}') + expect(send).not.toHaveBeenCalled() + expect(store.draft).toBe('Keep my draft') + store.updateAttachment('first', { ref: 'first.png', uploading: false }) + await waitFor(() => + expect(status).toHaveTextContent('Uploading 1 attachment') + ) + store.removeAttachment('second') + await waitFor(() => expect(status).toBeEmptyDOMElement()) + expect(screen.getByTestId('composer-upload-status')).toBe(status) + expect(screen.getByRole('button', { name: 'Send' })).toBeEnabled() + await user.keyboard('{Enter}') + expect(send).toHaveBeenCalledExactlyOnceWith('Keep my draft', [ + expect.objectContaining({ id: 'first', ref: 'first.png' }) + ]) + }) + + it('offers bounded page scrolling without changing the draft or included assets', async () => { + const resize = trackResizes() + const { store, editor } = setup() + const assets = Array.from({ length: 8 }, (_, index) => ({ + id: `asset-${index}`, + name: `asset-${index}.png`, + ref: `asset-${index}.png` + })) + store.replaceDraft({ + text: '', + attachments: assets, + workflowReferences: [] + }) + await userEvent.type(editor, 'Keep this draft') + const tray = screen.getByTestId('composer-asset-section') + setTrayGeometry(tray, 250, 720) + resize(tray) + await fireEvent.scroll(tray) + + const previous = screen.getByRole('button', { name: 'Scroll Left' }) + const next = screen.getByRole('button', { name: 'Scroll Right' }) + expect(previous).toBeDisabled() + expect(next).toBeEnabled() + await userEvent.click(next) + expect(tray.scrollLeft).toBe(250) + expect(previous).toBeEnabled() + expect(next).toBeEnabled() + await userEvent.keyboard('{Enter}') + expect(tray.scrollLeft).toBe(470) + expect(next).toBeDisabled() + await userEvent.click(previous) + expect(tray.scrollLeft).toBe(220) + expect(next).toBeEnabled() + expect(store.draft).toBe('Keep this draft') + expect(store.attachments).toEqual(assets) + }) + + it('updates overflow controls when the tray is resized', async () => { + const resize = trackResizes() + const { store } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await nextTick() + const tray = screen.getByTestId('composer-asset-section') + const dimensions = setTrayGeometry(tray, 240, 192) + resize(tray) + await nextTick() + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + + dimensions.width = 160 + resize(tray) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + + dimensions.width = 240 + resize(tray) + await nextTick() + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + expect(screen.queryByRole('button', { name: 'Scroll Left' })).toBeNull() + }) + + it('clears overflow controls when removing assets makes the tray fit', async () => { + const resize = trackResizes() + const { store } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await nextTick() + const tray = screen.getByTestId('composer-asset-section') + const dimensions = setTrayGeometry(tray, 160, 192) + resize(tray) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + + await userEvent.click(screen.getByRole('button', { name: 'Scroll Right' })) + expect(tray.scrollLeft).toBe(32) + + dimensions.contentWidth = 104 + await userEvent.click( + within(screen.getByRole('group', { name: 'other.png' })).getByRole( + 'button', + { name: 'Remove other.png' } + ) + ) + + await waitFor(() => + expect(screen.queryByRole('button', { name: 'Scroll Right' })).toBeNull() + ) + expect(store.attachments).toEqual([asset]) + + dimensions.contentWidth = 192 + store.addAttachment({ id: 'new', name: 'new.png', ref: 'new.png' }) + expect( + await screen.findByRole('button', { name: 'Scroll Right' }) + ).toBeEnabled() + expect(screen.getByRole('button', { name: 'Scroll Left' })).toBeDisabled() + store.removeAttachment('new') + store.removeAttachment(asset.id) + await nextTick() + expect(screen.queryByTestId('composer-asset-section')).toBeNull() + }) + + it('stages assets without changing the prompt, and sends unmentioned assets', async () => { + const { store, editor, send } = setup() + await userEvent.type(editor, 'Inspect these images') + store.addAttachment(asset) + await screen.findByRole('img', { name: asset.name }) + + expect(editor).toHaveTextContent('Inspect these images') + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect(store.draft).toBe('Inspect these images') + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith('Inspect these images', [asset]) + }) + + it.for([ + { + via: 'pointer', + pick: () => + userEvent.click(screen.getByRole('menuitem', { name: asset.name })) + }, + { via: 'keyboard', pick: () => userEvent.keyboard('{Enter}') } + ])( + 'inserts a matching asset at the caret via $via and retains the tray', + async ({ pick }) => { + const { store, editor, send } = setup() + store.addAttachment(asset) + store.addAttachment({ id: 'other', name: 'other.png', ref: 'other.png' }) + await userEvent.type(editor, 'Use @source') + const menu = screen.getByRole('menu', { name: 'Add to prompt' }) + expect( + within(menu).queryByRole('menuitem', { name: 'other.png' }) + ).toBeNull() + await pick() + await screen.findByTestId('asset-reference-chip') + expect(editor.textContent).toBe('Use source.png ') + expect(store.attachments).toEqual([ + asset, + { id: 'other', name: 'other.png', ref: 'other.png' } + ]) + expect( + within(screen.getByTestId('composer-asset-section')).getByRole( + 'group', + { name: asset.name } + ) + ).toBeVisible() + await userEvent.keyboard('as the subject') + expect(editor.textContent).toBe('Use source.png as the subject') + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith( + 'Use @[Image: source.png] as the subject', + [asset, { id: 'other', name: 'other.png', ref: 'other.png' }] + ) + } + ) + + it('keeps included assets through inline deletion, Undo and remount', async () => { + const { store, editor, unmount } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}{Backspace}{Backspace}') + await waitFor(() => + expect( + screen.queryByTestId('asset-reference-chip') + ).not.toBeInTheDocument() + ) + expect(store.attachments).toEqual([asset]) + await userEvent.keyboard('{Control>}z{/Control}') + await screen.findByTestId('asset-reference-chip') + expect(store.attachments).toEqual([asset]) + unmount() + setup() + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: asset.name + }) + ).toBeVisible() + expect(screen.getByTestId('asset-reference-chip')).toHaveTextContent( + asset.name + ) + }) + + it('allows repeated asset mentions, deletes only one, and sends the attachment once', async () => { + const { store, editor, send } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('then @source') + expect(screen.getByRole('menuitem', { name: asset.name })).toBeVisible() + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('and @source') + await userEvent.keyboard('{Enter}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(3) + ) + expect(editor.textContent).toBe( + 'Use source.png then source.png and source.png ' + ) + + await userEvent.keyboard('{Backspace}{Backspace}') + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + expect(store.attachments).toEqual([asset]) + await userEvent.keyboard('{Control>}z{/Control}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(3) + ) + await userEvent.click(screen.getByRole('button', { name: 'Send' })) + expect(send).toHaveBeenCalledWith( + 'Use @[Image: source.png] then @[Image: source.png] and @[Image: source.png]', + [asset] + ) + }) + + it('removes only the chosen inline asset with its hover button, retains the tray, and supports Undo and continued typing', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, 'Use @source') + await user.keyboard('{Enter}then @source{Enter}after') + const references = await screen.findAllByTestId('asset-reference-chip') + expect(references).toHaveLength(2) + await user.hover(references[0]) + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.click( + within(references[0]).getByRole('button', { + name: 'Remove source.png reference' + }) + ) + + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(1) + expect(editor.textContent).toBe('Use then source.png after') + expect(store.attachments).toEqual([asset]) + expect(editor).toHaveFocus() + expect(trayItem).not.toHaveAttribute('data-highlighted') + await user.keyboard('{Control>}z{/Control}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + ) + expect(editor.textContent).toBe('Use source.png then source.png after') + expect(trayItem).not.toHaveAttribute('data-highlighted') + await user.keyboard('continued ') + expect(editor).toHaveTextContent('continued') + expect(store.attachments).toEqual([asset]) + }) + + it('keeps typing uninterrupted after asset insertion and highlights its tray item on inline hover without opening a preview', async () => { + vi.useFakeTimers() + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, 'Use @source') + await user.keyboard('{Enter}') + const chip = screen.getByTestId('asset-reference-chip') + + chip.dispatchEvent( + new PointerEvent('pointerenter', { pointerType: 'mouse' }) + ) + await nextTick() + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + expect(editor).toHaveFocus() + await user.keyboard('and keep typing') + expect(editor.textContent).toBe('Use source.png and keep typing') + + await user.hover(chip) + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + expect(editor).toHaveFocus() + + await user.unhover(chip) + expect(trayItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset]) + }) + + it('highlights the tray while an inline asset has focus or hover, without opening a preview', async () => { + vi.useFakeTimers() + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }) + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}') + + await user.tab() + const chip = screen.getByTestId('asset-reference-chip') + expect(chip).toHaveFocus() + await vi.advanceTimersByTimeAsync(300) + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + + await user.hover(chip) + await user.tab({ shift: true }) + expect(editor).toHaveFocus() + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(chip) + expect(trayItem).not.toHaveAttribute('data-highlighted') + }) + + it('maps repeated inline references to their tray asset and clears only the activity that ended', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + const other = { id: 'other', name: 'other.png', ref: 'other.png' } + store.addAttachment(asset) + store.addAttachment(other) + await user.type(editor, '@source') + await user.keyboard('{Enter}then @source{Enter}and @other{Enter}') + const references = screen.getAllByTestId('asset-reference-chip') + expect(references).toHaveLength(3) + const tray = within(screen.getByTestId('composer-asset-section')) + const sourceItem = tray.getByRole('group', { name: asset.name }) + const otherItem = tray.getByRole('group', { name: other.name }) + + await user.tab() + expect(references[0]).toHaveFocus() + await user.hover(references[1]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).not.toHaveAttribute('data-highlighted') + await user.unhover(references[1]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + + await user.hover(references[2]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(references[2]) + expect(sourceItem).toHaveAttribute('data-highlighted', 'true') + expect(otherItem).not.toHaveAttribute('data-highlighted') + await user.click(editor) + expect(sourceItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset, other]) + }) + + it('retains the highlight from a hovered occurrence when another focused occurrence is removed', async () => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}then @source{Enter}') + const references = screen.getAllByTestId('asset-reference-chip') + const trayItem = within( + screen.getByTestId('composer-asset-section') + ).getByRole('group', { name: asset.name }) + + await user.tab() + expect(references[0]).toHaveFocus() + await user.hover(references[1]) + await user.tab() + expect( + within(references[0]).getByRole('button', { + name: 'Remove source.png reference' + }) + ).toHaveFocus() + await user.keyboard('{Enter}') + + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(1) + expect(editor).toHaveFocus() + expect(trayItem).toHaveAttribute('data-highlighted', 'true') + await user.unhover(references[1]) + expect(trayItem).not.toHaveAttribute('data-highlighted') + expect(store.attachments).toEqual([asset]) + }) + + it('opens a large asset preview from tray hover and closes it on leave or unmount', async () => { + const user = userEvent.setup() + const { store, editor, unmount } = setup() + store.addAttachment(asset) + await user.type(editor, '@source') + await user.keyboard('{Enter}') + await screen.findByTestId('asset-reference-chip') + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + + const trigger = screen.getByRole('button', { + name: `Preview ${asset.name}` + }) + + await user.hover(trigger) + const preview = await screen.findByRole('dialog', { name: asset.name }) + expect( + within(preview).getByRole('img', { name: asset.name }) + ).toHaveAttribute('src', asset.previewUrl) + await user.unhover(trigger) + await user.pointer({ + target: screen.getByRole('button', { name: 'Send' }), + coords: { clientX: 500, clientY: 500 } + }) + await waitFor(() => + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + ) + + await user.hover(trigger) + await screen.findByRole('dialog', { name: asset.name }) + unmount() + expect( + screen.queryByRole('dialog', { name: asset.name }) + ).not.toBeInTheDocument() + }) + + it('inserts an asset in the middle of a sentence and preserves the following text', async () => { + const { store, editor } = setup() + store.addAttachment(asset) + await userEvent.type(editor, 'Use as background') + await userEvent.keyboard('{Control>}a{/Control}{ArrowLeft}{ArrowRight>4/}') + await userEvent.paste('@source') + await userEvent.keyboard('{Enter}') + + await screen.findByTestId('asset-reference-chip') + expect(editor.textContent).toBe('Use source.png as background') + expect(store.prompt.text).toBe('Use as background') + }) + + it('retires a tray asset and its inline reference despite late upload completion or Undo', async () => { + const { store, editor } = setup() + store.addAttachment({ ...asset, uploading: true, ref: '' }) + await userEvent.type(editor, 'Use @source') + await userEvent.keyboard('{Enter}') + await userEvent.keyboard('and @source') + await userEvent.keyboard('{Enter}') + await waitFor(() => + expect(screen.getAllByTestId('asset-reference-chip')).toHaveLength(2) + ) + await userEvent.click( + within(screen.getByTestId('composer-asset-section')).getByRole('button', { + name: 'Remove source.png' + }) + ) + store.updateAttachment(asset.id, { uploading: false, ref: asset.ref }) + await waitFor(() => + expect( + screen.queryByTestId('composer-asset-section') + ).not.toBeInTheDocument() + ) + await userEvent.click(editor) + await userEvent.keyboard('{Control>}z{/Control}') + expect(store.attachments).toEqual([]) + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + }) + + it('shows nodes only inline without adding an asset', async () => { + const { store } = setup() + store.setNodes([{ id: '361', title: 'Save Image' }]) + await screen.findByTestId('node-reference-chip') + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect( + screen.queryByTestId('composer-asset-section') + ).not.toBeInTheDocument() + expect(screen.getByRole('textbox')).toHaveTextContent('Save Image #361') + }) +}) diff --git a/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts b/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts index d06f8eaf7e3..d7cef2ee973 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.inlineReferences.test.ts @@ -46,59 +46,28 @@ function renderComposer() { describe('inline node and asset references', () => { beforeEach(() => vi.useRealTimers()) - it('keeps upper-row removal and Undo synchronized with inline references', async () => { + it('keeps inline node deletion and Undo synchronized with selected context', async () => { const { store, selected, editor, send } = renderComposer() await userEvent.type(editor, 'Use ') selected.value = [{ id: '12', title: 'KSampler' }] - await waitFor(() => expect(editor.textContent).toBe('Use KSampler #12 ')) - await userEvent.keyboard('with ') + await screen.findByTestId('node-reference-chip') + await userEvent.keyboard('{Backspace}{Backspace}') + expect(store.nodes).toEqual([]) + expect(editor.textContent).toBe('Use ') + await userEvent.keyboard('{Control>}z{/Control}') + await screen.findByTestId('node-reference-chip') + expect(store.nodes).toEqual([{ id: '12', title: 'KSampler' }]) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + await userEvent.keyboard(' with ') store.addAttachment({ id: 'image', name: 'source.png', ref: 'uploaded.png' }) - await waitFor(() => - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') - ) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #12 reference' }) - ) - await waitFor(() => - expect( - screen.queryByTestId('composer-node-section') - ).not.toBeInTheDocument() - ) - expect(editor.textContent).toBe('Use with source.png ') - expect(store.nodes).toEqual([]) - editor.focus() - await userEvent.keyboard('{Control>}z{/Control}') - await waitFor(() => - expect(screen.getByTestId('composer-node-section')).toHaveTextContent( - 'KSampler' - ) - ) - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') - - await userEvent.click( - within(screen.getByTestId('composer-asset-section')).getByRole('button', { - name: 'Remove' - }) - ) - await waitFor(() => - expect( - screen.queryByTestId('composer-asset-section') - ).not.toBeInTheDocument() - ) - expect(editor.textContent).toBe('Use KSampler #12 with ') - editor.focus() - await userEvent.keyboard('{Control>}z{/Control}') - await waitFor(() => - expect(screen.getByTestId('composer-asset-section')).toHaveTextContent( - 'source.png' - ) - ) - expect(editor.textContent).toBe('Use KSampler #12 with source.png ') + store.referenceAttachment('image') + await screen.findByTestId('asset-reference-chip') await userEvent.click(screen.getByRole('button', { name: 'Send' })) expect(send).toHaveBeenCalledWith( 'Use @[Node: KSampler #12] with @[Image: source.png]', @@ -115,10 +84,11 @@ describe('inline node and asset references', () => { ref: '', uploading: true }) + store.referenceAttachment('image') await screen.findByTestId('asset-reference-chip') await userEvent.click( within(screen.getByTestId('composer-asset-section')).getByRole('button', { - name: 'Remove' + name: 'Remove pending.png' }) ) store.updateAttachment('image', { ref: 'complete.png', uploading: false }) @@ -132,15 +102,8 @@ describe('inline node and asset references', () => { ).not.toBeInTheDocument() editor.focus() await userEvent.keyboard('{Control>}z{/Control}') - await screen.findByTestId('asset-reference-chip') - expect(store.attachments).toEqual([ - { - id: 'image', - name: 'pending.png', - ref: 'complete.png', - uploading: false - } - ]) + expect(screen.queryByTestId('asset-reference-chip')).not.toBeInTheDocument() + expect(store.attachments).toEqual([]) }) it('deletes a node from the sentence without restaging the selected canvas node', async () => { @@ -166,6 +129,7 @@ describe('inline node and asset references', () => { selected.value = [{ id: '12', title: 'KSampler' }] await screen.findByTestId('node-reference-chip') store.addAttachment({ id: 'image', name: 'source.png', ref: 'source.png' }) + store.referenceAttachment('image') await screen.findByTestId('asset-reference-chip') await user.keyboard('{Control>}a{/Control}') const clipboard = await user.copy() diff --git a/src/workbench/extensions/agent/components/agent/Composer.test.ts b/src/workbench/extensions/agent/components/agent/Composer.test.ts index ec07df182ca..be8e20bf564 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.test.ts @@ -10,11 +10,12 @@ import { render, screen, waitFor, within } from '@testing-library/vue' import userEvent from '@testing-library/user-event' import { createPinia, setActivePinia } from 'pinia' import { beforeEach, describe, expect, it, vi } from 'vitest' -import { defineComponent, h, nextTick, ref } from 'vue' -import type { DirectiveBinding } from 'vue' +import { defineComponent, h, nextTick, ref, shallowRef } from 'vue' +import type { DirectiveBinding, ShallowRef } from 'vue' import type { ComponentProps } from 'vue-component-type-helpers' import { i18n } from '@/i18n' +import { LGraph, LGraphNode } from '@/lib/litegraph/src/litegraph' import { consultEscapeOverride } from '@/platform/keybindings/escapeOverride' import { useToastStore } from '@/platform/updates/common/toastStore' import { useAgentRunModeStore } from '../../stores/agent/agentRunModeStore' @@ -194,15 +195,38 @@ describe('Composer', () => { ).toBeTruthy() }) - it('hides the empty-composer hint once typing begins', async () => { - mount() - const box = screen.getByRole('textbox') + it.for([ + { tray: 'empty', attachments: [] }, + { + tray: 'with an asset', + attachments: [{ id: 'asset', name: 'cat.png', ref: 'cat.png' }] + } + ])( + 'hides the hint while typing and restores it after clearing (tray=$tray)', + async ({ attachments }) => { + const store = useAgentComposerStore() + store.replaceDraft({ text: '', workflowReferences: [], attachments }) + const { emitted } = mount() + const box = screen.getByRole('textbox') + expect( + screen.getByRole('button', { name: 'mention nodes' }) + ).toBeVisible() - await userEvent.type(box, 'hello') + await userEvent.click(box) + await userEvent.paste('hello') - expect(useAgentComposerStore().draft).toBe('hello') - expect(screen.queryByRole('button', { name: 'mention nodes' })).toBeNull() - }) + expect(store.draft).toBe('hello') + expect(screen.queryByRole('button', { name: 'mention nodes' })).toBeNull() + + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(store.draft).toBe('') + await userEvent.click( + screen.getByRole('button', { name: 'mention nodes' }) + ) + expect(emitted().selectNodes).toHaveLength(1) + expect(store.attachments).toEqual(attachments) + } + ) it('enters graph selection mode from the empty-composer hint', async () => { const getMentionNodes = vi.fn(() => []) @@ -799,6 +823,102 @@ describe('Composer', () => { ).toEqual(['Back', 'Alpha', 'KSampler', 'VAE Decode']) }) + it('loads current nodes when entering or re-entering the Nodes submenu', async () => { + const nodes = ref([]) + mount({ getMentionNodes: () => nodes.value }) + await openReferenceRoot() + + nodes.value = [NODES[0]] + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + + await userEvent.click(screen.getByRole('menuitem', { name: 'Back' })) + nodes.value = [NODES[2]] + await userEvent.click(screen.getByRole('menuitem', { name: 'Nodes' })) + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().draft).toBe('@') + }) + + it('filters current nodes after the graph changes with the submenu open', async () => { + const nodes = ref([NODES[0]]) + const { emitted } = mount({ getMentionNodes: () => nodes.value }) + await openReferenceSection('Nodes') + + nodes.value = [NODES[2]] + await userEvent.click(screen.getByRole('textbox')) + await userEvent.paste('vae') + + expect(screen.getByRole('menuitem', { name: 'VAE Decode' })).toBeVisible() + expect(screen.queryByRole('menuitem', { name: 'KSampler' })).toBeNull() + expect(useAgentComposerStore().draft).toBe('@vae') + await userEvent.keyboard('{Enter}') + expect(emitted().mentionPick).toEqual([[NODES[2]]]) + expect(emitted().send).toBeUndefined() + }) + + it.for<{ + event: string + act: (graph: ShallowRef, node: LGraphNode) => void + labels: string[] + }>([ + { + event: 'adding a node', + act: (graph) => graph.value.add(new LGraphNode('Alpha')), + labels: ['Back', 'Alpha', 'KSampler'] + }, + { + event: 'renaming a node', + act: (_graph, node) => { + node.title = 'Alpha' + }, + labels: ['Back', 'Alpha'] + }, + { + event: 'removing a node', + act: (graph, node) => graph.value.remove(node), + labels: ['Back'] + }, + { + event: 'replacing the viewed graph', + act: (graph) => { + const replacement = new LGraph() + replacement.add(new LGraphNode('VAE Decode')) + graph.value = replacement + }, + labels: ['Back', 'VAE Decode'] + } + ])( + 'updates the open Nodes submenu after $event without typing', + async ({ act, labels }) => { + const graph = shallowRef(new LGraph()) + const node = new LGraphNode('KSampler') + graph.value.add(node) + const { emitted } = mount({ + getMentionNodes: () => + graph.value.nodes.map((item) => ({ + id: String(item.id), + title: item.title + })) + }) + await openReferenceSection('Nodes') + expect(screen.getByRole('menuitem', { name: 'KSampler' })).toBeVisible() + + act(graph, node) + + await waitFor(() => + expect( + within(screen.getByRole('menu', { name: 'Add to prompt' })) + .getAllByRole('menuitem') + .map((item) => item.textContent.trim()) + ).toEqual(labels) + ) + expect(useAgentComposerStore().draft).toBe('@') + expect(emitted().mentionPick).toBeUndefined() + expect(emitted().send).toBeUndefined() + } + ) + // Re-picking a staged node is a no-op, so it drops out of the list. it('hides nodes already in the basket', async () => { mount({ @@ -815,11 +935,11 @@ describe('Composer', () => { expect(labels).toEqual(['Back', 'KSampler #7', 'VAE Decode']) }) - // The staged node is only hidden from the picker, not from the duplicate - // check - its chip must still show the id that tells it apart from the - // same-titled node still in the graph. it('keeps disambiguating a staged node against its graph twin', async () => { const twin = { id: '11', title: NODES[0].title } + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([NODES[0]]) mount({ getMentionNodes: () => [...NODES, twin], selectionTags: [NODES[0]] @@ -827,7 +947,12 @@ describe('Composer', () => { await openReferenceSection('Nodes') - expect(screen.getByText(`#${NODES[0].id}`)).toBeInTheDocument() + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler #5' + ) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() }) it('keeps type-to-filter behavior inside the selected reference type', async () => { @@ -1214,14 +1339,16 @@ describe('Composer', () => { it.for(['pointer', 'Enter', 'Space'])( 'opens a staged workflow with %s without consuming the draft', async (interaction) => { + const store = useAgentComposerStore() + store.setText('Keep this prompt') + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) const { emitted } = mount({ workflowReferences: [ { id: 'wf-1', name: 'Water world', textOffset: 0 } ], selectionTags: [{ id: '5', title: 'KSampler' }] }) - const textarea = screen.getByRole('textbox') - await userEvent.type(textarea, 'Keep this prompt') const chip = screen.getByRole('button', { name: 'Open Water world' }) if (interaction === 'pointer') await userEvent.click(chip) else { @@ -1232,9 +1359,15 @@ describe('Composer', () => { expect(emitted().openReferenceWorkflow).toEqual([['wf-1', 'Water world']]) expect(emitted().removeWorkflowReference).toBeUndefined() expect(emitted().send).toBeUndefined() - expect(useAgentComposerStore().draft).toBe('Keep this prompt') - expect(screen.getByTestId('composer-node-section')).toHaveTextContent( - 'KSampler' + expect(store.draft).toBe('Keep this prompt ') + expect(store.nodes).toEqual([{ id: '5', title: 'KSampler' }]) + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler #5' + ) + expect(store.draft).toBe('Keep this prompt ') + expect(store.nodes).toEqual([{ id: '5', title: 'KSampler' }]) + expect(screen.getByTestId('node-reference-chip')).toHaveTextContent( + 'KSampler #5' ) } ) @@ -1334,20 +1467,20 @@ describe('Composer', () => { expect(useAgentComposerStore().draft).toBe('tex') }) - it('keeps selected nodes in a dedicated section above the inline prompt', () => { + it('keeps selected nodes inline alongside workflow references', () => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) mount({ - selectionTags: [{ id: '5', title: 'KSampler' }], + selectionTags: store.nodes, workflowReferences: [{ id: 'wf-1', name: 'Water world', textOffset: 0 }] }) - - const nodeSection = screen.getByTestId('composer-node-section') - const inlineInput = screen.getByTestId('composer-inline-input') - - expect(nodeSection).toHaveClass('border-b', 'p-3') - expect(nodeSection).toHaveTextContent('KSampler') - expect(nodeSection).not.toContainElement(screen.getByRole('textbox')) - expect(inlineInput).not.toHaveTextContent('KSampler') - expect(inlineInput).toHaveTextContent('Water world') + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect(screen.getByRole('textbox')).toHaveTextContent( + 'Water worldKSampler #5' + ) }) it('keeps added assets in a separate padded section above the prompt', async () => { @@ -1400,9 +1533,11 @@ describe('Composer', () => { screen.getByRole('button', { name: 'Open Water world' }) ) expect(onOpenReferenceWorkflow).toHaveBeenCalledWith('wf-1', 'Water world') - expect(screen.getByTestId('composer-asset-section')).toHaveTextContent( - 'cat.png' - ) + expect( + within(screen.getByTestId('composer-asset-section')).getByRole('group', { + name: 'cat.png' + }) + ).toBeVisible() }) it('keeps the empty prompt hint visible when only nodes are selected', () => { @@ -1462,81 +1597,51 @@ describe('Composer', () => { expect(getMentionNodes).not.toHaveBeenCalled() }) - it('hides the id on a uniquely named selection chip', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - expect(screen.getByText('KSampler')).toBeInTheDocument() - expect(screen.queryByText('#5')).not.toBeInTheDocument() - }) - - it('emits removeTag when a selection chip is removed', async () => { - const { emitted } = mount({ - selectionTags: [{ id: '5', title: 'KSampler' }] - }) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #5 reference' }) - ) - - expect(emitted().removeTag).toEqual([['5']]) - }) - - it('builds the remove tooltip for a selection chip', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - const removeButton = screen.getByRole('button', { - name: 'Remove KSampler #5 reference' - }) - expect(tooltipBindings.get(removeButton)).toMatchObject({ value: 'Remove' }) - }) - - it('renders a selection chip label as non-interactive context', () => { - mount({ selectionTags: [{ id: '5', title: 'KSampler' }] }) - - expect(screen.getByText('KSampler')).toBeVisible() - expect( - screen.queryByRole('button', { name: 'Show KSampler #5 on canvas' }) - ).toBeNull() - }) - - // The remove button sits outside the focus trigger; removing a chip must not - // also fly the canvas to the node being removed. - it('removes a selection chip without focusing its node', async () => { - const { emitted } = mount({ - selectionTags: [{ id: '5', title: 'KSampler' }] - }) - - await userEvent.click( - screen.getByRole('button', { name: 'Remove KSampler #5 reference' }) - ) - - expect(emitted().removeTag).toEqual([['5']]) - expect(emitted().focusTag).toBeUndefined() - }) - - it('shows the id on a lone selection chip with a graph title twin', () => { - const selected = { id: '5', title: 'KSampler' } - mount({ - selectionTags: [selected], - getMentionNodes: () => [selected, { id: '7', title: 'KSampler' }] - }) - - expect(screen.getByText('KSampler')).toBeInTheDocument() - expect(screen.getByText('#5')).toBeInTheDocument() - expect(screen.queryByText('#7')).not.toBeInTheDocument() - }) - - it('shows ids when selection chips have duplicate titles', () => { - mount({ - selectionTags: [ + it.for([ + { + label: 'unique title', + nodes: [{ id: '5', title: 'KSampler' }], + labels: ['KSampler #5'] + }, + { + label: 'duplicate titles', + nodes: [ { id: '5', title: 'KSampler' }, { id: '7', title: 'KSampler' } - ] - }) + ], + labels: ['KSampler #5', 'KSampler #7'] + } + ])( + 'identifies inline nodes with $label by their IDs', + ({ nodes, labels }) => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes(nodes) + mount({ selectionTags: store.nodes }) + expect( + screen + .getAllByTestId('node-reference-chip') + .map((chip) => chip.textContent) + ).toEqual(labels) + expect( + screen.queryByTestId('composer-node-section') + ).not.toBeInTheDocument() + expect( + screen.queryByRole('button', { name: 'Show KSampler #5 on canvas' }) + ).toBeNull() + } + ) - expect(screen.getAllByText('KSampler')).toHaveLength(2) - expect(screen.getByText('#5')).toBeInTheDocument() - expect(screen.getByText('#7')).toBeInTheDocument() + it('removes an inline node without focusing its canvas node', async () => { + const store = useAgentComposerStore() + store.setNodeScope('workflow-a') + store.setNodes([{ id: '5', title: 'KSampler' }]) + const { emitted } = mount({ selectionTags: store.nodes }) + await userEvent.click(screen.getByRole('textbox')) + await userEvent.keyboard('{Control>}a{/Control}{Backspace}') + expect(emitted().removeTag).toEqual([['5']]) + expect(emitted().focusTag).toBeUndefined() + expect(store.nodes).toEqual([]) }) it('renders an attachment preview and removes it from the composer', async () => { @@ -1558,7 +1663,9 @@ describe('Composer', () => { 'https://example.com/cat.png' ) - await userEvent.click(screen.getByRole('button', { name: 'Remove' })) + await userEvent.click( + screen.getByRole('button', { name: 'Remove cat.png' }) + ) expect(screen.queryByRole('img', { name: 'cat.png' })).toBeNull() }) diff --git a/src/workbench/extensions/agent/components/agent/Composer.vue b/src/workbench/extensions/agent/components/agent/Composer.vue index 8585572759c..9e167d5a5ce 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.vue +++ b/src/workbench/extensions/agent/components/agent/Composer.vue @@ -1,18 +1,6 @@ + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts new file mode 100644 index 00000000000..307b3ea14fb --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.test.ts @@ -0,0 +1,29 @@ +import { render, screen } from '@testing-library/vue' +import { assert, describe, expect, it, vi } from 'vitest' + +import { i18n } from '@/i18n' + +import AssetMediaPreview from './AssetMediaPreview.vue' + +describe('retained asset media previews', () => { + it.for(['video', 'audio'] as const)( + 'stops and resets %s when closed before unmounting', + async (kind) => { + const { rerender } = render(AssetMediaPreview, { + props: { name: 'Media', mediaUrl: '/media', kind, active: true }, + global: { plugins: [i18n] } + }) + const player = screen.getByLabelText('Media', { selector: kind }) + assert.instanceOf(player, HTMLMediaElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await rerender({ active: false }) + expect(player).toBeInTheDocument() + expect(pause).toHaveBeenCalledOnce() + expect(player.currentTime).toBe(0) + await rerender({ active: true }) + expect(screen.getByLabelText('Media', { selector: kind })).toBe(player) + expect(player.currentTime).toBe(0) + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue new file mode 100644 index 00000000000..7df00f0285b --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue @@ -0,0 +1,74 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts new file mode 100644 index 00000000000..ffd754fe187 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.test.ts @@ -0,0 +1,44 @@ +import { fireEvent, render, screen } from '@testing-library/vue' +import { describe, expect, it } from 'vitest' + +import { i18n } from '@/i18n' + +import AssetThumbnail from './AssetThumbnail.vue' + +describe('asset poster loading', () => { + it.for(['tray', 'menu', 'inline'] as const)( + 'falls back after a failed poster and retries a replacement URL in the %s', + async (variant) => { + const { rerender } = render(AssetThumbnail, { + props: { + name: 'My video', + mediaKind: 'video', + previewUrl: '/expired.png', + variant + }, + global: { plugins: [i18n] } + }) + const poster = screen.getByAltText(variant === 'tray' ? 'My video' : '') + await fireEvent.error(poster) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + expect( + screen.queryByAltText(variant === 'tray' ? 'My video' : '') + ).not.toBeInTheDocument() + await rerender({ previewUrl: '/replacement.png' }) + const replacement = screen.getByAltText( + variant === 'tray' ? 'My video' : '' + ) + expect(replacement).toHaveAttribute('src', '/replacement.png') + await fireEvent.error(poster) + expect(replacement).toBeInTheDocument() + await fireEvent.error(replacement) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + await fireEvent.error(poster) + expect(screen.getByRole('img', { name: 'Video' })).toBeVisible() + await rerender({ previewUrl: '/expired.png' }) + expect( + screen.getByAltText(variant === 'tray' ? 'My video' : '') + ).toHaveAttribute('src', '/expired.png') + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue new file mode 100644 index 00000000000..af2c4eff2ab --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetThumbnail.vue @@ -0,0 +1,73 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts b/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts new file mode 100644 index 00000000000..710e51fe0bd --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetTray.test.ts @@ -0,0 +1,170 @@ +import userEvent from '@testing-library/user-event' +import { render, screen } from '@testing-library/vue' +import { assert, describe, expect, it, onTestFinished, vi } from 'vitest' +import { nextTick } from 'vue' + +import { i18n } from '@/i18n' + +import type { ComposerAttachment } from '../../../types/composerAttachment' +import AssetTray from './AssetTray.vue' + +const images: ComposerAttachment[] = [ + { id: 'cat', name: 'cat.png', ref: 'cat.png', previewUrl: '/cat.png' }, + { id: 'dog', name: 'dog.png', ref: 'dog.png', previewUrl: '/dog.png' } +] + +function renderTray(attachments = images) { + const view = render(AssetTray, { + props: { attachments }, + global: { plugins: [i18n], directives: { tooltip: () => {} } } + }) + onTestFinished(async () => { + view.unmount() + await nextTick() + }) + return { + ...view, + user: userEvent.setup({ advanceTimers: vi.advanceTimersByTimeAsync }) + } +} + +describe('asset tray previews', () => { + it('replaces a clicked preview when another asset is hovered', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.click(first) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + + await user.hover(second) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(first).toHaveAttribute('aria-expanded', 'false') + + await user.unhover(second) + await vi.advanceTimersByTimeAsync(150) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it('keeps the newly clicked preview open after the previous hover settles', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.hover(first) + await vi.advanceTimersByTimeAsync(250) + const preview = screen.getByRole('dialog', { name: 'cat.png' }) + expect(preview).toBeVisible() + await user.hover(screen.getByRole('region', { name: 'cat.png' })) + await vi.advanceTimersByTimeAsync(150) + + await user.click(second) + await vi.advanceTimersByTimeAsync(250) + await user.unhover(second) + await vi.advanceTimersByTimeAsync(150) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(preview).not.toBeInTheDocument() + + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + expect(second).toHaveFocus() + await user.hover(first) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + }) + + it('switches keyboard activation and returns focus to the current trigger on Escape', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + first.focus() + await user.keyboard('{Enter}') + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + second.focus() + await user.keyboard(' ') + + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + expect(second).toHaveFocus() + }) + + it('switches previews on touch activation without depending on hover', async () => { + const { user } = renderTray() + const first = screen.getByRole('button', { name: 'Preview cat.png' }) + const second = screen.getByRole('button', { name: 'Preview dog.png' }) + await user.pointer([ + { keys: '[TouchA>]', target: first }, + { keys: '[/TouchA]', target: first } + ]) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await user.pointer([ + { keys: '[TouchA>]', target: second }, + { keys: '[/TouchA]', target: second } + ]) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + }) + + it('does not reopen a removed active preview when the same attachment returns', async () => { + const { user, rerender } = renderTray() + await user.click(screen.getByRole('button', { name: 'Preview cat.png' })) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await rerender({ attachments: [images[1]] }) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + + await rerender({ attachments: images }) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + }) + + it('keeps typing focus and text while switching hover previews', async () => { + const { user } = renderTray() + const promptView = render({ template: '' }) + onTestFinished(promptView.unmount) + const prompt = screen.getByRole('textbox', { name: 'Prompt' }) + await user.type(prompt, 'Keep typing') + await user.hover(screen.getByRole('button', { name: 'Preview cat.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'cat.png' })).toBeVisible() + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(prompt).toHaveFocus() + await user.keyboard(' here') + expect(prompt).toHaveValue('Keep typing here') + }) + + it.for([ + { name: 'song.mp3', mediaKind: 'audio', selector: 'audio' }, + { name: 'clip.mp4', mediaKind: 'video', selector: 'video' } + ] as const)( + 'stops $mediaKind playback when a hovered asset replaces its interactive preview', + async ({ name, mediaKind, selector }) => { + const { user } = renderTray([ + { id: 'media', name, ref: name, mediaUrl: '/media', mediaKind }, + images[1] + ]) + await user.click(screen.getByRole('button', { name: `Preview ${name}` })) + const player = screen.getByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + + await user.hover(screen.getByRole('button', { name: 'Preview dog.png' })) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getAllByRole('dialog')).toHaveLength(1) + expect(screen.getByRole('dialog', { name: 'dog.png' })).toBeVisible() + expect(player.currentTime).toBe(0) + expect(pause).toHaveBeenCalled() + expect(player).not.toBeInTheDocument() + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue b/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue new file mode 100644 index 00000000000..1860beecc76 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/AssetTray.vue @@ -0,0 +1,137 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts index 6180dc23192..5aad2a9e65f 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts @@ -1,6 +1,15 @@ -import { render, screen } from '@testing-library/vue' -import { describe, expect, it } from 'vitest' +import userEvent from '@testing-library/user-event' +import { + fireEvent, + render, + screen, + waitFor, + within +} from '@testing-library/vue' +import { assert, describe, expect, it, onTestFinished, vi } from 'vitest' +import { nextTick } from 'vue' +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' import { i18n } from '@/i18n' import AttachmentChip from './AttachmentChip.vue' @@ -8,12 +17,19 @@ import AttachmentChip from './AttachmentChip.vue' function renderChip(props: { name: string previewUrl?: string + mediaUrl?: string + mediaKind?: MediaKind uploading?: boolean }) { - return render(AttachmentChip, { + const rendered = render(AttachmentChip, { props, global: { plugins: [i18n] } }) + onTestFinished(async () => { + rendered.unmount() + await nextTick() + }) + return rendered } function iconMarker(container: Element): string { @@ -25,6 +41,173 @@ function iconMarker(container: Element): string { } describe('AttachmentChip', () => { + it.for([ + { name: 'cat.png', previewUrl: 'blob:cat', key: '{Enter}' }, + { name: 'notes.txt', previewUrl: undefined, key: ' ' } + ])( + 'discloses $name from the keyboard and restores focus on Escape', + async ({ key, ...props }) => { + const user = userEvent.setup() + renderChip(props) + expect(screen.queryByText(props.name)).not.toBeInTheDocument() + await user.tab() + const trigger = screen.getByRole('button', { + name: `Preview ${props.name}` + }) + expect(trigger).toHaveFocus() + await user.keyboard(key) + const preview = await screen.findByRole('dialog', { name: props.name }) + expect(within(preview).getByText(props.name)).toBeVisible() + await user.keyboard('{Escape}') + await waitFor(() => expect(preview).not.toBeInTheDocument()) + expect(trigger).toHaveFocus() + } + ) + + it('names the removal action and keeps it separate from preview activation', async () => { + const user = userEvent.setup() + const { emitted } = renderChip({ name: 'notes.txt' }) + await user.click(screen.getByRole('button', { name: 'Remove notes.txt' })) + expect(emitted().remove).toHaveLength(1) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it.for([ + { name: 'clip.mp4', selector: 'video', key: '{Enter}' }, + { name: 'song.mp3', selector: 'audio', key: ' ' } + ])( + 'opens $name playback from the keyboard, focuses native controls and restores focus on Escape', + async ({ name, selector, key }) => { + const user = userEvent.setup() + renderChip({ name, mediaUrl: '/playable-media' }) + const trigger = screen.getByRole('button', { name: `Preview ${name}` }) + await user.tab() + expect(trigger).toHaveFocus() + await user.keyboard(key) + expect(await screen.findByRole('dialog', { name })).toBeVisible() + await waitFor(() => + expect(trigger).toHaveAttribute('aria-expanded', 'true') + ) + const player = screen.getByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + await waitFor(() => expect(player).toHaveFocus()) + expect(player).toHaveAttribute('controls') + expect(player).not.toHaveAttribute('tabindex', '-1') + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.keyboard('{Escape}') + await waitFor(() => expect(player).not.toBeInTheDocument()) + await waitFor(() => expect(trigger).toHaveFocus()) + expect(trigger).toHaveAttribute('aria-expanded', 'false') + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + await user.keyboard(key) + expect(await screen.findByLabelText(name, { selector })).toHaveAttribute( + 'src', + '/playable-media' + ) + } + ) + + it('keeps removal independent of opening media playback', async () => { + const user = userEvent.setup() + const { emitted } = renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + await user.click(screen.getByRole('button', { name: 'Remove clip.mp4' })) + expect(emitted().remove).toHaveLength(1) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + }) + + it('promotes a hovered preview to keyboard playback and keeps it open after the pointer leaves', async () => { + const user = userEvent.setup() + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + expect(trigger).toHaveFocus() + await user.hover(trigger) + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + expect(trigger).toHaveFocus() + await user.keyboard('{Enter}') + expect(player).toHaveFocus() + await user.unhover(trigger) + await user.pointer({ target: document.body }) + expect(screen.getByRole('dialog', { name: 'clip.mp4' })).toBeVisible() + await user.keyboard('{Escape}') + await waitFor(() => + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + ) + expect(trigger).toHaveFocus() + }) + + it('preserves typing focus when a hover preview opens and closes', async () => { + const user = userEvent.setup() + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const promptView = render({ template: '' }) + onTestFinished(promptView.unmount) + const prompt = screen.getByRole('textbox', { name: 'Prompt' }) + await user.click(prompt) + await user.type(prompt, 'Keep typing') + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.hover(trigger) + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + expect(prompt).toHaveFocus() + await user.unhover(trigger) + await user.pointer({ target: document.body }) + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(prompt).toHaveFocus() + expect(prompt).toHaveValue('Keep typing') + }) + + it('keeps keyboard playback focused when its source is replaced and clears an earlier load error', async () => { + const user = userEvent.setup() + const { rerender } = renderChip({ name: 'clip.mp4', mediaUrl: 'blob:clip' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + await user.keyboard('{Enter}') + const player = await screen.findByLabelText('clip.mp4', { + selector: 'video' + }) + assert.instanceOf(player, HTMLVideoElement) + await waitFor(() => expect(player).toHaveFocus()) + await fireEvent.error(player) + expect(screen.getByText('Video failed to load')).toBeVisible() + const pause = vi.spyOn(player, 'pause') + await rerender({ mediaUrl: '/uploaded.mp4' }) + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveFocus() + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveAttribute('src', '/uploaded.mp4') + expect(screen.queryByText('Video failed to load')).not.toBeInTheDocument() + expect(pause).toHaveBeenCalledOnce() + await user.keyboard('{Escape}') + await waitFor(() => + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + ) + expect(trigger).toHaveFocus() + }) + + it.for([ + { name: 'cat.png', previewUrl: 'blob:cat' }, + { name: 'song.mp3', previewUrl: undefined } + ])( + 'shows $name only on hover while retaining an accessible tray item', + async (props) => { + const user = userEvent.setup() + renderChip(props) + expect(screen.queryByText(props.name)).not.toBeInTheDocument() + await user.hover( + screen.getByRole('button', { name: `Preview ${props.name}` }) + ) + const preview = await screen.findByRole('dialog', { name: props.name }) + expect(within(preview).getByText(props.name)).toBeVisible() + } + ) + it('renders an image preview only for image files', () => { renderChip({ name: 'cat.png', previewUrl: 'blob:x' }) expect(screen.getByAltText('cat.png')).toBeInTheDocument() @@ -37,7 +220,8 @@ describe('AttachmentChip', () => { name: 'song.mp3', previewUrl: 'https://x/thumb.png' }) - expect(screen.queryByRole('img')).not.toBeInTheDocument() + expect(screen.queryByAltText('song.mp3')).not.toBeInTheDocument() + expect(screen.getByRole('img', { name: 'Audio' })).toBeInTheDocument() expect(iconMarker(container)).toContain('lucide--music') }) @@ -52,6 +236,175 @@ describe('AttachmentChip', () => { expect(iconMarker(container)).toContain(icon) }) + it('shows a video poster while keeping the playable file separate', async () => { + const user = userEvent.setup() + renderChip({ + name: 'My clip', + mediaKind: 'video', + previewUrl: '/poster.png', + mediaUrl: '/clip.mp4' + }) + expect(screen.getByRole('img', { name: 'My clip' })).toHaveAttribute( + 'src', + '/poster.png' + ) + await user.hover(screen.getByRole('button', { name: 'Preview My clip' })) + const player = await screen.findByLabelText('My clip', { + selector: 'video' + }) + expect(player).toHaveAttribute('src', '/clip.mp4') + expect(player).toHaveAttribute('poster', '/poster.png') + expect(player).toHaveAttribute('controls') + expect(player).not.toHaveAttribute('autoplay') + }) + + it('does not decode another video while waiting for the shared poster', () => { + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + expect( + screen.queryByLabelText('clip.mp4', { selector: 'video' }) + ).not.toBeInTheDocument() + expect(screen.getByRole('img', { name: 'Video' })).toBeInTheDocument() + }) + + it.for([ + { name: 'clip.mp4', mediaKind: 'video', selector: 'video' }, + { name: 'song.mp3', mediaKind: 'audio', selector: 'audio' } + ] as const)( + 'keeps $mediaKind controls usable inside hover preview and stops on close', + async ({ name, mediaKind, selector }) => { + const user = userEvent.setup() + renderChip({ + name, + mediaKind, + mediaUrl: '/playable-media', + previewUrl: '/poster.png' + }) + const trigger = screen.getByRole('button', { name: `Preview ${name}` }) + await user.hover(trigger) + const player = await screen.findByLabelText(name, { selector }) + assert.instanceOf(player, HTMLMediaElement) + expect(player).toHaveAttribute('controls') + expect(player).toHaveAttribute('src', '/playable-media') + expect(player).not.toHaveAttribute('autoplay') + await user.unhover(trigger) + await user.hover(player) + expect(screen.getByRole('region', { name })).toBeVisible() + expect(player).toBeInTheDocument() + await fireEvent.play(player) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.keyboard('{Escape}') + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + } + ) + + it('keeps hover usable after keyboard and hovered-content dismissals', async () => { + vi.useFakeTimers() + onTestFinished(() => { + vi.useRealTimers() + }) + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTimeAsync }) + renderChip({ name: 'clip.mp4', mediaUrl: '/clip.mp4' }) + const trigger = screen.getByRole('button', { name: 'Preview clip.mp4' }) + await user.tab() + await user.keyboard('{Enter}') + expect( + screen.getByLabelText('clip.mp4', { selector: 'video' }) + ).toHaveFocus() + await user.keyboard('{Escape}') + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + const first = screen.getByLabelText('clip.mp4', { selector: 'video' }) + await user.hover(first) + await vi.advanceTimersByTimeAsync(150) + await user.keyboard('{Escape}') + await vi.advanceTimersByTimeAsync(150) + expect(first).not.toBeInTheDocument() + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + const second = screen.getByLabelText('clip.mp4', { selector: 'video' }) + await user.unhover(trigger) + await user.pointer({ target: document.body }) + await vi.advanceTimersByTimeAsync(150) + expect(second).not.toBeInTheDocument() + // Escape also cancels a pointer hover whose delayed opening is still pending. + await user.hover(trigger) + await user.keyboard('{Enter}') + await user.keyboard('{Escape}') + await vi.advanceTimersByTimeAsync(250) + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() + await user.unhover(trigger) + await vi.advanceTimersByTimeAsync(150) + await user.hover(trigger) + await vi.advanceTimersByTimeAsync(250) + expect(screen.getByRole('dialog', { name: 'clip.mp4' })).toBeVisible() + }) + + it('stops audio after the pointer leaves the preview', async () => { + const user = userEvent.setup() + renderChip({ name: 'song.mp3', mediaUrl: '/song.mp3' }) + const trigger = screen.getByRole('button', { name: 'Preview song.mp3' }) + await user.hover(trigger) + const player = await screen.findByLabelText('song.mp3', { + selector: 'audio' + }) + assert.instanceOf(player, HTMLAudioElement) + player.currentTime = 1 + const pause = vi.spyOn(player, 'pause') + await user.unhover(trigger) + await user.pointer({ + target: document.body, + coords: { clientX: 500, clientY: 500 } + }) + await waitFor(() => expect(player).not.toBeInTheDocument()) + expect(pause).toHaveBeenCalledTimes(2) + expect(player.currentTime).toBe(0) + }) + + it('stops an audio preview on replacement and removal', async () => { + const user = userEvent.setup() + const { rerender, unmount } = renderChip({ + name: 'song.mp3', + mediaUrl: 'blob:song' + }) + await user.hover(screen.getByRole('button', { name: 'Preview song.mp3' })) + const first = await screen.findByLabelText('song.mp3', { + selector: 'audio' + }) + assert.instanceOf(first, HTMLAudioElement) + const pauseFirst = vi.spyOn(first, 'pause') + await rerender({ name: 'song.mp3', mediaUrl: '/stored-song.mp3' }) + const replacement = screen.getByLabelText('song.mp3', { selector: 'audio' }) + assert.instanceOf(replacement, HTMLAudioElement) + expect(replacement).toHaveAttribute('src', '/stored-song.mp3') + expect(pauseFirst).toHaveBeenCalledOnce() + unmount() + expect(pauseFirst).toHaveBeenCalledTimes(2) + }) + + it.for([ + { name: 'clip.mp4', selector: 'video', message: 'Video failed to load' }, + { name: 'song.mp3', selector: 'audio', message: 'Audio failed to load' } + ])( + 'reports an unplayable $name without losing the asset name', + async ({ name, selector, message }) => { + const user = userEvent.setup() + renderChip({ + name, + mediaUrl: '/unsupported-media', + previewUrl: '/poster.png' + }) + await user.hover(screen.getByRole('button', { name: `Preview ${name}` })) + const player = await screen.findByLabelText(name, { selector }) + await fireEvent.error(player) + expect(screen.getByText(message)).toBeVisible() + expect(screen.getByText(name)).toBeVisible() + } + ) + it('shows the spinner while uploading', () => { renderChip({ name: 'cat.png', uploading: true }) expect( diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue index 08a88c5469f..9ff5dbc4612 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue @@ -1,60 +1,82 @@ diff --git a/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue b/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue new file mode 100644 index 00000000000..1870958fa21 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/ComposerAddMenu.vue @@ -0,0 +1,166 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue b/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue new file mode 100644 index 00000000000..d24429696ae --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/ComposerPlaceholder.vue @@ -0,0 +1,55 @@ + + diff --git a/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts new file mode 100644 index 00000000000..284b4f98906 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts @@ -0,0 +1,55 @@ +import { render, screen } from '@testing-library/vue' +import { describe, expect, it } from 'vitest' + +import { i18n } from '@/i18n' + +import type { ComposerAttachment } from '../../../types/composerAttachment' +import InlineAssetReference from './InlineAssetReference.vue' + +function renderReference( + props: Pick +) { + return render(InlineAssetReference, { + props: { ...props, removeLabel: `Remove ${props.name} reference` }, + global: { plugins: [i18n] } + }) +} + +describe('inline asset media', () => { + it.for([ + { name: 'cat.png', mediaKind: 'image' }, + { name: 'My video', mediaKind: 'video' } + ] as const)( + 'shows the image/poster for $mediaKind', + ({ name, mediaKind }) => { + renderReference({ + name, + mediaKind, + previewUrl: '/poster.png' + }) + expect(screen.getByAltText('')).toHaveAttribute('src', '/poster.png') + expect(screen.getByText(name)).toBeVisible() + } + ) + + it('uses a video icon while awaiting the shared poster', () => { + renderReference({ + name: 'My video', + mediaKind: 'video' + }) + expect(screen.getByRole('img', { name: 'Video' })).toBeInTheDocument() + }) + + it.for([ + { name: 'recording', mediaKind: 'audio', label: 'Audio' }, + { name: 'song.mp3', mediaKind: undefined, label: 'Audio' }, + { name: 'My video', mediaKind: 'video', label: 'Video' } + ] as const)( + 'identifies $name by its media type when no thumbnail is available', + ({ name, mediaKind, label }) => { + renderReference({ name, mediaKind }) + expect(screen.getByRole('img', { name: label })).toBeInTheDocument() + expect(screen.getByText(name)).toBeVisible() + } + ) +}) diff --git a/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue new file mode 100644 index 00000000000..a19d0efa243 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.vue @@ -0,0 +1,64 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue b/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue index 085d51c834f..4206693df3a 100644 --- a/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue +++ b/src/workbench/extensions/agent/components/agent/composer/InlinePromptEditor.vue @@ -6,17 +6,17 @@ import { closeHistory, history, redo, undo } from '@tiptap/pm/history' import { keymap } from '@tiptap/pm/keymap' import { EditorState, TextSelection } from '@tiptap/pm/state' import { Decoration, DecorationSet, EditorView } from '@tiptap/pm/view' -import { onBeforeUnmount, onMounted, useTemplateRef, watch } from 'vue' -import DOMPurify from 'dompurify' -import { useI18n } from 'vue-i18n' - -import { buttonVariants } from '@/components/ui/button/button.variants' import { - tagRemoveButtonVariants, - tagVariants -} from '@/components/chip/tag.variants' -import { cn } from '@comfyorg/tailwind-utils' - + getCurrentInstance, + h, + onBeforeUnmount, + onMounted, + render, + useTemplateRef, + watch +} from 'vue' +import { default as DOMPurify } from 'dompurify' +import { useI18n } from 'vue-i18n' import type { ComposerPrompt } from '../../../types/composerPrompt' import { composerReferenceKey, @@ -30,6 +30,13 @@ import { import { selectedNodeKey } from '../../../composables/agent/useCanvasSelection' import type { PromptEditor } from '../../../types/promptEditor' import type { WorkflowReferenceMetadata } from '../../../types/workflowReference' +import InlineAssetReference from './InlineAssetReference.vue' +import { + inlineReferenceChipClass, + inlineReferenceRemoveAnchorClass, + inlineReferenceRemoveBadgeClass, + inlineReferenceRemoveButtonClass +} from './inlineReferenceChipStyles' import { inlinePromptSchema, promptDocument, @@ -67,10 +74,13 @@ const emit = defineEmits<{ openReferenceWorkflow: [id: string, name: string] removeWorkflowReference: [id: string] removeNodeReference: [id: string] + highlightAssets: [ids: string[]] }>() const { t } = useI18n() +const appContext = getCurrentInstance()?.appContext const host = useTemplateRef('host') let view: EditorView | undefined +const activeAssetReferences = new Map() const insertions = new Set<{ from: number; to: number }>() const plugins = [ history(), @@ -109,21 +119,93 @@ function referenceClipboardText(node: Node): string { : assetReferenceText(name) } -function passiveReferenceView(node: Node, iconClass: string) { +function assetReferenceView( + node: Node, + editor: EditorView, + getPos: () => number | undefined +) { + const dom = document.createElement('span') + const reference = promptNodeReference(node, 0) + if (reference?.kind !== 'asset') return { dom } + dom.contentEditable = 'false' + dom.className = + 'inline-flex rounded-sm align-middle ring-offset-base-background [&.ProseMirror-selectednode]:ring-2 [&.ProseMirror-selectednode]:ring-base-foreground [&.ProseMirror-selectednode]:ring-offset-1' + const vnode = h(InlineAssetReference, { + name: reference.attachment.name, + previewUrl: reference.attachment.previewUrl, + mediaKind: reference.attachment.mediaKind, + removeLabel: t('agent.removeAssetReference', { + name: reference.attachment.name + }), + onHighlight: (active: boolean) => { + if (active) activeAssetReferences.set(dom, reference.attachment.id) + else if (!activeAssetReferences.delete(dom)) return + emit('highlightAssets', [...new Set(activeAssetReferences.values())]) + }, + onRemove: () => { + const position = getPos() + if (position === undefined) return + editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() + } + }) + vnode.appContext = appContext ?? null + render(vnode, dom) + return { + dom, + stopEvent: () => true, + ignoreMutation: () => true, + destroy: () => render(null, dom) + } +} + +function nodeReferenceView( + node: Node, + editor: EditorView, + getPos: () => number | undefined +) { const dom = document.createElement('span') const reference = promptNodeReference(node, 0) - if (!reference) return { dom } + if (reference?.kind !== 'node') return { dom } dom.contentEditable = 'false' dom.dataset.testid = `${reference.kind}-reference-chip` - dom.className = cn(tagVariants(), 'align-middle') + dom.className = inlineReferenceChipClass + dom.tabIndex = 0 const icon = document.createElement('span') - icon.className = `${iconClass} size-3 shrink-0` + icon.className = 'icon-[comfy--node] size-3 shrink-0' icon.setAttribute('aria-hidden', 'true') const label = document.createElement('span') label.className = 'min-w-0 max-w-56 truncate' - label.textContent = composerReferenceName(reference) + label.textContent = reference.node.title dom.append(icon, label) - return { dom, ignoreMutation: () => true } + const id = document.createElement('span') + id.className = 'shrink-0 text-muted-foreground' + id.textContent = ` #${reference.node.id}` + const remove = document.createElement('button') + remove.type = 'button' + remove.setAttribute( + 'aria-label', + t('agent.removeNodeLabel', { node: composerReferenceName(reference) }) + ) + const removeAnchor = document.createElement('span') + removeAnchor.className = inlineReferenceRemoveAnchorClass + remove.className = inlineReferenceRemoveButtonClass + const badge = document.createElement('span') + badge.className = inlineReferenceRemoveBadgeClass + const cross = document.createElement('span') + cross.className = 'icon-[lucide--x] size-2' + cross.setAttribute('aria-hidden', 'true') + badge.append(cross) + remove.append(badge) + remove.onclick = () => { + const position = getPos() + if (position === undefined) return + editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() + } + removeAnchor.append(remove) + dom.append(id, removeAnchor) + return { dom, stopEvent: () => true, ignoreMutation: () => true } } onMounted(() => { @@ -294,8 +376,8 @@ onMounted(() => { referenceClipboardText(node) ), nodeViews: { - node: (node) => passiveReferenceView(node, 'icon-[comfy--node]'), - asset: (node) => passiveReferenceView(node, 'icon-[lucide--paperclip]'), + node: (node, editor, getPos) => nodeReferenceView(node, editor, getPos), + asset: assetReferenceView, workflow(node, editor, getPos) { const id: unknown = node.attrs.id const name: unknown = node.attrs.name @@ -303,10 +385,7 @@ onMounted(() => { if (typeof id !== 'string' || typeof name !== 'string') return { dom } dom.contentEditable = 'false' dom.dataset.testid = 'workflow-reference-chip' - dom.className = cn( - tagVariants({ interactive: true, removable: true }), - 'group/workflow align-middle' - ) + dom.className = inlineReferenceChipClass const open = document.createElement('button') open.type = 'button' open.tabIndex = 0 @@ -338,21 +417,16 @@ onMounted(() => { if (!unavailable) emit('openReferenceWorkflow', id, name) } const removeAnchor = document.createElement('span') - removeAnchor.className = 'relative inline-block h-4 w-0 align-middle' + removeAnchor.className = inlineReferenceRemoveAnchorClass const remove = document.createElement('button') remove.type = 'button' remove.setAttribute( 'aria-label', t('agent.removeWorkflowReference', { name }) ) - remove.className = cn( - buttonVariants({ variant: 'textonly', size: 'icon-sm' }), - tagRemoveButtonVariants(), - 'pointer-events-none absolute -top-2 -right-2 z-10 flex size-5 cursor-pointer items-center justify-center rounded-full p-0 text-base-foreground opacity-0 transition-opacity group-focus-within/workflow:pointer-events-auto group-focus-within/workflow:opacity-100 group-hover/workflow:pointer-events-auto group-hover/workflow:opacity-100 touch:pointer-events-auto touch:opacity-100' - ) + remove.className = inlineReferenceRemoveButtonClass const badge = document.createElement('span') - badge.className = - 'flex size-3 items-center justify-center rounded-full bg-base-background ring-1 ring-border-default hover:bg-secondary-background-hover' + badge.className = inlineReferenceRemoveBadgeClass const cross = document.createElement('span') cross.className = 'icon-[lucide--x] size-2' badge.append(cross) @@ -361,6 +435,7 @@ onMounted(() => { const position = getPos() if (position === undefined) return editor.dispatch(deleteReference(editor.state, position, node)) + editor.focus() } removeAnchor.append(remove) dom.append(open, removeAnchor) diff --git a/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue b/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue new file mode 100644 index 00000000000..dd05a61ca51 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/MentionMenuItem.vue @@ -0,0 +1,101 @@ + + + diff --git a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts index 53185944a5c..63889619692 100644 --- a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts +++ b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.test.ts @@ -72,6 +72,31 @@ describe('inline prompt', () => { } ) + it.for(['video', 'audio'] as const)( + 'retains $mediaKind preview metadata when round-tripping inline assets', + (mediaKind) => { + const draft: ComposerPrompt = { + text: 'Inspect ', + references: [ + { + kind: 'asset', + textOffset: 8, + attachment: { + id: 'media', + name: 'Renamed media', + ref: 'stored-file', + mediaKind, + mediaUrl: 'blob:media', + previewUrl: '/poster.png', + uploading: true + } + } + ] + } + expect(promptDraft(promptDocument(draft))).toEqual(draft) + } + ) + it('restores references before, between and after text, including adjacent tokens', () => { const draft: ComposerPrompt = { text: 'before 😀\nafter', diff --git a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts index b9c285f0fa0..c1b8d1567c1 100644 --- a/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts +++ b/src/workbench/extensions/agent/components/agent/composer/inlinePrompt.ts @@ -1,6 +1,7 @@ import { Schema } from '@tiptap/pm/model' import type { Node } from '@tiptap/pm/model' +import { zMediaKindSchema } from '@/platform/assets/schemas/mediaAssetSchema' import { isNodeLocatorId } from '@/types/nodeIdentification' import type { ComposerInsertionPoint, @@ -68,6 +69,8 @@ export const inlinePromptSchema = new Schema({ name: {}, ref: {}, previewUrl: { default: null }, + mediaUrl: { default: null }, + mediaKind: { default: null }, uploading: { default: false } }, toDOM: (node) => [ @@ -117,6 +120,30 @@ export function promptDocument(prompt: ComposerPrompt): Node { return inlinePromptSchema.nodes.doc.create(null, content) } +function assetNodeReference( + node: Node, + id: string, + name: string, + textOffset: number +): ComposerReference | undefined { + const { ref, previewUrl, mediaUrl, uploading } = node.attrs + const mediaKind = zMediaKindSchema.safeParse(node.attrs.mediaKind).data + if (typeof ref !== 'string') return + return { + kind: 'asset', + textOffset, + attachment: { + id, + name, + ref, + ...(typeof previewUrl === 'string' ? { previewUrl } : {}), + ...(typeof mediaUrl === 'string' ? { mediaUrl } : {}), + ...(mediaKind ? { mediaKind } : {}), + ...(uploading === true ? { uploading: true } : {}) + } + } +} + export function promptNodeReference( node: Node, textOffset: number @@ -145,21 +172,8 @@ export function promptNodeReference( } } } - if (node.type.name === 'asset') { - const { ref, previewUrl, uploading } = node.attrs - if (typeof ref !== 'string') return - return { - kind: 'asset', - textOffset, - attachment: { - id, - name, - ref, - ...(typeof previewUrl === 'string' ? { previewUrl } : {}), - ...(uploading === true ? { uploading: true } : {}) - } - } - } + if (node.type.name === 'asset') + return assetNodeReference(node, id, name, textOffset) } export function promptDraft(doc: Node): ComposerPrompt { diff --git a/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts b/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts new file mode 100644 index 00000000000..74b49ffc8e3 --- /dev/null +++ b/src/workbench/extensions/agent/components/agent/composer/inlineReferenceChipStyles.ts @@ -0,0 +1,20 @@ +import { buttonVariants } from '@/components/ui/button/button.variants' +import { + tagRemoveButtonVariants, + tagVariants +} from '@/components/chip/tag.variants' +import { cn } from '@comfyorg/tailwind-utils' + +export const inlineReferenceChipClass = cn( + tagVariants({ interactive: true, removable: true }), + 'group/inline-reference h-4 p-0 align-middle' +) +export const inlineReferenceRemoveAnchorClass = + 'relative inline-block h-4 w-0 align-middle' +export const inlineReferenceRemoveButtonClass = cn( + buttonVariants({ variant: 'textonly', size: 'icon-sm' }), + tagRemoveButtonVariants(), + 'pointer-events-none absolute -top-2 -right-2 z-10 flex size-5 cursor-pointer items-center justify-center rounded-full p-0 text-base-foreground opacity-0 transition-opacity group-focus-within/inline-reference:pointer-events-auto group-focus-within/inline-reference:opacity-100 group-hover/inline-reference:pointer-events-auto group-hover/inline-reference:opacity-100 touch:pointer-events-auto touch:opacity-100' +) +export const inlineReferenceRemoveBadgeClass = + 'flex size-3 items-center justify-center rounded-full bg-base-background ring-1 ring-border-default hover:bg-secondary-background-hover' diff --git a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts index 8208a272d9e..ec51447c691 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.test.ts @@ -6,12 +6,12 @@ import type { ComfyWorkflow } from '@/platform/workflow/management/stores/comfyW import { createMockLoadedWorkflow } from '@/utils/__tests__/litegraphTestUtils' import type { WorkflowReference } from '../../types/workflowReference' +import type { ComposerAttachment } from '../../types/composerAttachment' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' import type { SubmissionMeta } from './useAgentDraftSubmission' import { useAgentDraftSubmission } from './useAgentDraftSubmission' import type { SelectedNode } from './useCanvasSelection' import { useCanvasSelection } from './useCanvasSelection' -import type { ComposerAttachment } from './useComposer' type ComposerStore = ReturnType type Send = ( @@ -196,6 +196,37 @@ describe('Agent draft submission', () => { expect(revoke).not.toHaveBeenCalled() }) + it('retains media through a pending send and failed recovery, releasing it on success', async () => { + const { composer, submit, send, pending } = setup() + const revoke = vi.spyOn(URL, 'revokeObjectURL') + composer.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: 'clip.mp4', + mediaKind: 'video', + mediaUrl: 'blob:video' + }) + composer.referenceAttachment('video') + const sending = submit() + composer.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + pending.resolve(false) + await sending + await nextTick() + expect(composer.attachments).toContainEqual({ + id: 'video', + name: 'clip.mp4', + ref: 'clip.mp4', + mediaKind: 'video', + mediaUrl: 'blob:video' + }) + expect(revoke).not.toHaveBeenCalled() + send.mockResolvedValue(true) + await submit() + expect(composer.attachments).toEqual([]) + expect(revoke.mock.calls).toEqual([['blob:cat'], ['blob:video']]) + }) + it('does not recover over an edit that was subsequently cleared', async () => { const { composer, selection, submit, pending } = setup() const sending = submit() @@ -210,6 +241,42 @@ describe('Agent draft submission', () => { expect(selection.staged.value).toEqual([]) }) + it('restores repeated mentions and unmentioned assets with their original inline positions after failure', async () => { + const { composer, submit, send } = setup() + composer.referenceAttachment('upload-1') + composer.referenceAttachment('upload-1') + composer.addAttachment({ + id: 'upload-2', + name: 'background.png', + ref: 'background.png' + }) + const prompt = composer.prompt + const attachments = [...composer.attachments] + send.mockResolvedValue(false) + + await submit() + + expect(composer.prompt).toEqual(prompt) + expect(composer.attachments).toEqual(attachments) + expect( + composer.prompt.references.filter((item) => item.kind === 'asset') + ).toHaveLength(2) + }) + + it('keeps a newer tray attachment instead of restoring a failed submission', async () => { + const { composer, submit, pending } = setup() + const sending = submit() + composer.addAttachment({ id: 'new', name: 'new.png', ref: 'new.png' }) + pending.resolve(false) + + await sending + + expect(composer.prompt).toEqual({ text: '', references: [] }) + expect(composer.attachments).toEqual([ + { id: 'new', name: 'new.png', ref: 'new.png' } + ]) + }) + it.for(['pending', 'failed'])( 'keeps edits made while the panel is closed when the submission is %s', async (phase) => { diff --git a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts index 0860b2fbcf5..8707e962b35 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentDraftSubmission.ts @@ -9,7 +9,7 @@ import type { WorkflowReference } from '../../types/workflowReference' import type { AgentStarterPromptSource } from '../../utils/starterPrompts' import type { SelectedNode, useCanvasSelection } from './useCanvasSelection' import { selectedNodeKey } from './useCanvasSelection' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' interface UseAgentDraftSubmissionOptions { canSubmit: () => boolean @@ -59,22 +59,25 @@ export function useAgentDraftSubmission( const snapshot = composer.takeFailedSubmission() if (!snapshot || selection.staged.value.length > 0) return - composer.restorePrompt({ - text: snapshot.prompt.text, - references: snapshot.prompt.references.filter((reference) => { - if (reference.kind === 'workflow') - return reference.id !== options.editableWorkflowId() - if (reference.kind === 'node') - return ( - options.target() === snapshot.target && - snapshot.nodes.some( - (node) => - selectedNodeKey(node) === selectedNodeKey(reference.node) + composer.restorePrompt( + { + text: snapshot.prompt.text, + references: snapshot.prompt.references.filter((reference) => { + if (reference.kind === 'workflow') + return reference.id !== options.editableWorkflowId() + if (reference.kind === 'node') + return ( + options.target() === snapshot.target && + snapshot.nodes.some( + (node) => + selectedNodeKey(node) === selectedNodeKey(reference.node) + ) ) - ) - return true - }) - }) + return true + }) + }, + snapshot.attachments + ) if (options.target() === snapshot.target) selection.replace(snapshot.nodes) } diff --git a/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts b/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts index 02f111cca44..5492f1e677f 100644 --- a/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts +++ b/src/workbench/extensions/agent/composables/agent/useAgentMentionPicker.ts @@ -11,12 +11,14 @@ import type { import { transitionMentionPicker } from './mentionPickerState' import type { SelectedNode } from './useCanvasSelection' import { selectedNodeKey } from './useCanvasSelection' +import type { ComposerAttachment } from '../../types/composerAttachment' interface MentionPickerOptions { draft: () => string editor: () => PromptEditor | null selectionTags: () => SelectedNode[] workflows: () => WorkflowReferenceOption[] + assets: () => ComposerAttachment[] nodeReferenceDisabledReason: () => string | undefined workflowSelecting: () => boolean getMentionNodes: () => SelectedNode[] @@ -26,21 +28,19 @@ interface MentionPickerOptions { to: number ) => Promise pickNode: (node: SelectedNode) => void + pickAsset: (asset: ComposerAttachment) => void requestWorkflows: () => void } export function useAgentMentionPicker(options: MentionPickerOptions) { const { t } = useI18n() - const graphNodes = ref([]) - function loadMentionNodes(): void { - if (options.nodeReferenceDisabledReason()) { - graphNodes.value = [] - return - } - graphNodes.value = [...options.getMentionNodes()].sort((a, b) => - a.title.localeCompare(b.title) - ) - } + const graphNodes = computed(() => + options.nodeReferenceDisabledReason() + ? [] + : [...options.getMentionNodes()].sort((a, b) => + a.title.localeCompare(b.title) + ) + ) const mention = ref({ status: 'closed' }) const mentionSection = computed(() => @@ -57,6 +57,7 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { | { kind: 'section'; id: 'nodes' | 'workflows'; label: string } | { kind: 'back'; id: 'back'; label: string } | { kind: 'node'; id: string; label: string; node: SelectedNode } + | { kind: 'asset'; id: string; label: string; asset: ComposerAttachment } | { kind: 'workflow' id: string @@ -78,9 +79,20 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { { kind: 'section', id: 'nodes', label: t('agent.nodes') }, { kind: 'section', id: 'workflows', label: t('agent.workflows') } ] - return sections.filter(({ label }) => - label.toLowerCase().includes(search) - ) + return [ + ...options + .assets() + .filter((asset) => asset.name.toLowerCase().includes(search)) + .map( + (asset): MentionMatch => ({ + kind: 'asset', + id: asset.id, + label: asset.name, + asset + }) + ), + ...sections.filter(({ label }) => label.toLowerCase().includes(search)) + ] } const back: MentionMatch = { kind: 'back', id: 'back', label: t('g.back') } @@ -149,15 +161,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { } const graphDupes = computed(() => duplicatedTitles(graphNodes.value)) - const tagDupes = computed(() => duplicatedTitles(options.selectionTags())) - - watch( - () => options.selectionTags(), - (tags) => { - if (tags.length) loadMentionNodes() - }, - { immediate: true } - ) function dispatchMention(event: MentionPickerEvent): void { mention.value = transitionMentionPicker(mention.value, event) @@ -187,7 +190,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { dispatchMention({ type: 'closed' }) return } - if (mention.value.status === 'closed') loadMentionNodes() dispatchMention({ type: 'queryChanged', start: at, @@ -207,11 +209,7 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { watch( () => options.nodeReferenceDisabledReason(), (reason) => { - if (!reason) { - if (mention.value.status === 'open') loadMentionNodes() - return - } - graphNodes.value = [] + if (!reason) return if ( mention.value.status === 'open' && mention.value.section === 'nodes' @@ -225,15 +223,25 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { { flush: 'sync' } ) + function selectMentionSection( + section: Exclude, + from: number, + to: number + ): void { + options.editor()?.replaceText(from, to, '') + dispatchMention({ type: 'sectionSelected', section }) + if (section === 'workflows') options.requestWorkflows() + } + async function pickMention(match: MentionMatch): Promise { const state = mention.value if (state.status === 'closed' || isMentionDisabled(match)) return if (match.kind === 'section') { - options - .editor() - ?.replaceText(state.start + 1, state.start + 1 + state.query.length, '') - dispatchMention({ type: 'sectionSelected', section: match.id }) - if (match.id === 'workflows') options.requestWorkflows() + selectMentionSection( + match.id, + state.start + 1, + state.start + 1 + state.query.length + ) return } if (match.kind === 'back') { @@ -254,7 +262,8 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { options .editor() ?.replaceText(state.start, state.start + 1 + state.query.length, '') - options.pickNode(match.node) + if (match.kind === 'asset') options.pickAsset(match.asset) + else options.pickNode(match.node) dispatchMention({ type: 'closed' }) options.editor()?.focus() } @@ -314,7 +323,6 @@ export function useAgentMentionPicker(options: MentionPickerOptions) { mentionVisible, mentionHasResults, graphDupes, - tagDupes, syncMention, pickMention, isNodeReferenceDisabled, diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts index a5c2c552518..7b4b9d1c7c4 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.test.ts @@ -5,7 +5,8 @@ import { AgentApiError, AgentResponseUnreadableError } from '../../services/agent/agentRestClient' -import type { ComposerAttachment } from './useComposer' +import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' +import type { ComposerAttachment } from '../../types/composerAttachment' import { MAX_ATTACHMENT_BATCH_SIZE, MAX_ATTACHMENT_BYTES, @@ -26,7 +27,10 @@ function chipRegistry() { const chips: ComposerAttachment[] = [] return { chips, - stage: (attachment: ComposerAttachment) => chips.push(attachment), + stage: (attachment: ComposerAttachment) => { + chips.push(attachment) + return true + }, update: (id: string, patch: Partial) => { const index = chips.findIndex((chip) => chip.id === id) if (index >= 0) chips[index] = { ...chips[index], ...patch } @@ -39,9 +43,175 @@ function chipRegistry() { } describe('useAttachment', () => { - it('adds previews for picked images but not picked videos', async () => { - // A video object URL in an renders as a broken thumbnail, so only - // images get a previewUrl. + it('deduplicates a file within a batch and across pending drops', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + const batch = addFiles([file, file, file]) + expect(onDuplicate).toHaveBeenCalledExactlyOnceWith(['cat.png', 'cat.png']) + const repeated = addFiles([ + new File(['image'], 'cat.png', { lastModified: 1 }) + ]) + expect(store.attachments).toHaveLength(1) + await expect(repeated).resolves.toBe(false) + await expect(batch).resolves.toBe(true) + expect(upload).toHaveBeenCalledOnce() + expect(onDuplicate).toHaveBeenLastCalledWith(['cat.png']) + expect(onDuplicate).toHaveBeenCalledTimes(2) + }) + + it('ignores a settled duplicate but allows a removed file to be added again', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + await addFiles([file]) + expect(onDuplicate).not.toHaveBeenCalled() + const original = store.attachments[0] + store.referenceAttachment(original.id) + store.referenceAttachment(original.id) + await expect(addFiles([file])).resolves.toBe(false) + expect(store.attachments).toEqual([original]) + expect(store.prompt.references).toHaveLength(2) + store.removeAttachment(original.id) + await expect(addFiles([file])).resolves.toBe(true) + expect(store.attachments).toHaveLength(1) + expect(store.attachments[0].id).not.toBe(original.id) + expect(upload).toHaveBeenCalledTimes(2) + expect(onDuplicate).toHaveBeenCalledExactlyOnceWith(['cat.png']) + }) + + it.for([ + { body: 'different size', lastModified: 1 }, + { body: 'image', lastModified: 2 } + ])( + 'keeps same-named files with distinct source metadata: %j', + async (source) => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const { addFiles } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + await addFiles([ + new File(['image'], 'cat.png', { lastModified: 1 }), + new File([source.body], 'cat.png', { + lastModified: source.lastModified + }) + ]) + expect(store.attachments.map(({ name }) => name)).toEqual([ + 'cat.png', + 'cat.png' + ]) + expect(upload).toHaveBeenCalledTimes(2) + } + ) + + it('allows a failed file upload to be retried', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + upload.mockRejectedValueOnce(new Error('Upload failed')) + const onDuplicate = vi.fn() + const { addFiles } = useAttachment({ + upload, + onDuplicate, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + await expect(addFiles([file])).resolves.toBe(false) + await expect(addFiles([file])).resolves.toBe(true) + expect(store.attachments).toHaveLength(1) + expect(upload).toHaveBeenCalledTimes(2) + expect(onDuplicate).not.toHaveBeenCalled() + }) + + it('keeps a fresh re-addition after a cancelled upload settles late', async () => { + const store = useAgentComposerStore() + let resolveOldUpload: (result: { ref: string }) => void = () => {} + const oldUpload = new Promise<{ ref: string }>((resolve) => { + resolveOldUpload = resolve + }) + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + upload.mockReturnValueOnce(oldUpload) + const { addFiles, cancelUpload } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const file = new File(['image'], 'cat.png', { lastModified: 1 }) + const pending = addFiles([file]) + const oldId = store.attachments[0].id + cancelUpload(oldId) + await expect(addFiles([file])).resolves.toBe(true) + const replacement = store.attachments[0] + expect(replacement.id).not.toBe(oldId) + resolveOldUpload({ ref: 'obsolete.png' }) + await pending + expect(store.attachments).toEqual([replacement]) + expect(replacement.ref).toBe('cat.png') + }) + + it('keeps same-named deferred files from distinct source URIs', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const resolve = async () => new File(['image'], 'cat.png') + const { addDeferredFile } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + await Promise.all([ + addDeferredFile('cat.png', resolve, 'uri:/first'), + addDeferredFile('cat.png', resolve, 'uri:/second') + ]) + expect(store.attachments).toHaveLength(2) + expect(upload).toHaveBeenCalledTimes(2) + }) + + it('deduplicates a deferred source while loading and after upload', async () => { + const store = useAgentComposerStore() + const upload = vi.fn(async (file: File) => ({ ref: file.name })) + const resolve = vi.fn(async () => new File(['image'], 'cat.png')) + const { addDeferredFile } = useAttachment({ + upload, + stage: store.addAttachment, + update: store.updateAttachment, + remove: store.removeAttachment + }) + const pending = addDeferredFile('cat.png', resolve, 'uri:/source') + const repeated = addDeferredFile('cat.png', resolve, 'uri:/source') + expect(store.attachments).toHaveLength(1) + await expect(repeated).resolves.toBe('duplicate') + await expect(pending).resolves.toBe('uploaded') + await expect( + addDeferredFile('cat.png', resolve, 'uri:/source') + ).resolves.toBe('duplicate') + expect(resolve).toHaveBeenCalledOnce() + expect(upload).toHaveBeenCalledOnce() + }) + + it('keeps image preview URLs separate from picked video sources', async () => { const upload = vi.fn(async (file: File) => ({ ref: file.name })) const registry = chipRegistry() const { addFiles } = useAttachment({ upload, ...registry }) @@ -58,6 +228,45 @@ describe('useAttachment', () => { expect(upload).toHaveBeenCalledTimes(2) }) + it.for([ + { name: 'clip.mp4', type: 'video/mp4', mediaKind: 'video' }, + { name: 'recording', type: 'audio/mpeg', mediaKind: 'audio' } + ])( + 'previews picked $mediaKind while uploading and then uses its server file', + async ({ name, type, mediaKind }) => { + let resolveUpload: (result: { + ref: string + url: string + }) => void = () => {} + const registry = chipRegistry() + const { addFiles } = useAttachment({ + ...registry, + upload: () => + new Promise((resolve) => { + resolveUpload = resolve + }) + }) + const pending = addFiles([new File(['media'], name, { type })]) + expect(registry.chips[0]).toMatchObject({ + mediaKind, + mediaUrl: expect.stringMatching(/^blob:/), + uploading: true + }) + expect(registry.chips[0].previewUrl).toBeUndefined() + resolveUpload({ + ref: 'stored-media', + url: '/api/view?filename=stored-media&type=input' + }) + await pending + expect(registry.chips[0]).toMatchObject({ + mediaKind, + mediaUrl: '/api/view?filename=stored-media&type=input', + uploading: false + }) + expect(registry.chips[0].previewUrl).toBeUndefined() + } + ) + it('rejects files over 20MB before staging or uploading', async () => { const upload = vi.fn() const onError = vi.fn() diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts index 3ddbf62a4b1..b93b380bb65 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.transport.test.ts @@ -2,7 +2,7 @@ import { assert, expect, it, onTestFinished, vi } from 'vitest' import { createAgentRestClient } from '../../services/agent/agentRestClient' import { useAttachment } from './useAttachment' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' it('keeps a 4 MiB upload alive past 60 seconds and aborts it at 124 seconds', async () => { vi.useFakeTimers() @@ -34,7 +34,10 @@ it('keeps a 4 MiB upload alive past 60 seconds and aborts it at 124 seconds', as ) return { ref: result.name ?? file.name } }, - stage: (chip) => chips.set(chip.id, chip), + stage: (chip) => { + chips.set(chip.id, chip) + return true + }, update: (id, patch) => { const chip = chips.get(id) assert.exists(chip) diff --git a/src/workbench/extensions/agent/composables/agent/useAttachment.ts b/src/workbench/extensions/agent/composables/agent/useAttachment.ts index 1f76ce6047e..5902824d247 100644 --- a/src/workbench/extensions/agent/composables/agent/useAttachment.ts +++ b/src/workbench/extensions/agent/composables/agent/useAttachment.ts @@ -1,12 +1,18 @@ import { i18n } from '@/i18n' import { reportError } from '@/platform/telemetry/reportError' -import { DroppedAssetTooLargeError, hasImageType } from '@/utils/eventUtils' -import { formatSize } from '@/utils/formatUtil' +import { + DroppedAssetTooLargeError, + hasAudioType, + hasImageType, + hasVideoType +} from '@/utils/eventUtils' +import { formatSize, getMediaTypeFromFilename } from '@/utils/formatUtil' +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' import { AgentApiError, AgentResponseUnreadableError } from '../../services/agent/agentRestClient' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024 const UPLOAD_HANDSHAKE_TIMEOUT_MS = 60 * 1000 @@ -31,13 +37,21 @@ interface UploadResult { uploadType?: string } +type DeferredFileResult = + | 'uploaded' + | 'unsupported' + | 'cancelled' + | 'failed' + | 'duplicate' + export interface UseAttachmentOptions { upload: (file: File, signal: AbortSignal) => Promise uploadTimeoutMs?: number maxBytes?: (file: File) => number onError?: (message: string) => void onUploaded?: () => void - stage: (attachment: ComposerAttachment) => void + onDuplicate?: (names: string[]) => void + stage: (attachment: ComposerAttachment) => boolean update: (id: string, patch: Partial) => void remove: (id: string) => void isPresent?: (id: string) => boolean @@ -77,6 +91,36 @@ async function withDeadline( let stagedCount = 0 +function attachmentMediaKind(file: File): MediaKind { + if (hasImageType(file)) return 'image' + if (hasVideoType(file)) return 'video' + if (hasAudioType(file)) return 'audio' + return getMediaTypeFromFilename(file.name) +} + +function localPreview( + file: File, + kind: MediaKind +): Pick { + const playable = kind === 'video' || kind === 'audio' + const url = + kind === 'image' || playable ? URL.createObjectURL(file) : undefined + return { + previewUrl: kind === 'image' ? url : undefined, + mediaUrl: playable ? url : undefined + } +} + +function uploadedPreview( + kind: MediaKind, + url?: string +): Partial { + if (!url) return {} + if (kind === 'audio' || kind === 'video') return { mediaUrl: url } + if (kind === 'image') return { previewUrl: url } + return {} +} + export function useAttachment(options: UseAttachmentOptions) { const pending = new Set() const inFlight = new Map() @@ -84,10 +128,11 @@ export function useAttachment(options: UseAttachmentOptions) { const waiting: Array<() => void> = [] let activeUploads = 0 - function stage(name: string): string { + function stage(name: string, sourceKey?: string): string | undefined { const id = `upload-${++stagedCount}:${name}` + if (!options.stage({ id, name, ref: '', uploading: true, sourceKey })) + return undefined pending.add(id) - options.stage({ id, name, ref: '', uploading: true }) return id } @@ -184,9 +229,11 @@ export function useAttachment(options: UseAttachmentOptions) { if (slot) await slot try { if (cancelled.has(id)) return 'cancelled' + const mediaKind = attachmentMediaKind(file) options.update(id, { name: file.name, - previewUrl: hasImageType(file) ? URL.createObjectURL(file) : undefined + mediaKind, + ...localPreview(file, mediaKind) }) const controller = new AbortController() inFlight.set(id, controller) @@ -197,7 +244,7 @@ export function useAttachment(options: UseAttachmentOptions) { ) options.update(id, { ref: result.ref, - ...(result.url ? { previewUrl: result.url } : {}), + ...uploadedPreview(mediaKind, result.url), ...(result.subfolder ? { subfolder: result.subfolder } : {}), ...(result.uploadType ? { uploadType: result.uploadType } : {}), uploading: false @@ -227,23 +274,14 @@ export function useAttachment(options: UseAttachmentOptions) { for (const id of [...pending]) cancelUpload(id) } - async function addDeferredFile( + async function uploadDeferredFile( + id: string, name: string, resolve: ( signal: AbortSignal, maxBytes: number ) => Promise - ): Promise<'uploaded' | 'unsupported' | 'cancelled' | 'failed'> { - if (pending.size >= MAX_ATTACHMENT_BATCH_SIZE) { - options.onError?.( - i18n.global.t('agent.attachmentBatchLimit', { - count: 1, - limit: MAX_ATTACHMENT_BATCH_SIZE - }) - ) - return 'failed' - } - const id = stage(name) + ): Promise { const slot = acquireUploadSlot() if (slot) await slot try { @@ -296,7 +334,32 @@ export function useAttachment(options: UseAttachmentOptions) { } } - async function addFiles(files: Iterable): Promise { + async function addDeferredFile( + name: string, + resolve: ( + signal: AbortSignal, + maxBytes: number + ) => Promise, + sourceKey?: string + ): Promise { + if (pending.size >= MAX_ATTACHMENT_BATCH_SIZE) { + options.onError?.( + i18n.global.t('agent.attachmentBatchLimit', { + count: 1, + limit: MAX_ATTACHMENT_BATCH_SIZE + }) + ) + return 'failed' + } + const id = stage(name, sourceKey) + if (!id) { + options.onDuplicate?.([name]) + return 'duplicate' + } + return uploadDeferredFile(id, name, resolve) + } + + async function addFiles(files: Iterable): Promise { const candidates = [...files] const accepted: File[] = [] let oversized = 0 @@ -323,10 +386,8 @@ export function useAttachment(options: UseAttachmentOptions) { ) } const availableSlots = Math.max(0, MAX_ATTACHMENT_BATCH_SIZE - pending.size) - const staged = accepted - .slice(0, availableSlots) - .map((file) => ({ file, id: stage(file.name) })) - const omitted = accepted.length - staged.length + const admitted = accepted.slice(0, availableSlots) + const omitted = accepted.length - admitted.length if (omitted > 0) options.onError?.( i18n.global.t('agent.attachmentBatchLimit', { @@ -334,12 +395,24 @@ export function useAttachment(options: UseAttachmentOptions) { limit: MAX_ATTACHMENT_BATCH_SIZE }) ) + const duplicates: string[] = [] + const staged = admitted.flatMap((file) => { + const sourceKey = `file:${JSON.stringify([file.name, file.size, file.lastModified, file.type])}` + const id = stage(file.name, sourceKey) + if (!id) duplicates.push(file.name) + return id ? [{ file, id }] : [] + }) + if (duplicates.length) options.onDuplicate?.(duplicates) + let uploaded = 0 await Promise.all( staged.map(async ({ id, file }) => { - if ((await uploadStagedFile(id, file)) === 'uploaded') + if ((await uploadStagedFile(id, file)) === 'uploaded') { + uploaded += 1 options.onUploaded?.() + } }) ) + return uploaded > 0 } return { addDeferredFile, addFiles, cancelUpload, cancelAllUploads } diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts index 93c5546028f..6380fd0b19e 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.test.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.test.ts @@ -4,7 +4,7 @@ import { useTelemetry } from '@/platform/telemetry' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' -import type { ComposerAttachment } from './useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import { useComposer } from './useComposer' vi.mock(import('@/platform/telemetry')) @@ -45,10 +45,8 @@ describe('useComposer', () => { composer.submit() - expect(onSend).toHaveBeenCalledWith('make a cat @[Image: cat.png]', [ - attachment - ]) - expect(composer.draft.value).toBe(' make a cat ') + expect(onSend).toHaveBeenCalledWith('make a cat', [attachment]) + expect(composer.draft.value).toBe(' make a cat ') expect(composer.attachments.value).toEqual([attachment]) }) @@ -105,7 +103,7 @@ describe('useComposer', () => { expect(composer.canSend.value).toBe(true) composer.submit() - expect(onSend).toHaveBeenCalledWith('@[Image: cat.png]', [ + expect(onSend).toHaveBeenCalledWith('', [ { id: 'a1', name: 'cat.png', ref: 'r' } ]) }) @@ -233,14 +231,14 @@ describe('useComposer', () => { first.addAttachment({ id: 'a1', name: 'cat.png', ref: 'r' }) const { composer: second, onSend } = setup() - expect(second.draft.value).toBe('still here ') + expect(second.draft.value).toBe('still here') expect(second.attachments.value.map((a) => a.id)).toEqual(['a1']) second.submit() - expect(onSend).toHaveBeenCalledWith('still here@[Image: cat.png]', [ + expect(onSend).toHaveBeenCalledWith('still here', [ { id: 'a1', name: 'cat.png', ref: 'r' } ]) - expect(first.draft.value).toBe('still here ') + expect(first.draft.value).toBe('still here') expect(first.attachments.value.map((attachment) => attachment.id)).toEqual([ 'a1' ]) diff --git a/src/workbench/extensions/agent/composables/agent/useComposer.ts b/src/workbench/extensions/agent/composables/agent/useComposer.ts index 4249d474ed6..407c582fd84 100644 --- a/src/workbench/extensions/agent/composables/agent/useComposer.ts +++ b/src/workbench/extensions/agent/composables/agent/useComposer.ts @@ -1,3 +1,4 @@ +import type { ComposerAttachment } from '../../types/composerAttachment' import { storeToRefs } from 'pinia' import { computed, getCurrentScope, onScopeDispose } from 'vue' import { v4 as uuidv4 } from 'uuid' @@ -8,16 +9,6 @@ import { composerPromptForSend } from '../../utils/composerPrompt' import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' import { useAgentComposerStore } from '../../stores/agent/agentComposerStore' -export interface ComposerAttachment { - id: string - name: string - ref: string - previewUrl?: string - subfolder?: string - uploadType?: string - uploading?: boolean -} - export interface UseComposerOptions { onSend: (text: string, attachments: ComposerAttachment[]) => void isRunning: () => boolean @@ -32,7 +23,9 @@ export function useComposer(options: UseComposerOptions) { const canSend = computed( () => - (draft.value.trim().length > 0 || prompt.value.references.length > 0) && + (draft.value.trim().length > 0 || + prompt.value.references.length > 0 || + attachments.value.length > 0) && !attachments.value.some((item) => item.uploading) ) @@ -60,7 +53,9 @@ export function useComposer(options: UseComposerOptions) { starterPrompt?: AgentStarterPromptAttribution ): void { const draftWasEmpty = - !draft.value.trim() && prompt.value.references.length === 0 + !draft.value.trim() && + prompt.value.references.length === 0 && + attachments.value.length === 0 store.setText(draft.value ? `${draft.value} ${text}` : text) if (!starterPrompt) { store.markSuggestedPrompt() @@ -94,6 +89,7 @@ export function useComposer(options: UseComposerOptions) { setText: store.setText, replacePrompt: store.replacePrompt, addAttachment: store.addAttachment, + referenceAttachment: store.referenceAttachment, updateAttachment: store.updateAttachment, removeAttachment: store.removeAttachment } diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts index ccb4fd8723b..db0a84f416e 100644 --- a/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.test.ts @@ -5,7 +5,41 @@ import { createMockLoadedWorkflow } from '@/utils/__tests__/litegraphTestUtils' import { useAgentComposerStore } from './agentComposerStore' describe('composer reference ownership', () => { - it('updates detached uploads without attaching them until an explicit Undo', () => { + it('releases invalidated snapshot previews while retaining the current tray and ignoring repeated invalidation and stale settlement', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + const retained = { + id: 'retained', + name: 'retained.png', + ref: 'retained.png', + previewUrl: 'blob:retained' + } + store.addAttachment({ + id: 'old', + name: 'old.png', + ref: 'old.png', + previewUrl: 'blob:old' + }) + store.addAttachment(retained) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + store.restorePrompt({ text: 'New draft', references: [] }, [retained]) + store.invalidateSubmission() + store.invalidateSubmission() + store.settleSubmission(id, false) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:old') + expect(store.attachments).toEqual([retained]) + expect(store.draft).toBe('New draft') + expect(store.takeFailedSubmission()).toBeUndefined() + expect(store.referenceAttachment('old')).toBe(false) + expect(store.referenceAttachment('retained')).toBe(true) + }) + + it('keeps every asset occurrence and updates all of them after upload', () => { const store = useAgentComposerStore() store.addAttachment({ id: 'image', @@ -13,10 +47,43 @@ describe('composer reference ownership', () => { ref: '', uploading: true }) + store.referenceAttachment('image') + store.referenceAttachment('image') + const snapshot = store.prompt + expect(snapshot.references).toHaveLength(2) + + store.updateAttachment('image', { ref: 'uploaded.png', uploading: false }) + store.applyEditorPrompt(snapshot) + expect(store.prompt.references).toEqual( + snapshot.references.map((reference) => ({ + ...reference, + attachment: { + id: 'image', + name: 'source.png', + ref: 'uploaded.png', + uploading: false + } + })) + ) + expect(store.attachments).toHaveLength(1) + }) + + it('updates a tray upload without reinserting its deleted inline mention', () => { + const store = useAgentComposerStore() + store.addAttachment({ + id: 'image', + name: 'source.png', + ref: '', + uploading: true + }) + store.referenceAttachment('image') const snapshot = store.prompt store.removeReference('asset:image') store.updateAttachment('image', { ref: 'uploaded.png', uploading: false }) - expect(store.attachments).toEqual([]) + expect(store.prompt.references).toEqual([]) + expect(store.attachments).toEqual([ + { id: 'image', name: 'source.png', ref: 'uploaded.png', uploading: false } + ]) store.applyEditorPrompt(snapshot) expect(store.attachments).toEqual([ { id: 'image', name: 'source.png', ref: 'uploaded.png', uploading: false } @@ -34,6 +101,7 @@ describe('composer reference ownership', () => { uploading: true, previewUrl: 'blob:source' }) + store.referenceAttachment('image') const snapshot = store.prompt store.removeAttachment('image') store.updateAttachment('image', { ref: 'too-late.png', uploading: false }) @@ -42,7 +110,7 @@ describe('composer reference ownership', () => { expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:source') }) - it('retains previews for Undo, then releases only unused previews on unmount', () => { + it('retains included previews after inline deletion and unmount', () => { const store = useAgentComposerStore() const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) store.addAttachment({ @@ -57,11 +125,83 @@ describe('composer reference ownership', () => { ref: 'second.png', previewUrl: 'blob:second' }) + store.referenceAttachment('first') store.removeReference('asset:first') expect(revoke).not.toHaveBeenCalled() store.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + expect(store.attachments.map(({ id }) => id)).toEqual(['first', 'second']) + store.removeAttachment('first') expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:first') - expect(store.attachments.map(({ id }) => id)).toEqual(['second']) + }) + + it('releases excluded previews when replacing the tray and rejects late updates', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'old', + name: 'old.png', + ref: 'old.png', + previewUrl: 'blob:old' + }) + store.referenceAttachment('old') + const oldPrompt = store.prompt + store.replaceDraft({ + text: 'New draft', + workflowReferences: [], + attachments: [ + { id: 'new', name: 'new.png', ref: 'new.png', previewUrl: 'blob:new' } + ] + }) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:old') + store.updateAttachment('old', { previewUrl: 'blob:late' }) + expect(revoke).toHaveBeenLastCalledWith('blob:late') + store.applyEditorPrompt({ ...oldPrompt, text: 'Keep typing' }) + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(store.attachments.map(({ id }) => id)).toEqual(['new']) + expect(revoke).not.toHaveBeenCalledWith('blob:new') + }) + + it('retains included media until replacement/removal and rejects late object URLs', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: '', + mediaUrl: 'blob:video' + }) + store.referenceAttachment('video') + store.removeReference('asset:video') + store.releaseUnusedAssets() + expect(revoke).not.toHaveBeenCalled() + store.updateAttachment('video', { mediaUrl: '/clip.mp4', ref: 'clip.mp4' }) + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:video') + store.updateAttachment('video', { mediaUrl: 'blob:replacement' }) + store.removeAttachment('video') + store.updateAttachment('video', { mediaUrl: 'blob:late' }) + expect(revoke.mock.calls).toEqual([ + ['blob:video'], + ['blob:replacement'], + ['blob:late'] + ]) + expect(store.attachments).toEqual([]) + }) + + it('releases a shared blob only after both its image and media uses end', () => { + const store = useAgentComposerStore() + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + store.addAttachment({ + id: 'video', + name: 'clip.mp4', + ref: '', + previewUrl: 'blob:shared', + mediaUrl: 'blob:shared' + }) + store.updateAttachment('video', { previewUrl: undefined }) + expect(revoke).not.toHaveBeenCalled() + store.removeAttachment('video') + expect(revoke).toHaveBeenCalledExactlyOnceWith('blob:shared') }) it('rejects node history from a different target even when node IDs collide', () => { diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts new file mode 100644 index 00000000000..d2dc8c126bc --- /dev/null +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts @@ -0,0 +1,315 @@ +import { assert, beforeEach, describe, expect, it, vi } from 'vitest' + +import { createMockLoadedWorkflow } from '@/utils/__tests__/litegraphTestUtils' + +import type { ComposerAttachment } from '../../types/composerAttachment' +import * as videoThumbnail from '../../utils/videoThumbnail' +import { useAgentComposerStore } from './agentComposerStore' + +const video: ComposerAttachment = { + id: 'clip', + name: 'My video', + ref: '/clip.mp4', + mediaKind: 'video', + mediaUrl: '/clip.mp4' +} + +type ComposerStore = ReturnType + +function pendingThumbnail() { + let resolve!: (value: string | undefined) => void + const promise = new Promise((complete) => { + resolve = complete + }) + const result = { promise, resolve } + vi.mocked(videoThumbnail.createVideoThumbnail).mockReturnValueOnce( + result.promise + ) + return result +} + +beforeEach(() => { + vi.spyOn(videoThumbnail, 'createVideoThumbnail').mockResolvedValue(undefined) + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) +}) + +describe('composer shared video thumbnail lifetime', () => { + it('cancels an invalidated send capture and rejects its late poster while a new draft remains usable', async () => { + const obsolete = pendingThumbnail() + const current = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:old-video' }) + const obsoleteSignal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[0][1] + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + store.addAttachment({ + ...video, + id: 'new-video', + mediaUrl: 'blob:new-video' + }) + const currentSignal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[1][1] + store.setText('New chat draft') + store.invalidateSubmission() + expect(obsoleteSignal.aborted).toBe(true) + expect(currentSignal.aborted).toBe(false) + expect(URL.revokeObjectURL).toHaveBeenCalledExactlyOnceWith( + 'blob:old-video' + ) + obsolete.resolve('blob:late-poster') + current.resolve('blob:current-poster') + await Promise.all([obsolete.promise, current.promise]) + store.settleSubmission(id, false) + expect(store.takeFailedSubmission()).toBeUndefined() + expect(store.draft).toBe('New chat draft') + expect(store.attachments).toMatchObject([ + { id: 'new-video', previewUrl: 'blob:current-poster' } + ]) + expect(store.referenceAttachment(video.id)).toBe(false) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:late-poster') + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith('blob:new-video') + expect(URL.revokeObjectURL).not.toHaveBeenCalledWith('blob:current-poster') + }) + + it('releases the current poster completed during an invalidated send exactly once', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:video' }) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + result.resolve('blob:completed-poster') + await result.promise + store.invalidateSubmission() + store.invalidateSubmission() + store.settleSubmission(id, true) + expect(URL.revokeObjectURL).toHaveBeenCalledTimes(2) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:video') + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:completed-poster') + expect(store.submission).toBeNull() + expect(store.attachments).toEqual([]) + }) + + it('keeps a snapshot asset and decoder retained by the current draft through invalidation and a repeated send', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment({ ...video, mediaUrl: 'blob:retained-video' }) + store.referenceAttachment(video.id) + const snapshot = { + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + } + const first = store.startSubmission(snapshot) + store.restorePrompt(snapshot.prompt, snapshot.attachments) + store.invalidateSubmission() + expect( + vi.mocked(videoThumbnail.createVideoThumbnail).mock.calls[0][1].aborted + ).toBe(false) + result.resolve('blob:retained-poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:retained-poster') + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + const second = store.startSubmission({ + ...snapshot, + attachments: store.attachments + }) + store.settleSubmission(first, false) + expect(store.submission?.id).toBe(second) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + store.settleSubmission(second, false) + const failed = store.takeFailedSubmission() + assert.exists(failed) + store.restorePrompt(failed.prompt, failed.attachments) + expect(store.attachments[0].previewUrl).toBe('blob:retained-poster') + expect(store.prompt.references).toHaveLength(1) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + }) + + it('generates once for the included asset and shares the result with every occurrence', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + store.referenceAttachment(video.id) + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { name: 'Renamed' }) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + result.resolve('blob:poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(store.prompt.references.map((reference) => reference.kind)).toEqual([ + 'asset', + 'asset' + ]) + expect( + store.prompt.references + .filter((reference) => reference.kind === 'asset') + .map((reference) => reference.attachment.previewUrl) + ).toEqual(['blob:poster', 'blob:poster']) + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { + mediaUrl: '/uploaded.mp4', + uploading: false + }) + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + store.removeReference(`asset:${video.id}`) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + store.removeAttachment(video.id) + expect(URL.revokeObjectURL).toHaveBeenCalledExactlyOnceWith('blob:poster') + }) + + it.for([ + { + event: 'upload metadata', + act: (store: ComposerStore) => { + store.addAttachment({ + id: video.id, + name: 'clip.mp4', + ref: '', + uploading: true + }) + expect(videoThumbnail.createVideoThumbnail).not.toHaveBeenCalled() + store.updateAttachment(video.id, { + mediaKind: 'video', + mediaUrl: 'blob:clip' + }) + } + }, + { + event: 'a restored draft', + act: (store: ComposerStore) => + store.replaceDraft({ + text: '', + workflowReferences: [], + attachments: [video] + }) + } + ])( + 'starts shared capture when the video source arrives through $event', + async ({ act }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + act(store) + result.resolve('blob:poster') + await result.promise + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + } + ) + + it('uses a trusted poster immediately without decoding video', () => { + const store = useAgentComposerStore() + store.replaceDraft({ + text: '', + workflowReferences: [], + attachments: [{ ...video, previewUrl: '/server-poster.png' }] + }) + store.referenceAttachment(video.id) + expect(store.attachments[0].previewUrl).toBe('/server-poster.png') + expect(videoThumbnail.createVideoThumbnail).not.toHaveBeenCalled() + }) + + it.for([ + { + event: 'removing the asset', + act: (store: ComposerStore) => store.removeAttachment(video.id) + }, + { + event: 'replacing the video source', + act: (store: ComposerStore) => + store.updateAttachment(video.id, { mediaUrl: '/new.mp4' }) + }, + { + event: 'receiving a trusted poster', + act: (store: ComposerStore) => + store.updateAttachment(video.id, { previewUrl: '/server.png' }) + }, + { + event: 'disposing the composer', + act: (store: ComposerStore) => store.$dispose() + } + ])( + 'discards and releases an obsolete capture after $event', + async ({ act }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + const signal = vi.mocked(videoThumbnail.createVideoThumbnail).mock + .calls[0][1] + result.resolve('blob:stale') + act(store) + expect(signal.aborted).toBe(true) + await result.promise + expect( + store.attachments.every((asset) => asset.previewUrl !== 'blob:stale') + ).toBe(true) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:stale') + } + ) + + it('does not repeatedly retry failed decoding when referencing or updating the same asset', async () => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + result.resolve(undefined) + await result.promise + store.referenceAttachment(video.id) + store.updateAttachment(video.id, { name: 'Renamed' }) + expect(store.attachments[0].previewUrl).toBeUndefined() + expect(store.prompt.references).toHaveLength(1) + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + }) + + it.for([ + { + outcome: 'reclaims the poster after a successful send', + sent: true, + assertOutcome: (store: ComposerStore) => { + expect(store.attachments).toEqual([]) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:poster') + } + }, + { + outcome: 'retains the poster when recovering a failed send', + sent: false, + assertOutcome: (store: ComposerStore) => { + const failed = store.takeFailedSubmission() + assert.exists(failed) + store.restorePrompt(failed.prompt, failed.attachments) + expect(store.attachments[0].previewUrl).toBe('blob:poster') + expect(store.prompt.references).toHaveLength(1) + expect(URL.revokeObjectURL).not.toHaveBeenCalled() + expect(videoThumbnail.createVideoThumbnail).toHaveBeenCalledTimes(1) + } + } + ])( + '$outcome when capture completes during submission', + async ({ sent, assertOutcome }) => { + const result = pendingThumbnail() + const store = useAgentComposerStore() + store.addAttachment(video) + store.referenceAttachment(video.id) + const id = store.startSubmission({ + prompt: store.prompt, + attachments: store.attachments, + nodes: [], + target: createMockLoadedWorkflow({ path: 'workflows/target.json' }) + }) + result.resolve('blob:poster') + await result.promise + store.settleSubmission(id, sent) + assertOutcome(store) + } + ) +}) diff --git a/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts b/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts index 6007607029b..08001531ba5 100644 --- a/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts +++ b/src/workbench/extensions/agent/stores/agent/agentComposerStore.ts @@ -1,10 +1,12 @@ import { defineStore } from 'pinia' -import { computed, ref, shallowRef } from 'vue' +import { computed, onScopeDispose, ref, shallowReactive, shallowRef } from 'vue' + +import { getMediaTypeFromFilename } from '@/utils/formatUtil' import type { AgentInputMethod } from '@/platform/telemetry/types' import type { ComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' -import type { ComposerAttachment } from '../../composables/agent/useComposer' +import type { ComposerAttachment } from '../../types/composerAttachment' import type { SelectedNode } from '../../composables/agent/useCanvasSelection' import { selectedNodeKey } from '../../composables/agent/useCanvasSelection' import type { @@ -18,6 +20,7 @@ import type { WorkflowReference } from '../../types/workflowReference' import { insertComposerReference } from '../../utils/composerPrompt' +import { createVideoThumbnail } from '../../utils/videoThumbnail' import type { AgentStarterPromptSource } from '../../utils/starterPrompts' interface ComposerDraft extends PromptSnapshot { @@ -48,10 +51,13 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const draftState = shallowRef({ text: '', references: [] }) const prompt = computed(() => draftState.value) const draft = computed(() => prompt.value.text) + const attachmentIds = ref([]) + const assetsById = shallowReactive(new Map()) const attachments = computed(() => - prompt.value.references.flatMap((item) => - item.kind === 'asset' ? [item.attachment] : [] - ) + attachmentIds.value.flatMap((id) => { + const attachment = assetsById.get(id) + return attachment ? [attachment] : [] + }) ) const workflowReferences = computed(() => prompt.value.references.flatMap((item) => { @@ -78,9 +84,49 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { textOffset: 0, referenceIndex: 0 }) - // Detached assets stay available to the editor's Undo history until it unmounts. - const undoAssets = new Map() const retiredAssets = new Set() + const thumbnailJobs = new Map< + string, + { source: string; controller: AbortController } + >() + onScopeDispose(() => { + for (const job of thumbnailJobs.values()) job.controller.abort() + thumbnailJobs.clear() + for (const attachment of assetsById.values()) revokePreview(attachment) + }) + + function cancelThumbnail(id: string): void { + thumbnailJobs.get(id)?.controller.abort() + thumbnailJobs.delete(id) + } + + function ensureThumbnail(attachment: ComposerAttachment): void { + const { id, mediaUrl, previewUrl } = attachment + const kind = + attachment.mediaKind ?? getMediaTypeFromFilename(attachment.name) + if (kind !== 'video' || !mediaUrl || previewUrl) { + cancelThumbnail(id) + return + } + if (thumbnailJobs.get(id)?.source === mediaUrl) return + cancelThumbnail(id) + const job = { source: mediaUrl, controller: new AbortController() } + thumbnailJobs.set(id, job) + void createVideoThumbnail(mediaUrl, job.controller.signal).then((url) => { + if (!url) return + const current = assetsById.get(id) + if ( + thumbnailJobs.get(id) !== job || + !current || + current.previewUrl || + current.mediaUrl !== mediaUrl + ) { + URL.revokeObjectURL(url) + return + } + updateAttachment(id, { previewUrl: url }) + }) + } const submission = shallowRef<{ id: number phase: 'pending' | 'failed' @@ -125,13 +171,13 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const seen = new Set() const references = next.references.flatMap((item): ComposerReference[] => { const key = composerReferenceKey(item) - if (seen.has(key)) return [] + if (item.kind !== 'asset' && seen.has(key)) return [] seen.add(key) if (item.kind === 'node' && item.scope !== nodeScope.value) return [] if (item.kind !== 'asset') return [item] - if (retiredAssets.has(item.attachment.id)) return [] - const attachment = undoAssets.get(item.attachment.id) ?? item.attachment - undoAssets.set(attachment.id, attachment) + if (!attachmentIds.value.includes(item.attachment.id)) return [] + const attachment = assetsById.get(item.attachment.id) + if (!attachment) return [] return [{ ...item, attachment }] }) if ( @@ -172,31 +218,41 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { }) } - function restorePrompt(next: ComposerPrompt): void { + function restorePrompt( + next: ComposerPrompt, + included: ComposerAttachment[] = attachments.value + ): void { + for (const attachment of included) { + const current = assetsById.get(attachment.id) + const restored = { + ...attachment, + ...(current?.mediaUrl === attachment.mediaUrl && + current?.previewUrl && + !attachment.previewUrl + ? { previewUrl: current.previewUrl } + : {}) + } + if (current) revokePreview(current, restored) + assetsById.set(attachment.id, restored) + retiredAssets.delete(attachment.id) + ensureThumbnail(restored) + } + attachmentIds.value = included.map(({ id }) => id) resetPromptHistory() applyEditorPrompt(next) + releaseUnusedAssets() } function replaceDraft(next: ComposerDraft): void { - for (const attachment of next.attachments) { - undoAssets.set(attachment.id, { ...attachment }) - retiredAssets.delete(attachment.id) - } - restorePrompt({ - text: next.text, - references: [ - ...next.workflowReferences.map( - (item): ComposerReference => ({ ...item, kind: 'workflow' }) - ), - ...next.attachments.map( - (attachment): ComposerReference => ({ - kind: 'asset', - attachment: { ...attachment }, - textOffset: next.text.length - }) - ) - ].sort((a, b) => a.textOffset - b.textOffset) - }) + restorePrompt( + { + text: next.text, + references: next.workflowReferences + .map((item): ComposerReference => ({ ...item, kind: 'workflow' })) + .sort((a, b) => a.textOffset - b.textOffset) + }, + next.attachments + ) } function setWorkflowReferences(references: WorkflowReference[]): void { @@ -294,10 +350,24 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { }) } - function addAttachment(attachment: ComposerAttachment): void { - if (undoAssets.has(attachment.id) || retiredAssets.has(attachment.id)) - return - undoAssets.set(attachment.id, { ...attachment }) + function addAttachment(attachment: ComposerAttachment): boolean { + if (assetsById.has(attachment.id) || retiredAssets.has(attachment.id)) + return false + if ( + attachment.sourceKey && + attachments.value.some((item) => item.sourceKey === attachment.sourceKey) + ) + return false + assetsById.set(attachment.id, { ...attachment }) + attachmentIds.value = [...attachmentIds.value, attachment.id] + ++revision + ensureThumbnail(attachment) + return true + } + + function referenceAttachment(id: string): boolean { + const attachment = assetsById.get(id) + if (!attachment || !attachmentIds.value.includes(id)) return false const inserted = insertComposerReference( prompt.value, { kind: 'asset', attachment: { ...attachment }, textOffset: 0 }, @@ -305,46 +375,58 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { ) insertionPoint.value = inserted.insertion updateDraft(inserted.prompt) + return true } - function revokePreview(attachment: ComposerAttachment): void { - if (attachment.previewUrl?.startsWith('blob:')) - URL.revokeObjectURL(attachment.previewUrl) + function revokePreview( + attachment: Pick, + retained?: ComposerAttachment + ): void { + const retainedUrls = new Set([retained?.previewUrl, retained?.mediaUrl]) + for (const url of new Set([attachment.previewUrl, attachment.mediaUrl])) + if (url?.startsWith('blob:') && !retainedUrls.has(url)) + URL.revokeObjectURL(url) } function updateAttachment( id: string, patch: Partial ): void { - const previous = undoAssets.get(id) + const previous = assetsById.get(id) if (!previous) { - if (patch.previewUrl?.startsWith('blob:')) - URL.revokeObjectURL(patch.previewUrl) + revokePreview(patch) return } - if ( - patch.previewUrl !== undefined && - patch.previewUrl !== previous.previewUrl - ) - revokePreview(previous) const attachment = { ...previous, ...patch, id } - undoAssets.set(id, attachment) - if (!attachments.value.some((item) => item.id === id)) return - updateDraft({ - text: draft.value, - references: prompt.value.references.map((item) => - item.kind === 'asset' && item.attachment.id === id - ? { ...item, attachment } - : item + if (attachment.mediaUrl !== previous.mediaUrl) cancelThumbnail(id) + revokePreview(previous, attachment) + assetsById.set(id, attachment) + ensureThumbnail(attachment) + if ( + prompt.value.references.some( + (item) => item.kind === 'asset' && item.attachment.id === id ) - }) + ) + updateDraft({ + text: draft.value, + references: prompt.value.references.map((item) => + item.kind === 'asset' && item.attachment.id === id + ? { ...item, attachment } + : item + ) + }) } function removeAttachment(id: string): void { - const attachment = undoAssets.get(id) + cancelThumbnail(id) + const attachment = assetsById.get(id) if (attachment) revokePreview(attachment) - undoAssets.delete(id) + assetsById.delete(id) retiredAssets.add(id) + if (attachmentIds.value.includes(id)) { + attachmentIds.value = attachmentIds.value.filter((item) => item !== id) + ++revision + } removeReference(`asset:${id}`) } @@ -352,10 +434,11 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const retained = new Set(attachments.value.map(({ id }) => id)) for (const attachment of submission.value?.snapshot.attachments ?? []) retained.add(attachment.id) - for (const [id, attachment] of undoAssets) + for (const [id, attachment] of assetsById) if (!retained.has(id)) { + cancelThumbnail(id) revokePreview(attachment) - undoAssets.delete(id) + assetsById.delete(id) retiredAssets.add(id) } } @@ -369,6 +452,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { const chip = starterPrompt.value promptOrigin.value = 'typed' starterPrompt.value = null + attachmentIds.value = [] updateDraft({ text: '', references: [] }) insertionPoint.value = { textOffset: 0, referenceIndex: 0 } const id = ++nextSubmissionId @@ -413,6 +497,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { function invalidateSubmission(): void { submission.value = null + releaseUnusedAssets() } return { @@ -442,6 +527,7 @@ export const useAgentComposerStore = defineStore('agentComposer', () => { setNodeScope, setNodes, addAttachment, + referenceAttachment, updateAttachment, removeAttachment, releaseUnusedAssets, diff --git a/src/workbench/extensions/agent/types/assetPreview.ts b/src/workbench/extensions/agent/types/assetPreview.ts new file mode 100644 index 00000000000..d67a6600c66 --- /dev/null +++ b/src/workbench/extensions/agent/types/assetPreview.ts @@ -0,0 +1 @@ +export type AssetPreviewMode = 'closed' | 'hover' | 'interactive' | 'dismissed' diff --git a/src/workbench/extensions/agent/types/composerAttachment.ts b/src/workbench/extensions/agent/types/composerAttachment.ts new file mode 100644 index 00000000000..78ed554149e --- /dev/null +++ b/src/workbench/extensions/agent/types/composerAttachment.ts @@ -0,0 +1,14 @@ +import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' + +export interface ComposerAttachment { + id: string + name: string + ref: string + sourceKey?: string + previewUrl?: string + mediaUrl?: string + mediaKind?: MediaKind + subfolder?: string + uploadType?: string + uploading?: boolean +} diff --git a/src/workbench/extensions/agent/types/composerPrompt.ts b/src/workbench/extensions/agent/types/composerPrompt.ts index 39355371cc2..d3b8280c0d0 100644 --- a/src/workbench/extensions/agent/types/composerPrompt.ts +++ b/src/workbench/extensions/agent/types/composerPrompt.ts @@ -1,6 +1,6 @@ import type { SelectedNode } from '../composables/agent/useCanvasSelection' import { selectedNodeKey } from '../composables/agent/useCanvasSelection' -import type { ComposerAttachment } from '../composables/agent/useComposer' +import type { ComposerAttachment } from './composerAttachment' import type { WorkflowReference } from './workflowReference' export type ComposerReference = diff --git a/src/workbench/extensions/agent/utils/videoThumbnail.test.ts b/src/workbench/extensions/agent/utils/videoThumbnail.test.ts new file mode 100644 index 00000000000..50525626588 --- /dev/null +++ b/src/workbench/extensions/agent/utils/videoThumbnail.test.ts @@ -0,0 +1,110 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + createMockCanvasRenderingContext2D, + createTestCanvasElement +} from '@/utils/__tests__/canvasTestUtils' + +import { createVideoThumbnail } from './videoThumbnail' + +function mediaPlatform(canvas = document.createElement('canvas')) { + const video = document.createElement('video') + const createElement = document.createElement.bind(document) + vi.spyOn(document, 'createElement').mockImplementation((tag: string) => { + if (tag === 'video') return video + if (tag === 'canvas') return canvas + return createElement(tag) + }) + Object.defineProperties(video, { + videoWidth: { value: 1920 }, + videoHeight: { value: 1080 } + }) + vi.spyOn(video, 'pause').mockImplementation(() => {}) + vi.spyOn(video, 'load').mockImplementation(() => {}) + return { video, canvas } +} + +beforeEach(() => { + vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:poster') +}) + +describe('video thumbnail capture platform boundary', () => { + it('captures a bounded first frame without playback, then releases the decoder', async () => { + const context = createMockCanvasRenderingContext2D({ drawImage: vi.fn() }) + const { video, canvas } = mediaPlatform( + createTestCanvasElement({ ctx: context }) + ) + vi.spyOn(canvas, 'toBlob').mockImplementation((callback) => + callback(new Blob(['frame'], { type: 'image/jpeg' })) + ) + const result = createVideoThumbnail( + '/clip.mp4', + new AbortController().signal + ) + expect(video.src).toContain('/clip.mp4') + expect(video.crossOrigin).toBe('anonymous') + expect(video.muted).toBe(true) + expect(video.autoplay).toBe(false) + video.dispatchEvent(new Event('loadeddata')) + expect(await result).toBe('blob:poster') + expect(canvas.width).toBe(320) + expect(canvas.height).toBe(180) + expect(context.drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180) + expect(video).not.toHaveAttribute('src') + expect(video.pause).toHaveBeenCalled() + }) + + it('does not allocate a poster after cancellation during image encoding', async () => { + const { video, canvas } = mediaPlatform( + createTestCanvasElement({ + ctx: createMockCanvasRenderingContext2D({ drawImage: vi.fn() }) + }) + ) + let complete!: BlobCallback + vi.spyOn(canvas, 'toBlob').mockImplementation((callback) => { + complete = callback + }) + const controller = new AbortController() + const result = createVideoThumbnail('/clip.mp4', controller.signal) + video.dispatchEvent(new Event('loadeddata')) + controller.abort() + expect(await result).toBeUndefined() + complete(new Blob(['frame'], { type: 'image/jpeg' })) + expect(URL.createObjectURL).not.toHaveBeenCalled() + expect(video).not.toHaveAttribute('src') + }) + + it.for([ + { + event: 'a decoding error', + act: (video: HTMLVideoElement) => video.dispatchEvent(new Event('error')) + }, + { + event: 'a timeout', + act: () => vi.advanceTimersByTime(15_000) + }, + { + event: 'cancellation', + act: (_video: HTMLVideoElement, controller: AbortController) => + controller.abort() + }, + { + event: 'an unavailable canvas context', + act: (video: HTMLVideoElement) => + video.dispatchEvent(new Event('loadeddata')) + } + ])( + 'resolves to a fallback and releases the decoder after $event', + async ({ act }) => { + const { video } = mediaPlatform() + const controller = new AbortController() + const result = createVideoThumbnail('/clip.mp4', controller.signal) + act(video, controller) + expect(await result).toBeUndefined() + expect(video).not.toHaveAttribute('src') + expect(video.pause).toHaveBeenCalled() + video.dispatchEvent(new Event('loadeddata')) + expect(URL.createObjectURL).not.toHaveBeenCalled() + } + ) +}) diff --git a/src/workbench/extensions/agent/utils/videoThumbnail.ts b/src/workbench/extensions/agent/utils/videoThumbnail.ts new file mode 100644 index 00000000000..6b3c0a53a4f --- /dev/null +++ b/src/workbench/extensions/agent/utils/videoThumbnail.ts @@ -0,0 +1,59 @@ +export function createVideoThumbnail( + source: string, + signal: AbortSignal +): Promise { + if (signal.aborted) return Promise.resolve(undefined) + const video = document.createElement('video') + video.crossOrigin = 'anonymous' + video.preload = 'auto' + video.muted = true + video.playsInline = true + + return new Promise((resolve) => { + let settled = false + const finish = (url?: string) => { + if (settled) return + settled = true + clearTimeout(timer) + signal.removeEventListener('abort', cancel) + video.removeEventListener('error', cancel) + video.removeEventListener('loadeddata', capture) + video.pause() + video.removeAttribute('src') + video.load() + resolve(url) + } + const cancel = () => finish() + const capture = () => { + if (!video.videoWidth || !video.videoHeight) return finish() + try { + const canvas = document.createElement('canvas') + const scale = Math.min( + 320 / video.videoWidth, + 320 / video.videoHeight, + 1 + ) + canvas.width = Math.max(1, Math.round(video.videoWidth * scale)) + canvas.height = Math.max(1, Math.round(video.videoHeight * scale)) + const context = canvas.getContext('2d') + if (!context) return finish() + context.drawImage(video, 0, 0, canvas.width, canvas.height) + canvas.toBlob( + (blob) => { + if (!settled) finish(blob ? URL.createObjectURL(blob) : undefined) + }, + 'image/jpeg', + 0.8 + ) + } catch { + finish() + } + } + const timer = setTimeout(cancel, 15_000) + signal.addEventListener('abort', cancel, { once: true }) + video.addEventListener('error', cancel, { once: true }) + video.addEventListener('loadeddata', capture, { once: true }) + video.src = source + video.load() + }) +}