diff --git a/browser_tests/fixtures/components/AgentPanel.ts b/browser_tests/fixtures/components/AgentPanel.ts index 2cf1ba647a1..9b2388473f6 100644 --- a/browser_tests/fixtures/components/AgentPanel.ts +++ b/browser_tests/fixtures/components/AgentPanel.ts @@ -139,6 +139,12 @@ export class AgentPanel { ) } + attachmentPreviewPlayer(name: string): Locator { + return this.page + .getByLabel(name, { exact: true }) + .and(this.page.locator('video, audio')) + } + async open(timeout?: number): Promise { if (await this.root.isVisible()) return this.root diff --git a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts index 26cf330b2a2..19cf59fd8e7 100644 --- a/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts +++ b/browser_tests/tests/agent/agentComposerAssetDrop.spec.ts @@ -18,7 +18,8 @@ import { assetPath } from '@e2e/fixtures/utils/paths' // The drop side accepts a drop purely on `dataTransfer.types` containing // `application/x-comfy-asset-info` (`AgentPanelRoot` `isAssetDrag`), so it is // view-agnostic by construction; what these cases pin is that every view mode -// actually publishes that payload, and that the chip names the asset dragged. +// actually publishes that payload, that the chip names the asset dragged, and +// that a dropped MP4's chip previews it with a generated poster. const ASSET_NAME = AGENT_VIDEO_ASSET.name test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { @@ -71,6 +72,40 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => { }) } + test( + 'previews a video from the keyboard and restores focus on Escape', + { tag: '@ui' }, + async ({ page }) => { + const agentPanel = new AgentPanel(page) + await agentPanel.open() + + const assets = new AssetsSidebarTab(page) + await assets.open() + await assets.assetCards.first().dragTo(agentPanel.root) + + const chip = agentPanel.attachmentChip(ASSET_NAME) + const trigger = chip.getByRole('button', { name: ASSET_NAME }) + const poster = trigger.getByRole('img', { name: ASSET_NAME }) + await expect(poster).toBeVisible() + await expect(poster).toHaveAttribute('src', /^blob:/) + await expect + .poll(() => + poster.evaluate((image: HTMLImageElement) => image.naturalWidth) + ) + .toBeGreaterThan(0) + + await trigger.press('Enter') + + const player = agentPanel.attachmentPreviewPlayer(ASSET_NAME) + await expect(player).toBeVisible() + await expect(player).toBeFocused() + + await page.keyboard.press('Escape') + await expect(player).toHaveCount(0) + await expect(trigger).toBeFocused() + } + ) + // A filename is user data and may contain a quote or a backslash, which would // break the attribute selector the chip is matched by. Exercised end to end // rather than unit-tested, so a regression in the escaping shows up as the diff --git a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md index 67cc764e172..203dbcd3ca6 100644 --- a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md +++ b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md @@ -29,6 +29,13 @@ remain owned by the store while included or held by a pending snapshot, and are released when no longer retained. UI hover/focus highlighting and native scroll measurements remain local to components and never change the draft. +Playable media URLs and image poster URLs are distinct metadata. The store owns +one cancellable video-thumbnail job per asset/source and publishes its result to +the tray and all prompt occurrences. UI surfaces render that poster; only tray +hover mounts a player. A source change, removal or disposal cancels obsolete +capture. Pending snapshots retain media resources and failed recovery preserves +a poster completed for the same source while the send was pending. + ## Alternatives considered - Continue deriving attachments from references: cannot express an unmentioned diff --git a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts index eaa3189970a..052a3a34373 100644 --- a/src/components/sidebar/tabs/AssetsSidebarListView.test.ts +++ b/src/components/sidebar/tabs/AssetsSidebarListView.test.ts @@ -230,7 +230,9 @@ describe('AssetsSidebarListView', () => { display_name: 'Clip', attachment_ref: 'clip.mp4', media_kind: 'video', - preview_url: undefined + preview_url: undefined, + media_url: + 'http://localhost:3000/api/view?filename=clip.mp4&type=output&subfolder=' }), MIME_ASSET_INFO ) diff --git a/src/locales/en/main.json b/src/locales/en/main.json index a5c7865ca33..eaf33d5bc4e 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5456,6 +5456,7 @@ "failedDetail": "Your onboarding state could not be reset. Please try again." }, "agent": { + "previewAsset": "Preview {name}", "title": "Comfy Agent", "diagnosticReport": { "retry": "Retry copy report", diff --git a/src/platform/assets/schemas/mediaAssetSchema.ts b/src/platform/assets/schemas/mediaAssetSchema.ts index deab1462d8c..a65a000d542 100644 --- a/src/platform/assets/schemas/mediaAssetSchema.ts +++ b/src/platform/assets/schemas/mediaAssetSchema.ts @@ -5,7 +5,7 @@ import { zResultItem } from '@/platform/remote/comfyui/execution/types' import { assetItemSchema } from './assetSchema' -const zMediaKindSchema = z.enum([ +export const zMediaKindSchema = z.enum([ 'video', 'audio', 'image', @@ -56,7 +56,8 @@ const zDraggedAssetInfo = zResultItem.and( z.object({ attachment_ref: z.string().min(1).optional(), media_kind: zMediaKindSchema.optional(), - preview_url: z.string().url().optional() + preview_url: z.string().url().optional(), + media_url: z.string().url().optional() }) ) diff --git a/src/platform/assets/utils/assetDragUtil.test.ts b/src/platform/assets/utils/assetDragUtil.test.ts new file mode 100644 index 00000000000..76004b96598 --- /dev/null +++ b/src/platform/assets/utils/assetDragUtil.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it, vi } from 'vitest' + +import { useFeatureFlags } from '@/composables/useFeatureFlags' +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { api } from '@/scripts/api' +import { getDroppedAsset } from '@/utils/eventUtils' + +import { startAssetDrag } from './assetDragUtil' + +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@/scripts/api')) + +function drag(asset: AssetItem) { + const dataTransfer = new DataTransfer() + const event = new DragEvent('dragstart', { cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return getDroppedAsset(dataTransfer) +} + +describe('asset drag media sources', () => { + it.for([ + { + label: 'server poster', + assetsEnabled: true, + preview_id: 'poster-id', + thumbnail_url: undefined, + preview_url: undefined, + poster: 'http://localhost:8188/api/assets/poster-id/content', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'distinct thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '/view?filename=poster.png', + preview_url: '/view?filename=clip.mp4', + poster: 'http://localhost:8188/api/view?filename=poster.png', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'absolute CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: 'https://cdn.example/thumb.webp?signature=poster', + preview_url: '/view?filename=clip.mp4', + poster: 'https://cdn.example/thumb.webp?signature=poster', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'protocol-relative CDN thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '//cdn.example/thumb.webp', + preview_url: '/view?filename=clip.mp4', + poster: 'http://cdn.example/thumb.webp', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'history video without a poster', + assetsEnabled: false, + preview_id: undefined, + thumbnail_url: '/view?filename=clip.mp4', + preview_url: '/view?filename=clip.mp4', + poster: undefined, + media: 'http://localhost:3000/view?filename=clip.mp4' + }, + { + label: 'assets API video without a poster', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: undefined, + preview_url: undefined, + poster: undefined, + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + } + ])( + 'preserves distinct file/image sources for $label', + ({ + assetsEnabled, + preview_id, + thumbnail_url, + preview_url, + poster, + media + }) => { + vi.mocked(useFeatureFlags().flags).assetsEnabled = assetsEnabled + vi.mocked(api.apiURL).mockImplementation( + (path) => `http://localhost:8188/api${path}` + ) + const asset = { + id: 'video-id', + name: 'clip.mp4', + display_name: 'My clip', + tags: ['input'], + created_at: '2026-10-05T00:00:00Z', + updated_at: '2026-10-05T00:00:00Z', + preview_id, + thumbnail_url, + preview_url + } satisfies AssetItem + expect(drag(asset)).toMatchObject({ + name: 'My clip', + kind: 'video', + mediaUrl: media, + previewUrl: poster + }) + } + ) +}) diff --git a/src/platform/assets/utils/assetDragUtil.ts b/src/platform/assets/utils/assetDragUtil.ts index d7184304162..4a4623dfcfc 100644 --- a/src/platform/assets/utils/assetDragUtil.ts +++ b/src/platform/assets/utils/assetDragUtil.ts @@ -9,6 +9,7 @@ * row that starts a drag carrying nothing. */ import { getMediaTypeFromFilename } from '@/utils/formatUtil' +import { api } from '@/scripts/api' import { getAssetType } from '../composables/media/assetMappers' import { getOutputAssetMetadata } from '../schemas/assetMetadataSchema' @@ -18,6 +19,39 @@ import { getAssetUrlFilename } from './assetMetadataUtils' import { resolvePreviewUrl } from './assetPreviewUtil' import { getAssetFileUrl } from './assetUrlUtil' +function resolveThumbnailUrl(url: string): URL | null { + const source = + URL.canParse(url) || url.startsWith('//') ? url : api.apiURL(url) + return URL.parse(source, location.href) +} + +function assetMediaSources(asset: AssetItem) { + const mediaKind = getMediaTypeFromFilename(asset.name) + const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) + const playable = mediaKind === 'video' || mediaKind === 'audio' + const mediaUrl = playable + ? URL.parse( + getAssetFileUrl(asset, { disposition: 'inline' }), + location.href + ) + : undefined + const posterUrl = asset.preview_id + ? previewUrl + : asset.thumbnail_url && asset.thumbnail_url !== asset.preview_url + ? resolveThumbnailUrl(asset.thumbnail_url) + : undefined + return { + media_kind: mediaKind, + preview_url: + mediaKind === 'image' + ? previewUrl?.toString() + : mediaKind === 'video' + ? posterUrl?.toString() + : undefined, + media_url: mediaUrl?.toString() + } +} + /** * Start a native drag carrying `asset`. * @@ -43,9 +77,6 @@ export function startAssetDrag( if (!dataTransfer) return const output = getOutputAssetMetadata(asset.user_metadata)?.allOutputs?.[0] - const mediaKind = getMediaTypeFromFilename(asset.name) - const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) - const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) const assetInfo = { ...(output?.filename ? { @@ -60,11 +91,11 @@ export function startAssetDrag( display_name: asset.display_name ?? undefined }), attachment_ref: getAssetUrlFilename(asset), - media_kind: mediaKind, - preview_url: mediaKind === 'image' ? previewUrl?.toString() : undefined + ...assetMediaSources(asset) } dataTransfer.setData(MIME_ASSET_INFO, JSON.stringify(assetInfo)) + const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) if (!fileUrl) return dataTransfer.setData('text/uri-list', fileUrl.toString()) diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts index 3dc5be6e58a..af1822bd411 100644 --- a/src/utils/eventUtils.ts +++ b/src/utils/eventUtils.ts @@ -7,6 +7,7 @@ export interface DroppedAsset { ref?: string kind?: MediaKind previewUrl?: string + mediaUrl?: string } export function getDroppedAsset( @@ -27,7 +28,8 @@ export function getDroppedAsset( uri, ref, kind: asset?.media_kind, - previewUrl: asset?.preview_url + previewUrl: asset?.preview_url, + mediaUrl: asset?.media_url } : undefined } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index b6685b36c2d..f3347704c98 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -2113,7 +2113,10 @@ function fileOfSize(name: string, size: number, type: string): File { return file } -function assetPanelDrag(displayName: string | null) { +function assetPanelDrag( + displayName: string | null, + overrides: Partial = {} +) { const asset = { id: 'library-source', name: 'library.png', @@ -2121,7 +2124,8 @@ function assetPanelDrag(displayName: string | null) { display_name: displayName, tags: ['input'], created_at: '2026-10-03T00:00:00Z', - updated_at: '2026-10-03T00:00:00Z' + updated_at: '2026-10-03T00:00:00Z', + ...overrides } satisfies AssetItem const dataTransfer = new DataTransfer() const event = new DragEvent('dragstart', { cancelable: true }) @@ -2415,6 +2419,67 @@ describe('AgentPanelRoot attach flow', () => { } ) + it.for([ + { + filename: 'clip.mp4', + label: 'My clip', + kind: 'video', + previewId: 'poster', + previewUrl: 'http://localhost:3000/api/assets/poster/content', + indicatorLabel: 'My clip', + indicatorSource: 'http://localhost:3000/api/assets/poster/content' + }, + { + filename: 'song.mp3', + label: 'My recording', + kind: 'audio', + previewId: undefined, + previewUrl: undefined, + indicatorLabel: 'Audio', + indicatorSource: null + } + ])( + 'preserves playable $kind metadata from the assets panel through the chat tray', + async ({ + filename, + label, + kind, + previewId, + previewUrl, + indicatorLabel, + indicatorSource + }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep typing') + await nextTick() + const data = assetPanelDrag(label, { + name: filename, + hash: `stored-${filename}`, + preview_id: previewId + }) + expect(dispatchDrag(screen.getByRole('textbox'), 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: label }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: label, + mediaKind: kind, + previewUrl, + mediaUrl: + 'http://localhost:3000/api/assets/library-source/content?disposition=inline' + }) + ]) + expect( + within(trayItem) + .getByRole('img', { name: indicatorLabel }) + .getAttribute('src') + ).toBe(indicatorSource) + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(uploaded).toEqual([]) + } + ) + it('allows a removed library asset to be explicitly dropped again while retaining retirement of its old identity', async () => { stubUploadFetch() renderWithSelectedTarget() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index bf9934dc1e9..9e497d90f8c 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -1838,7 +1838,9 @@ async function attachDroppedAsset(event: DragEvent): Promise { id: `asset:${crypto.randomUUID()}`, name: asset.name, ref: asset.ref, - previewUrl: asset.previewUrl + previewUrl: asset.previewUrl, + mediaUrl: asset.mediaUrl, + mediaKind: asset.kind }) } diff --git a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts index 9dad4487e4c..1cc504c6b0f 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts @@ -24,6 +24,12 @@ const asset = { previewUrl: 'https://example.com/source.png' } +type MediaSurfaces = { + chip: HTMLElement + trayItem: HTMLElement + menuItem: HTMLElement +} + function setup() { const store = useAgentComposerStore() store.setNodeScope('workflow-a') @@ -104,6 +110,121 @@ function setTrayGeometry( } describe('composer asset tray', () => { + it('updates every inline video thumbnail without changing its references or opening a preview', async () => { + const user = userEvent.setup() + const { store } = setup() + store.addAttachment({ + id: 'video', + name: 'My video', + ref: '', + mediaKind: 'video', + mediaUrl: 'blob:video', + uploading: true + }) + store.referenceAttachment('video') + store.referenceAttachment('video') + const chips = await screen.findAllByTestId('asset-reference-chip') + expect( + chips.map((chip) => + within(chip) + .getByRole('img', { name: 'Video' }) + .getAttribute('aria-label') + ) + ).toEqual(['Video', 'Video']) + store.updateAttachment('video', { + ref: 'stored.mp4', + mediaUrl: '/stored.mp4', + previewUrl: '/poster.png', + uploading: false + }) + await waitFor(() => + expect( + screen + .getAllByTestId('asset-reference-chip') + .map((chip) => within(chip).getByAltText('').getAttribute('src')) + ).toEqual(['/poster.png', '/poster.png']) + ) + await user.hover(screen.getAllByTestId('asset-reference-chip')[0]) + expect(screen.getByRole('group', { name: 'My video' })).toHaveAttribute( + 'data-highlighted', + 'true' + ) + expect( + screen.queryByRole('region', { name: 'My video' }) + ).not.toBeInTheDocument() + expect(store.prompt.references).toHaveLength(2) + expect(store.attachments).toHaveLength(1) + }) + + it('shows the audio type of a renamed inline attachment', async () => { + const { store } = setup() + store.addAttachment({ + id: 'audio', + name: 'Recording', + ref: 'song.mp3', + mediaKind: 'audio', + mediaUrl: '/song.mp3' + }) + store.referenceAttachment('audio') + const chip = await screen.findByTestId('asset-reference-chip') + expect(within(chip).getByRole('img', { name: 'Audio' })).toBeInTheDocument() + }) + + it.for([ + { + name: 'My video', + mediaKind: 'video', + previewUrl: '/poster.png', + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => [ + within(chip).getByAltText(''), + within(trayItem).getByAltText('My video'), + within(menuItem).getByAltText('') + ], + sources: ['/poster.png', '/poster.png', '/poster.png'], + labels: [null, null, null] + }, + { + name: 'Recording', + mediaKind: 'audio', + previewUrl: undefined, + indicators: ({ chip, trayItem, menuItem }: MediaSurfaces) => + [chip, trayItem, menuItem].map((surface) => + within(surface).getByRole('img', { name: 'Audio' }) + ), + sources: [null, null, null], + labels: ['Audio', 'Audio', 'Audio'] + } + ] as const)( + 'uses the same media indicator for $mediaKind in the tray, menu and inline', + async ({ name, mediaKind, previewUrl, indicators, sources, labels }) => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment({ + id: 'media', + name, + ref: '/file', + mediaKind, + previewUrl + }) + store.referenceAttachment('media') + const chip = await screen.findByTestId('asset-reference-chip') + await user.click(editor) + await user.keyboard('@') + const menuItem = await screen.findByRole('menuitem', { name }) + const trayItem = screen.getByRole('group', { name }) + const images = indicators({ chip, trayItem, menuItem }) + expect(images.map((image) => image.getAttribute('src'))).toEqual(sources) + expect(images.map((image) => image.getAttribute('aria-label'))).toEqual( + labels + ) + await user.click(menuItem) + expect(await screen.findAllByTestId('asset-reference-chip')).toHaveLength( + 2 + ) + expect(store.attachments).toHaveLength(1) + } + ) + it('announces all pending uploads outside the scroll viewport and preserves the draft on Enter', async () => { const user = userEvent.setup() const { store, editor, send } = setup() @@ -536,7 +657,7 @@ describe('composer asset tray', () => { screen.queryByRole('tooltip', { name: asset.name }) ).not.toBeInTheDocument() - const trigger = screen.getByTestId('agent-attachment-chip') + const trigger = screen.getByTestId('agent-asset-preview-trigger') await user.hover(trigger) const preview = await screen.findByRole('tooltip', { name: asset.name }) expect( diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue b/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue index c6d51472e1f..8314d95b940 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue +++ b/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue @@ -1,18 +1,157 @@