From 5df033883b5c6d172e29b8787e20cf263fdfe4a9 Mon Sep 17 00:00:00 2001 From: ryan-di Date: Mon, 5 Oct 2026 11:43:46 +0800 Subject: [PATCH 1/6] feat(agent): share video posters and add media hover playback --- ...ndependent-assets-and-prompt-references.md | 7 + .../assets/schemas/mediaAssetSchema.ts | 5 +- .../assets/utils/assetDragUtil.test.ts | 95 ++++++++++ src/platform/assets/utils/assetDragUtil.ts | 35 +++- src/utils/eventUtils.ts | 4 +- .../extensions/agent/AgentPanelRoot.test.ts | 60 +++++- .../extensions/agent/AgentPanelRoot.vue | 4 +- .../agent/Composer.assetTray.test.ts | 110 +++++++++++ .../agent/composer/AssetHoverPreview.vue | 41 +++- .../agent/composer/AssetMediaPreview.vue | 54 ++++++ .../agent/composer/AssetThumbnail.vue | 13 +- .../components/agent/composer/AssetTray.vue | 4 +- .../agent/composer/AttachmentChip.test.ts | 136 +++++++++++++- .../agent/composer/AttachmentChip.vue | 27 ++- .../composer/InlineAssetReference.test.ts | 55 ++++++ .../agent/composer/InlineAssetReference.vue | 5 +- .../agent/composer/InlinePromptEditor.vue | 1 + .../agent/composer/MentionMenuItem.vue | 1 + .../agent/composer/inlinePrompt.test.ts | 25 +++ .../components/agent/composer/inlinePrompt.ts | 44 +++-- .../agent/useAgentDraftSubmission.test.ts | 31 +++ .../composables/agent/useAttachment.test.ts | 43 ++++- .../agent/composables/agent/useAttachment.ts | 34 +++- .../stores/agent/agentComposerStore.test.ts | 42 +++++ .../agentComposerStore.thumbnail.test.ts | 176 ++++++++++++++++++ .../agent/stores/agent/agentComposerStore.ts | 72 ++++++- .../agent/types/composerAttachment.ts | 4 + .../agent/utils/videoThumbnail.test.ts | 97 ++++++++++ .../extensions/agent/utils/videoThumbnail.ts | 59 ++++++ 29 files changed, 1225 insertions(+), 59 deletions(-) create mode 100644 src/platform/assets/utils/assetDragUtil.test.ts create mode 100644 src/workbench/extensions/agent/components/agent/composer/AssetMediaPreview.vue create mode 100644 src/workbench/extensions/agent/components/agent/composer/InlineAssetReference.test.ts create mode 100644 src/workbench/extensions/agent/stores/agent/agentComposerStore.thumbnail.test.ts create mode 100644 src/workbench/extensions/agent/utils/videoThumbnail.test.ts create mode 100644 src/workbench/extensions/agent/utils/videoThumbnail.ts diff --git a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md index 67cc764e172..203dbcd3ca6 100644 --- a/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md +++ b/docs/adr/AGENT-COMPOSER-0040-independent-assets-and-prompt-references.md @@ -29,6 +29,13 @@ remain owned by the store while included or held by a pending snapshot, and are released when no longer retained. UI hover/focus highlighting and native scroll measurements remain local to components and never change the draft. +Playable media URLs and image poster URLs are distinct metadata. The store owns +one cancellable video-thumbnail job per asset/source and publishes its result to +the tray and all prompt occurrences. UI surfaces render that poster; only tray +hover mounts a player. A source change, removal or disposal cancels obsolete +capture. Pending snapshots retain media resources and failed recovery preserves +a poster completed for the same source while the send was pending. + ## Alternatives considered - Continue deriving attachments from references: cannot express an unmentioned diff --git a/src/platform/assets/schemas/mediaAssetSchema.ts b/src/platform/assets/schemas/mediaAssetSchema.ts index deab1462d8c..a65a000d542 100644 --- a/src/platform/assets/schemas/mediaAssetSchema.ts +++ b/src/platform/assets/schemas/mediaAssetSchema.ts @@ -5,7 +5,7 @@ import { zResultItem } from '@/platform/remote/comfyui/execution/types' import { assetItemSchema } from './assetSchema' -const zMediaKindSchema = z.enum([ +export const zMediaKindSchema = z.enum([ 'video', 'audio', 'image', @@ -56,7 +56,8 @@ const zDraggedAssetInfo = zResultItem.and( z.object({ attachment_ref: z.string().min(1).optional(), media_kind: zMediaKindSchema.optional(), - preview_url: z.string().url().optional() + preview_url: z.string().url().optional(), + media_url: z.string().url().optional() }) ) diff --git a/src/platform/assets/utils/assetDragUtil.test.ts b/src/platform/assets/utils/assetDragUtil.test.ts new file mode 100644 index 00000000000..2156d538dac --- /dev/null +++ b/src/platform/assets/utils/assetDragUtil.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it, vi } from 'vitest' + +import { useFeatureFlags } from '@/composables/useFeatureFlags' +import type { AssetItem } from '@/platform/assets/schemas/assetSchema' +import { api } from '@/scripts/api' +import { getDroppedAsset } from '@/utils/eventUtils' + +import { startAssetDrag } from './assetDragUtil' + +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@/scripts/api')) + +function drag(asset: AssetItem) { + const dataTransfer = new DataTransfer() + const event = new DragEvent('dragstart', { cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { value: dataTransfer }) + startAssetDrag(event, asset) + return getDroppedAsset(dataTransfer) +} + +describe('asset drag media sources', () => { + it.for([ + { + label: 'server poster', + assetsEnabled: true, + preview_id: 'poster-id', + thumbnail_url: undefined, + preview_url: undefined, + poster: 'http://localhost:8188/api/assets/poster-id/content', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'distinct thumbnail', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: '/view?filename=poster.png', + preview_url: '/view?filename=clip.mp4', + poster: 'http://localhost:8188/api/view?filename=poster.png', + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + }, + { + label: 'history video without a poster', + assetsEnabled: false, + preview_id: undefined, + thumbnail_url: '/view?filename=clip.mp4', + preview_url: '/view?filename=clip.mp4', + poster: undefined, + media: 'http://localhost:3000/view?filename=clip.mp4' + }, + { + label: 'assets API video without a poster', + assetsEnabled: true, + preview_id: undefined, + thumbnail_url: undefined, + preview_url: undefined, + poster: undefined, + media: + 'http://localhost:8188/api/assets/video-id/content?disposition=inline' + } + ])( + 'preserves distinct file/image sources for $label', + ({ + assetsEnabled, + preview_id, + thumbnail_url, + preview_url, + poster, + media + }) => { + vi.mocked(useFeatureFlags().flags).assetsEnabled = assetsEnabled + vi.mocked(api.apiURL).mockImplementation( + (path) => `http://localhost:8188/api${path}` + ) + const asset = { + id: 'video-id', + name: 'clip.mp4', + display_name: 'My clip', + tags: ['input'], + created_at: '2026-10-05T00:00:00Z', + updated_at: '2026-10-05T00:00:00Z', + preview_id, + thumbnail_url, + preview_url + } satisfies AssetItem + expect(drag(asset)).toMatchObject({ + name: 'My clip', + kind: 'video', + mediaUrl: media, + previewUrl: poster + }) + } + ) +}) diff --git a/src/platform/assets/utils/assetDragUtil.ts b/src/platform/assets/utils/assetDragUtil.ts index d7184304162..8d09c2ddd7d 100644 --- a/src/platform/assets/utils/assetDragUtil.ts +++ b/src/platform/assets/utils/assetDragUtil.ts @@ -9,6 +9,7 @@ * row that starts a drag carrying nothing. */ import { getMediaTypeFromFilename } from '@/utils/formatUtil' +import { api } from '@/scripts/api' import { getAssetType } from '../composables/media/assetMappers' import { getOutputAssetMetadata } from '../schemas/assetMetadataSchema' @@ -18,6 +19,33 @@ import { getAssetUrlFilename } from './assetMetadataUtils' import { resolvePreviewUrl } from './assetPreviewUtil' import { getAssetFileUrl } from './assetUrlUtil' +function assetMediaSources(asset: AssetItem) { + const mediaKind = getMediaTypeFromFilename(asset.name) + const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) + const playable = mediaKind === 'video' || mediaKind === 'audio' + const mediaUrl = playable + ? URL.parse( + getAssetFileUrl(asset, { disposition: 'inline' }), + location.href + ) + : undefined + const posterUrl = asset.preview_id + ? previewUrl + : asset.thumbnail_url && asset.thumbnail_url !== asset.preview_url + ? URL.parse(api.apiURL(asset.thumbnail_url), location.href) + : undefined + return { + media_kind: mediaKind, + preview_url: + mediaKind === 'image' + ? previewUrl?.toString() + : mediaKind === 'video' + ? posterUrl?.toString() + : undefined, + media_url: mediaUrl?.toString() + } +} + /** * Start a native drag carrying `asset`. * @@ -43,9 +71,6 @@ export function startAssetDrag( if (!dataTransfer) return const output = getOutputAssetMetadata(asset.user_metadata)?.allOutputs?.[0] - const mediaKind = getMediaTypeFromFilename(asset.name) - const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href) - const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) const assetInfo = { ...(output?.filename ? { @@ -60,11 +85,11 @@ export function startAssetDrag( display_name: asset.display_name ?? undefined }), attachment_ref: getAssetUrlFilename(asset), - media_kind: mediaKind, - preview_url: mediaKind === 'image' ? previewUrl?.toString() : undefined + ...assetMediaSources(asset) } dataTransfer.setData(MIME_ASSET_INFO, JSON.stringify(assetInfo)) + const fileUrl = URL.parse(getAssetFileUrl(asset), location.href) if (!fileUrl) return dataTransfer.setData('text/uri-list', fileUrl.toString()) diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts index 3dc5be6e58a..af1822bd411 100644 --- a/src/utils/eventUtils.ts +++ b/src/utils/eventUtils.ts @@ -7,6 +7,7 @@ export interface DroppedAsset { ref?: string kind?: MediaKind previewUrl?: string + mediaUrl?: string } export function getDroppedAsset( @@ -27,7 +28,8 @@ export function getDroppedAsset( uri, ref, kind: asset?.media_kind, - previewUrl: asset?.preview_url + previewUrl: asset?.preview_url, + mediaUrl: asset?.media_url } : undefined } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index 565d438c69b..c98c28463ee 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -2113,7 +2113,10 @@ function fileOfSize(name: string, size: number, type: string): File { return file } -function assetPanelDrag(displayName: string | null) { +function assetPanelDrag( + displayName: string | null, + overrides: Partial = {} +) { const asset = { id: 'library-source', name: 'library.png', @@ -2121,7 +2124,8 @@ function assetPanelDrag(displayName: string | null) { display_name: displayName, tags: ['input'], created_at: '2026-10-03T00:00:00Z', - updated_at: '2026-10-03T00:00:00Z' + updated_at: '2026-10-03T00:00:00Z', + ...overrides } satisfies AssetItem const dataTransfer = new DataTransfer() const event = new DragEvent('dragstart', { cancelable: true }) @@ -2415,6 +2419,58 @@ describe('AgentPanelRoot attach flow', () => { } ) + it.for([ + { + filename: 'clip.mp4', + label: 'My clip', + kind: 'video', + previewId: 'poster', + previewUrl: 'http://localhost:3000/api/assets/poster/content' + }, + { + filename: 'song.mp3', + label: 'My recording', + kind: 'audio', + previewId: undefined, + previewUrl: undefined + } + ])( + 'preserves playable $kind metadata from the assets panel through the chat tray', + async ({ filename, label, kind, previewId, previewUrl }) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + const store = useAgentComposerStore() + store.setText('Keep typing') + await nextTick() + const data = assetPanelDrag(label, { + name: filename, + hash: `stored-${filename}`, + preview_id: previewId + }) + expect(dispatchDrag(screen.getByRole('textbox'), 'drop', data)).toBe(true) + const trayItem = await screen.findByRole('group', { name: label }) + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: label, + mediaKind: kind, + previewUrl, + mediaUrl: + 'http://localhost:3000/api/assets/library-source/content?disposition=inline' + }) + ]) + if (kind === 'video') + expect( + within(trayItem).getByRole('img', { name: label }) + ).toHaveAttribute('src', previewUrl) + else + expect( + within(trayItem).getByRole('img', { name: 'Audio' }) + ).toBeInTheDocument() + expect(store.prompt).toEqual({ text: 'Keep typing', references: [] }) + expect(uploaded).toEqual([]) + } + ) + it('allows a removed library asset to be explicitly dropped again while retaining retirement of its old identity', async () => { stubUploadFetch() renderWithSelectedTarget() diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 283e93ceef0..8719c2df261 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -1811,7 +1811,9 @@ async function attachDroppedAsset(event: DragEvent): Promise { id: `asset:${crypto.randomUUID()}`, name: asset.name, ref: asset.ref, - previewUrl: asset.previewUrl + previewUrl: asset.previewUrl, + mediaUrl: asset.mediaUrl, + mediaKind: asset.kind }) } diff --git a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts index 9dad4487e4c..8197741795e 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts @@ -104,6 +104,116 @@ function setTrayGeometry( } describe('composer asset tray', () => { + it('updates every inline video thumbnail without changing its references or opening a preview', async () => { + const user = userEvent.setup() + const { store } = setup() + store.addAttachment({ + id: 'video', + name: 'My video', + ref: '', + mediaKind: 'video', + mediaUrl: 'blob:video', + uploading: true + }) + store.referenceAttachment('video') + store.referenceAttachment('video') + await screen.findAllByTestId('asset-reference-chip') + for (const chip of screen.getAllByTestId('asset-reference-chip')) { + expect( + within(chip).getByRole('img', { name: 'Video' }) + ).toBeInTheDocument() + expect( + within(chip).queryByTestId('inline-asset-video') + ).not.toBeInTheDocument() + } + store.updateAttachment('video', { + ref: 'stored.mp4', + mediaUrl: '/stored.mp4', + previewUrl: '/poster.png', + uploading: false + }) + await waitFor(() => + expect( + screen + .getAllByTestId('asset-reference-chip') + .map((chip) => within(chip).getByAltText('').getAttribute('src')) + ).toEqual(['/poster.png', '/poster.png']) + ) + await user.hover(screen.getAllByTestId('asset-reference-chip')[0]) + expect(screen.getByRole('group', { name: 'My video' })).toHaveAttribute( + 'data-highlighted', + 'true' + ) + expect( + screen.queryByRole('region', { name: 'My video' }) + ).not.toBeInTheDocument() + expect(store.prompt.references).toHaveLength(2) + expect(store.attachments).toHaveLength(1) + }) + + it('shows the audio type of a renamed inline attachment', async () => { + const { store } = setup() + store.addAttachment({ + id: 'audio', + name: 'Recording', + ref: 'song.mp3', + mediaKind: 'audio', + mediaUrl: '/song.mp3' + }) + store.referenceAttachment('audio') + const chip = await screen.findByTestId('asset-reference-chip') + expect(within(chip).getByRole('img', { name: 'Audio' })).toBeInTheDocument() + }) + + it.for([ + { + name: 'My video', + mediaKind: 'video', + previewUrl: '/poster.png', + label: 'Video' + }, + { + name: 'Recording', + mediaKind: 'audio', + previewUrl: undefined, + label: 'Audio' + } + ] as const)( + 'uses the same media indicator for $mediaKind in the tray, menu and inline', + async ({ name, mediaKind, previewUrl, label }) => { + const user = userEvent.setup() + const { store, editor } = setup() + store.addAttachment({ + id: 'media', + name, + ref: '/file', + mediaKind, + previewUrl + }) + store.referenceAttachment('media') + const chip = await screen.findByTestId('asset-reference-chip') + await user.click(editor) + await user.keyboard('@') + const menuItem = await screen.findByRole('menuitem', { name }) + const trayItem = screen.getByRole('group', { name }) + for (const surface of [chip, trayItem, menuItem]) { + if (previewUrl) + expect( + within(surface).getByAltText(surface === trayItem ? name : '') + ).toHaveAttribute('src', previewUrl) + else + expect(within(surface).getByRole('img', { name: label })).toHaveClass( + 'icon-[lucide--music]' + ) + } + await user.click(menuItem) + expect(await screen.findAllByTestId('asset-reference-chip')).toHaveLength( + 2 + ) + expect(store.attachments).toHaveLength(1) + } + ) + it('announces all pending uploads outside the scroll viewport and preserves the draft on Enter', async () => { const user = userEvent.setup() const { store, editor, send } = setup() diff --git a/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue b/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue index c6d51472e1f..b6a0c015bd1 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue +++ b/src/workbench/extensions/agent/components/agent/composer/AssetHoverPreview.vue @@ -1,27 +1,56 @@