Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions browser_tests/fixtures/components/AgentPanel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,12 @@ export class AgentPanel {
)
}

attachmentPreviewPlayer(name: string): Locator {
return this.page
.getByLabel(name, { exact: true })
.and(this.page.locator('video, audio'))
}

async open(timeout?: number): Promise<Locator> {
if (await this.root.isVisible()) return this.root

Expand Down
37 changes: 36 additions & 1 deletion browser_tests/tests/agent/agentComposerAssetDrop.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ import { assetPath } from '@e2e/fixtures/utils/paths'
// The drop side accepts a drop purely on `dataTransfer.types` containing
// `application/x-comfy-asset-info` (`AgentPanelRoot` `isAssetDrag`), so it is
// view-agnostic by construction; what these cases pin is that every view mode
// actually publishes that payload, and that the chip names the asset dragged.
// actually publishes that payload, that the chip names the asset dragged, and
// that a dropped MP4's chip previews it with a generated poster.
const ASSET_NAME = AGENT_VIDEO_ASSET.name

test.describe('Agent composer asset drop', { tag: '@cloud' }, () => {
Expand Down Expand Up @@ -71,6 +72,40 @@ test.describe('Agent composer asset drop', { tag: '@cloud' }, () => {
})
}

test(
'previews a video from the keyboard and restores focus on Escape',
{ tag: '@ui' },
async ({ page }) => {
const agentPanel = new AgentPanel(page)
await agentPanel.open()

const assets = new AssetsSidebarTab(page)
await assets.open()
await assets.assetCards.first().dragTo(agentPanel.root)

const chip = agentPanel.attachmentChip(ASSET_NAME)
const trigger = chip.getByRole('button', { name: ASSET_NAME })
const poster = trigger.getByRole('img', { name: ASSET_NAME })
await expect(poster).toBeVisible()
await expect(poster).toHaveAttribute('src', /^blob:/)
await expect
.poll(() =>
poster.evaluate((image: HTMLImageElement) => image.naturalWidth)
)
.toBeGreaterThan(0)

await trigger.press('Enter')

const player = agentPanel.attachmentPreviewPlayer(ASSET_NAME)
await expect(player).toBeVisible()
await expect(player).toBeFocused()

await page.keyboard.press('Escape')
await expect(player).toHaveCount(0)
await expect(trigger).toBeFocused()
}
)

// A filename is user data and may contain a quote or a backslash, which would
// break the attribute selector the chip is matched by. Exercised end to end
// rather than unit-tested, so a regression in the escaping shows up as the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ remain owned by the store while included or held by a pending snapshot, and are
released when no longer retained. UI hover/focus highlighting and native scroll
measurements remain local to components and never change the draft.

Playable media URLs and image poster URLs are distinct metadata. The store owns
one cancellable video-thumbnail job per asset/source and publishes its result to
the tray and all prompt occurrences. UI surfaces render that poster; only tray
hover mounts a player. A source change, removal or disposal cancels obsolete
capture. Pending snapshots retain media resources and failed recovery preserves
a poster completed for the same source while the send was pending.

## Alternatives considered

- Continue deriving attachments from references: cannot express an unmentioned
Expand Down
4 changes: 3 additions & 1 deletion src/components/sidebar/tabs/AssetsSidebarListView.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,9 @@ describe('AssetsSidebarListView', () => {
display_name: 'Clip',
attachment_ref: 'clip.mp4',
media_kind: 'video',
preview_url: undefined
preview_url: undefined,
media_url:
'http://localhost:3000/api/view?filename=clip.mp4&type=output&subfolder='
}),
MIME_ASSET_INFO
)
Expand Down
1 change: 1 addition & 0 deletions src/locales/en/main.json
Original file line number Diff line number Diff line change
Expand Up @@ -5456,6 +5456,7 @@
"failedDetail": "Your onboarding state could not be reset. Please try again."
},
"agent": {
"previewAsset": "Preview {name}",
"title": "Comfy Agent",
"diagnosticReport": {
"retry": "Retry copy report",
Expand Down
5 changes: 3 additions & 2 deletions src/platform/assets/schemas/mediaAssetSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { zResultItem } from '@/platform/remote/comfyui/execution/types'

import { assetItemSchema } from './assetSchema'

const zMediaKindSchema = z.enum([
export const zMediaKindSchema = z.enum([
'video',
'audio',
'image',
Expand Down Expand Up @@ -56,7 +56,8 @@ const zDraggedAssetInfo = zResultItem.and(
z.object({
attachment_ref: z.string().min(1).optional(),
media_kind: zMediaKindSchema.optional(),
preview_url: z.string().url().optional()
preview_url: z.string().url().optional(),
media_url: z.string().url().optional()
})
)

Expand Down
115 changes: 115 additions & 0 deletions src/platform/assets/utils/assetDragUtil.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { describe, expect, it, vi } from 'vitest'

import { useFeatureFlags } from '@/composables/useFeatureFlags'
import type { AssetItem } from '@/platform/assets/schemas/assetSchema'
import { api } from '@/scripts/api'
import { getDroppedAsset } from '@/utils/eventUtils'

import { startAssetDrag } from './assetDragUtil'

vi.mock(import('@/composables/useFeatureFlags'))
vi.mock(import('@/scripts/api'))

function drag(asset: AssetItem) {
const dataTransfer = new DataTransfer()
const event = new DragEvent('dragstart', { cancelable: true })
Object.defineProperty(event, 'dataTransfer', { value: dataTransfer })
startAssetDrag(event, asset)
return getDroppedAsset(dataTransfer)
}

describe('asset drag media sources', () => {
it.for([
{
label: 'server poster',
assetsEnabled: true,
preview_id: 'poster-id',
thumbnail_url: undefined,
preview_url: undefined,
poster: 'http://localhost:8188/api/assets/poster-id/content',
media:
'http://localhost:8188/api/assets/video-id/content?disposition=inline'
},
{
label: 'distinct thumbnail',
assetsEnabled: true,
preview_id: undefined,
thumbnail_url: '/view?filename=poster.png',
preview_url: '/view?filename=clip.mp4',
poster: 'http://localhost:8188/api/view?filename=poster.png',
media:
'http://localhost:8188/api/assets/video-id/content?disposition=inline'
},
{
label: 'absolute CDN thumbnail',
assetsEnabled: true,
preview_id: undefined,
thumbnail_url: 'https://cdn.example/thumb.webp?signature=poster',
preview_url: '/view?filename=clip.mp4',
poster: 'https://cdn.example/thumb.webp?signature=poster',
media:
'http://localhost:8188/api/assets/video-id/content?disposition=inline'
},
{
label: 'protocol-relative CDN thumbnail',
assetsEnabled: true,
preview_id: undefined,
thumbnail_url: '//cdn.example/thumb.webp',
preview_url: '/view?filename=clip.mp4',
poster: 'http://cdn.example/thumb.webp',
media:
'http://localhost:8188/api/assets/video-id/content?disposition=inline'
},
{
label: 'history video without a poster',
assetsEnabled: false,
preview_id: undefined,
thumbnail_url: '/view?filename=clip.mp4',
preview_url: '/view?filename=clip.mp4',
poster: undefined,
media: 'http://localhost:3000/view?filename=clip.mp4'
},
{
label: 'assets API video without a poster',
assetsEnabled: true,
preview_id: undefined,
thumbnail_url: undefined,
preview_url: undefined,
poster: undefined,
media:
'http://localhost:8188/api/assets/video-id/content?disposition=inline'
}
])(
'preserves distinct file/image sources for $label',
({
assetsEnabled,
preview_id,
thumbnail_url,
preview_url,
poster,
media
}) => {
vi.mocked(useFeatureFlags().flags).assetsEnabled = assetsEnabled
vi.mocked(api.apiURL).mockImplementation(
(path) => `http://localhost:8188/api${path}`
)
const asset = {
id: 'video-id',
name: 'clip.mp4',
display_name: 'My clip',
tags: ['input'],
created_at: '2026-10-05T00:00:00Z',
updated_at: '2026-10-05T00:00:00Z',
preview_id,
thumbnail_url,
preview_url
} satisfies AssetItem
expect(drag(asset)).toMatchObject({
name: 'My clip',
kind: 'video',
mediaUrl: media,
previewUrl: poster
})
}
)
})
41 changes: 36 additions & 5 deletions src/platform/assets/utils/assetDragUtil.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* row that starts a drag carrying nothing.
*/
import { getMediaTypeFromFilename } from '@/utils/formatUtil'
import { api } from '@/scripts/api'

import { getAssetType } from '../composables/media/assetMappers'
import { getOutputAssetMetadata } from '../schemas/assetMetadataSchema'
Expand All @@ -18,6 +19,39 @@ import { getAssetUrlFilename } from './assetMetadataUtils'
import { resolvePreviewUrl } from './assetPreviewUtil'
import { getAssetFileUrl } from './assetUrlUtil'

function resolveThumbnailUrl(url: string): URL | null {
const source =
URL.canParse(url) || url.startsWith('//') ? url : api.apiURL(url)
return URL.parse(source, location.href)
}

function assetMediaSources(asset: AssetItem) {
const mediaKind = getMediaTypeFromFilename(asset.name)
const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href)
const playable = mediaKind === 'video' || mediaKind === 'audio'
const mediaUrl = playable
? URL.parse(
getAssetFileUrl(asset, { disposition: 'inline' }),
location.href
)
: undefined
const posterUrl = asset.preview_id
? previewUrl
: asset.thumbnail_url && asset.thumbnail_url !== asset.preview_url
? resolveThumbnailUrl(asset.thumbnail_url)
: undefined
return {
media_kind: mediaKind,
preview_url:
mediaKind === 'image'
? previewUrl?.toString()
: mediaKind === 'video'
? posterUrl?.toString()
: undefined,
media_url: mediaUrl?.toString()
}
}

/**
* Start a native drag carrying `asset`.
*
Expand All @@ -43,9 +77,6 @@ export function startAssetDrag(
if (!dataTransfer) return

const output = getOutputAssetMetadata(asset.user_metadata)?.allOutputs?.[0]
const mediaKind = getMediaTypeFromFilename(asset.name)
const previewUrl = URL.parse(resolvePreviewUrl(asset), location.href)
const fileUrl = URL.parse(getAssetFileUrl(asset), location.href)
const assetInfo = {
...(output?.filename
? {
Expand All @@ -60,11 +91,11 @@ export function startAssetDrag(
display_name: asset.display_name ?? undefined
}),
attachment_ref: getAssetUrlFilename(asset),
media_kind: mediaKind,
preview_url: mediaKind === 'image' ? previewUrl?.toString() : undefined
...assetMediaSources(asset)
}
dataTransfer.setData(MIME_ASSET_INFO, JSON.stringify(assetInfo))

const fileUrl = URL.parse(getAssetFileUrl(asset), location.href)
if (!fileUrl) return

dataTransfer.setData('text/uri-list', fileUrl.toString())
Expand Down
4 changes: 3 additions & 1 deletion src/utils/eventUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export interface DroppedAsset {
ref?: string
kind?: MediaKind
previewUrl?: string
mediaUrl?: string
}

export function getDroppedAsset(
Expand All @@ -27,7 +28,8 @@ export function getDroppedAsset(
uri,
ref,
kind: asset?.media_kind,
previewUrl: asset?.preview_url
previewUrl: asset?.preview_url,
mediaUrl: asset?.media_url
}
: undefined
}
Expand Down
Loading
Loading