From d19a86f3db1d7ab5a954e2d1e044050be1a250d0 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 26 Oct 2026 12:44:53 -0700 Subject: [PATCH 1/3] fix(agent): harden reply asset previews (graph-API-path partition) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #16430 so the fix can land on its own. Every changed path is inside src/workbench/extensions/agent/**, which is one of the four Agent globs in the ProtectMain required_reviewers entry whose minimum_approvals is 0, and is owned by the agent CODEOWNERS entry on line 68. Content is byte-identical to #16430 at head c3358d58f6 for these six files; main has not moved on any of them since that PR's merge base (e910ac2e8e), so nothing was re-authored. - split audio replies into ReplyAudioAssetGroup.vue and visual replies into ReplyVisualAsset.vue, so a non-visual asset no longer renders a thumbnail affordance it cannot satisfy - add isReplyVisualAsset / isReplyAudioAsset narrowing to replyAssets.ts - abort a hidden tile's in-flight thumbnail generation on re-render instead of leaving the tile permanently blank Red on parent, proven: with only the two test files applied onto main at 8d7f60b9b7, `vitest run ReplyAssetGroup.test.ts ReplyVisualAsset.test.ts` is 12 failed / 23 passed across 2 failed files. With the four production files added it is 36 passed / 2 files passed. Deliberately NOT carried here, and left on #16430: the browser_tests half, which is what pushes the PR outside the Agent globs and therefore needs a comfy_frontend_devs approval. - browser_tests/tests/agent/agentReplyAssets.spec.ts (+175/-0) — inside the globs, but it depends on the two fixture changes below - browser_tests/fixtures/components/AgentPanel.ts (+6/-0) — outside the ruleset globs, though CODEOWNERS line 83 already assigns it to the Agent reviewers - browser_tests/fixtures/utils/viewFileMocks.ts (+1/-1) — outside the globs and unowned by CODEOWNERS Local gates: vitest on the two files (36 passed), the whole src/workbench/extensions/agent/components/agent/message directory plus replyAssets (17 files, 305 passed), `vue-tsc --noEmit` clean, eslint clean. --- .../agent/message/ReplyAssetGroup.test.ts | 462 +++++++++++++++--- .../agent/message/ReplyAssetGroup.vue | 442 +++++++++++------ .../agent/message/ReplyAudioAssetGroup.vue | 46 ++ .../agent/message/ReplyVisualAsset.test.ts | 34 ++ .../agent/message/ReplyVisualAsset.vue | 89 ++++ .../extensions/agent/utils/replyAssets.ts | 16 + 6 files changed, 846 insertions(+), 243 deletions(-) create mode 100644 src/workbench/extensions/agent/components/agent/message/ReplyAudioAssetGroup.vue create mode 100644 src/workbench/extensions/agent/components/agent/message/ReplyVisualAsset.test.ts create mode 100644 src/workbench/extensions/agent/components/agent/message/ReplyVisualAsset.vue diff --git a/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.test.ts b/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.test.ts index 1242b8a705b..00d70461963 100644 --- a/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.test.ts +++ b/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.test.ts @@ -2,32 +2,32 @@ import { render, screen, waitFor, within } from '@testing-library/vue' import userEvent from '@testing-library/user-event' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ModelThumbnailResult } from '@/extensions/core/load3d/components/modelThumbnail' import { i18n } from '@/i18n' import { useDialogStore } from '@/stores/dialogStore' -import type { - generateModelThumbnail as generateModelThumbnailContract, - ModelThumbnailResult -} from '@/extensions/core/load3d/components/modelThumbnail' import type { ReplyAsset } from '@/workbench/extensions/agent/utils/replyAssets' import ReplyAssetGroup from './ReplyAssetGroup.vue' -const isAssetPreviewSupported = vi.hoisted(() => vi.fn(() => false)) -const findServerPreviewUrl = vi.hoisted(() => - vi.fn(async (): Promise => null) -) -const findOutputAsset = vi.hoisted(() => - vi.fn(async (): Promise<{ name: string } | undefined> => undefined) -) -vi.mock(import('@/platform/assets/utils/assetPreviewUtil'), () => ({ - isAssetPreviewSupported, - findServerPreviewUrl, - findOutputAsset -})) +vi.mock(import('@/platform/assets/utils/assetPreviewUtil'), { spy: true }) + +import { + findOutputAsset as findOutputAssetImplementation, + findServerPreviewUrl as findServerPreviewUrlImplementation, + isAssetPreviewSupported as isAssetPreviewSupportedImplementation +} from '@/platform/assets/utils/assetPreviewUtil' + +const findOutputAsset = vi.mocked(findOutputAssetImplementation) +const findServerPreviewUrl = vi.mocked(findServerPreviewUrlImplementation) +const isAssetPreviewSupported = vi.mocked(isAssetPreviewSupportedImplementation) const generateModelThumbnail = vi.hoisted(() => - vi.fn( - async (): Promise => ({ status: 'failed' }) + vi.fn( + async ( + _modelUrl: string, + _assetName: string, + _callerSignal?: AbortSignal + ): Promise => ({ status: 'failed' }) ) ) vi.mock(import('@/extensions/core/load3d/components/modelThumbnail'), () => ({ @@ -78,8 +78,30 @@ function renderGroup(assets: ReplyAsset[]) { const thumbs = () => screen.getAllByRole('button').filter((b) => b.getAttribute('aria-label')) -const toggle = () => - screen.getAllByRole('button').find((b) => !b.getAttribute('aria-label')) +const toggle = () => { + const button = screen + .getAllByRole('button') + .find((candidate) => !candidate.getAttribute('aria-label')) + if (!button) throw new Error('Expected the asset group toggle') + return button +} + +function deferred(): { + promise: Promise + resolve: (value: T) => void +} { + let settle: ((value: T) => void) | undefined + const promise = new Promise((resolve) => { + settle = resolve + }) + return { + promise, + resolve: (value) => { + if (!settle) throw new Error('Deferred promise was not initialized') + settle(value) + } + } +} describe('ReplyAssetGroup', () => { beforeEach(() => { @@ -105,16 +127,6 @@ describe('ReplyAssetGroup', () => { 'reply-video-affordance' ) ).toBeInTheDocument() - expect( - within(screen.getByRole('button', { name: 'i1.png' })).queryByTestId( - 'reply-video-affordance' - ) - ).toBeNull() - expect( - within(screen.getByRole('button', { name: 'mesh.glb' })).queryByTestId( - 'reply-video-affordance' - ) - ).toBeNull() }) it('T-09 / PM-652 / FE-1326 opens inspect view at the clicked visual asset', async () => { @@ -127,26 +139,6 @@ describe('ReplyAssetGroup', () => { expect(lightbox.dataset.count).toBe('2') }) - it('plays a video preview on hover and pauses on leave', async () => { - const play = vi - .spyOn(HTMLMediaElement.prototype, 'play') - .mockResolvedValue() - const pause = vi - .spyOn(HTMLMediaElement.prototype, 'pause') - .mockImplementation(() => {}) - renderGroup([video, image(1)]) - const element = screen.getByTestId('reply-video-preview') - - await userEvent.hover(element) - expect(play).toHaveBeenCalledTimes(1) - - await userEvent.unhover(element) - expect(pause).toHaveBeenCalledTimes(1) - - play.mockRestore() - pause.mockRestore() - }) - it('renders an audio card per audio asset outside the visual grid', () => { renderGroup([audio]) @@ -229,7 +221,10 @@ describe('ReplyAssetGroup', () => { it('titles audio cards with the resolved asset name, falling back to filename', async () => { isAssetPreviewSupported.mockReturnValue(true) - findOutputAsset.mockResolvedValue({ name: 'qa_audio_opus_00001' }) + findOutputAsset.mockResolvedValue({ + id: 'audio', + name: 'qa_audio_opus_00001' + }) renderGroup([audio]) await waitFor(() => @@ -295,7 +290,10 @@ describe('ReplyAssetGroup', () => { await vi.runAllTimersAsync() - expect(generateModelThumbnail).toHaveBeenCalledTimes(30) + // The initial attempt plus MAX_THUMBNAIL_BUSY_RETRIES. The backoff is + // exponential and capped at eight minutes, so this covers a longer busy + // period than a flat ladder of the same length would. + expect(generateModelThumbnail).toHaveBeenCalledTimes(10) }) it('generates thumbnails only for model URLs that name a filename query', async () => { @@ -315,40 +313,232 @@ describe('ReplyAssetGroup', () => { ) }) - it('cancels thumbnail rendering on teardown and remounts with a fresh signal', async () => { + it('does not generate a thumbnail after unmounting during preview lookup', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const preview = deferred() + findServerPreviewUrl.mockReturnValueOnce(preview.promise) + const { unmount } = renderGroup([model]) + + unmount() + preview.resolve(null) + await Promise.resolve() + + expect(generateModelThumbnail).not.toHaveBeenCalled() + }) + + it('settles a rejected preview lookup into the bounded retry path', async () => { + isAssetPreviewSupported.mockReturnValue(true) + findServerPreviewUrl + .mockRejectedValueOnce(new Error('preview lookup failed')) + .mockResolvedValueOnce('https://x/mesh_preview.png') + renderGroup([model]) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + + await vi.advanceTimersByTimeAsync(2_000) + + expect(findServerPreviewUrl).toHaveBeenCalledTimes(2) + expect(screen.getByRole('img', { name: 'mesh.glb' })).toHaveAttribute( + 'src', + 'https://x/mesh_preview.png' + ) + }) + + it('cancels a pending retry when the viewer close finds a server preview', async () => { + isAssetPreviewSupported.mockReturnValue(true) + renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + + findServerPreviewUrl.mockResolvedValue('https://x/mesh_preview.png') + await userEvent.click(screen.getByRole('button', { name: 'mesh.glb' })) + const dialog = vi.mocked(useDialogStore().showDialog).mock.calls.at(-1)?.[0] + const onClose = dialog?.dialogComponentProps?.onClose + if (typeof onClose !== 'function') throw new Error('Expected onClose') + onClose() + await vi.advanceTimersByTimeAsync(0) + + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(10_000) + expect(generateModelThumbnail).toHaveBeenCalledOnce() + expect(screen.getByRole('img', { name: 'mesh.glb' })).toHaveAttribute( + 'src', + 'https://x/mesh_preview.png' + ) + }) + + it('retries a model deferred by queue backpressure', async () => { + isAssetPreviewSupported.mockReturnValue(true) + generateModelThumbnail + .mockResolvedValueOnce({ status: 'busy' }) + .mockResolvedValueOnce({ + status: 'rendered', + dataUrl: 'data:image/png;base64,retried' + }) + renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + + await vi.advanceTimersByTimeAsync(2_000) + + expect(generateModelThumbnail).toHaveBeenCalledTimes(2) + expect(screen.getByRole('img', { name: 'mesh.glb' })).toHaveAttribute( + 'src', + 'data:image/png;base64,retried' + ) + }) + + it('bounds sustained queue backpressure with exponential retries and one server lookup', async () => { + vi.setSystemTime(0) + isAssetPreviewSupported.mockReturnValue(true) + generateModelThumbnail.mockResolvedValue({ status: 'busy' }) + renderGroup([model]) + await vi.advanceTimersByTimeAsync(0) + expect(generateModelThumbnail).toHaveBeenCalledOnce() + + const retryDelays = [2, 4, 8, 16, 32, 64, 128, 256, 480] + for (const [index, delaySeconds] of retryDelays.entries()) { + await vi.advanceTimersByTimeAsync(delaySeconds * 1000 - 1) + expect(generateModelThumbnail).toHaveBeenCalledTimes(index + 1) + await vi.advanceTimersByTimeAsync(1) + expect(generateModelThumbnail).toHaveBeenCalledTimes(index + 2) + } + + expect(findServerPreviewUrl).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + + it('cancels a backpressure retry when hidden and restarts cleanly when shown', async () => { + isAssetPreviewSupported.mockReturnValue(true) + generateModelThumbnail.mockResolvedValue({ status: 'busy' }) + const { rerender } = renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + await rerender({ assets: [audio] }) + await vi.advanceTimersByTimeAsync(60_000) + expect(generateModelThumbnail).toHaveBeenCalledOnce() + + await rerender({ assets: [model] }) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledTimes(2) + ) + expect(findServerPreviewUrl).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(991_000) + expect(generateModelThumbnail).toHaveBeenCalledTimes(10) + expect(vi.getTimerCount()).toBe(0) + }) + + it('cancels a backpressure retry when unmounted', async () => { + isAssetPreviewSupported.mockReturnValue(true) + generateModelThumbnail.mockResolvedValue({ status: 'busy' }) + const { unmount } = renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + unmount() + await vi.advanceTimersByTimeAsync(60_000) + expect(generateModelThumbnail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + + it('bounds retries to exactly the initial attempt plus MAX_THUMBNAIL_RETRIES', async () => { + isAssetPreviewSupported.mockReturnValue(true) + renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + + await vi.advanceTimersByTimeAsync(2_000) + expect(generateModelThumbnail).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(2_000) + expect(generateModelThumbnail).toHaveBeenCalledTimes(3) + + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(10_000) + expect(generateModelThumbnail).toHaveBeenCalledTimes(3) + expect(screen.queryByRole('img', { name: 'mesh.glb' })).toBeNull() + }) + + it('cancels a pending retry timer when the asset is hidden', async () => { isAssetPreviewSupported.mockReturnValue(true) - generateModelThumbnail.mockImplementationOnce( - async (_url, _filename, signal) => - new Promise((resolve) => { - signal?.addEventListener( - 'abort', - () => resolve({ status: 'cancelled' }), - { once: true } - ) - }) + const { rerender } = renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() ) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) - const first = renderGroup([model]) + await rerender({ assets: [audio] }) + + await vi.advanceTimersByTimeAsync(10_000) + expect(generateModelThumbnail).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + + it('preserves the spent retry attempt when a pending retry is hidden', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const { rerender } = renderGroup([model]) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledOnce() + ) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + + await rerender({ assets: [audio] }) + await rerender({ assets: [model] }) + await vi.waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledTimes(2) + ) + await vi.advanceTimersByTimeAsync(2_000) + + expect(generateModelThumbnail).toHaveBeenCalledTimes(3) + expect(vi.getTimerCount()).toBe(0) + }) + + it('aborts queued generation when unmounted', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const generation = deferred<{ + status: 'rendered' + dataUrl: string + }>() + generateModelThumbnail.mockReturnValueOnce(generation.promise) + const { unmount } = renderGroup([model]) await waitFor(() => expect(generateModelThumbnail).toHaveBeenCalledOnce()) - const firstSignal = generateModelThumbnail.mock.calls[0][2] - first.unmount() + const [, , signal] = generateModelThumbnail.mock.calls[0] + expect(signal?.aborted).toBe(false) - expect(firstSignal).toBeInstanceOf(AbortSignal) - expect(firstSignal?.aborted).toBe(true) + unmount() - generateModelThumbnail.mockResolvedValueOnce({ + expect(signal?.aborted).toBe(true) + generation.resolve({ status: 'rendered', - dataUrl: 'data:image/png;base64,remounted' + dataUrl: 'data:image/png;base64,x' }) - renderGroup([model]) + }) - const thumbnail = await screen.findByRole('img', { name: 'mesh.glb' }) - const secondSignal = generateModelThumbnail.mock.calls[1][2] - expect(secondSignal).toBeInstanceOf(AbortSignal) - expect(secondSignal).not.toBe(firstSignal) - expect(secondSignal?.aborted).toBe(false) - expect(thumbnail).toHaveAttribute('src', 'data:image/png;base64,remounted') + it('clears a pending thumbnail refresh when unmounted', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const { unmount } = renderGroup([model]) + await vi.waitFor(() => expect(findServerPreviewUrl).toHaveBeenCalled()) + await vi.waitFor(() => expect(vi.getTimerCount()).toBe(1)) + await userEvent.click(screen.getByRole('button', { name: 'mesh.glb' })) + const dialog = vi.mocked(useDialogStore().showDialog).mock.calls.at(-1)?.[0] + const onClose = dialog?.dialogComponentProps?.onClose + if (typeof onClose !== 'function') throw new Error('Expected onClose') + onClose() + await vi.advanceTimersByTimeAsync(0) + const callsBeforeUnmount = findServerPreviewUrl.mock.calls.length + expect(vi.getTimerCount()).toBe(1) + + unmount() + expect(vi.getTimerCount()).toBe(0) + + await vi.advanceTimersByTimeAsync(2_000) + expect(findServerPreviewUrl).toHaveBeenCalledTimes(callsBeforeUnmount) }) it('refreshes the tile thumbnail after the viewer closes', async () => { @@ -360,16 +550,59 @@ describe('ReplyAssetGroup', () => { findServerPreviewUrl.mockResolvedValue('https://x/mesh_preview.png') const dialog = vi.mocked(useDialogStore().showDialog).mock.calls.at(-1)?.[0] const onClose = dialog?.dialogComponentProps?.onClose - expect(onClose).toBeTypeOf('function') - onClose!() + if (typeof onClose !== 'function') throw new Error('Expected onClose') + onClose() const thumb = await screen.findByRole('img', { name: 'mesh.glb' }) expect(thumb).toHaveAttribute('src', 'https://x/mesh_preview.png') }) + it('restarts thumbnail generation when the viewer creates no server preview', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const generation = deferred() + generateModelThumbnail.mockReturnValueOnce(generation.promise) + renderGroup([model]) + await waitFor(() => expect(generateModelThumbnail).toHaveBeenCalledOnce()) + const [, , initialSignal] = generateModelThumbnail.mock.calls[0] + + await userEvent.click(screen.getByRole('button', { name: 'mesh.glb' })) + const dialog = vi.mocked(useDialogStore().showDialog).mock.calls.at(-1)?.[0] + const onClose = dialog?.dialogComponentProps?.onClose + if (typeof onClose !== 'function') throw new Error('Expected onClose') + onClose() + + expect(initialSignal?.aborted).toBe(true) + await vi.advanceTimersByTimeAsync(2_000) + await waitFor(() => expect(generateModelThumbnail).toHaveBeenCalledTimes(2)) + }) + + it('cancels dialog refresh work when its model becomes hidden', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const { rerender } = renderGroup([model]) + await waitFor(() => expect(findServerPreviewUrl).toHaveBeenCalledOnce()) + await userEvent.click(screen.getByRole('button', { name: 'mesh.glb' })) + + const refresh = deferred() + findServerPreviewUrl.mockReturnValueOnce(refresh.promise) + const dialog = vi.mocked(useDialogStore().showDialog).mock.calls.at(-1)?.[0] + const onClose = dialog?.dialogComponentProps?.onClose + if (typeof onClose !== 'function') throw new Error('Expected onClose') + onClose() + await rerender({ assets: [audio] }) + + refresh.resolve('https://x/mesh_preview.png') + await Promise.resolve() + + expect(screen.queryByRole('img', { name: 'mesh.glb' })).toBeNull() + expect(findServerPreviewUrl).toHaveBeenCalledTimes(2) + }) + it('titles the 3D viewer with the resolved asset name', async () => { isAssetPreviewSupported.mockReturnValue(true) - findOutputAsset.mockResolvedValue({ name: '3d/ComfyUI_00001_.glb' }) + findOutputAsset.mockResolvedValue({ + id: 'model', + name: '3d/ComfyUI_00001_.glb' + }) renderGroup([model]) await waitFor(() => @@ -388,12 +621,79 @@ describe('ReplyAssetGroup', () => { expect(thumbs()).toHaveLength(12) expect(toggle()).toHaveTextContent('Show more') - await userEvent.click(toggle()!) + await userEvent.click(toggle()) expect(thumbs()).toHaveLength(13) expect(toggle()).toHaveTextContent('Show less') - await userEvent.click(toggle()!) + await userEvent.click(toggle()) expect(thumbs()).toHaveLength(12) expect(toggle()).toHaveTextContent('Show more') }) + + it('generates thumbnails only for currently visible 3D entries', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const models = Array.from({ length: 13 }, (_, n) => ({ + ...model, + url: `https://x/api/view?filename=mesh-${n}.glb`, + filename: `mesh-${n}.glb` + })) + renderGroup(models) + await waitFor(() => expect(findServerPreviewUrl).toHaveBeenCalledTimes(12)) + await waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledTimes(12) + ) + expect(generateModelThumbnail).not.toHaveBeenCalledWith( + 'https://x/api/view?filename=mesh-12.glb', + 'mesh-12.glb', + expect.any(AbortSignal) + ) + + await userEvent.click(toggle()) + await waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledWith( + 'https://x/api/view?filename=mesh-12.glb', + 'mesh-12.glb', + expect.any(AbortSignal) + ) + ) + expect(generateModelThumbnail).toHaveBeenCalledTimes(13) + }) + + it('aborts a hidden render and restarts it on re-render without leaving the tile permanently blank', async () => { + isAssetPreviewSupported.mockReturnValue(true) + const generation = deferred<{ + status: 'rendered' + dataUrl: string + }>() + generateModelThumbnail.mockReturnValueOnce(generation.promise) + const { rerender } = renderGroup([model]) + await waitFor(() => expect(generateModelThumbnail).toHaveBeenCalledOnce()) + const [, , firstSignal] = generateModelThumbnail.mock.calls[0] + + await rerender({ assets: [audio] }) + expect(firstSignal?.aborted).toBe(true) + + generation.resolve({ + status: 'rendered', + dataUrl: 'data:image/png;base64,x' + }) + await Promise.resolve() + + generateModelThumbnail.mockResolvedValueOnce({ + status: 'rendered', + dataUrl: 'data:image/png;base64,restarted' + }) + await rerender({ assets: [model] }) + await waitFor(() => + expect(generateModelThumbnail).toHaveBeenCalledWith( + model.url, + model.filename, + expect.any(AbortSignal) + ) + ) + const restartedCall = generateModelThumbnail.mock.calls.find( + (call) => call[0] === model.url && call[2] !== firstSignal + ) + expect(restartedCall).toBeDefined() + }) }) diff --git a/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.vue b/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.vue index 09072c1cd63..622a7e487fd 100644 --- a/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.vue +++ b/src/workbench/extensions/agent/components/agent/message/ReplyAssetGroup.vue @@ -2,6 +2,7 @@ import { computed, defineAsyncComponent, + markRaw, onBeforeUnmount, ref, watch @@ -9,6 +10,8 @@ import { import { useI18n } from 'vue-i18n' import { LOAD3D_VIEWER_DIALOG_PROPS } from '@/extensions/core/load3d/components/load3dViewerDialog' +import type { ModelThumbnailResult } from '@/extensions/core/load3d/components/modelThumbnail' +import { generateModelThumbnail } from '@/extensions/core/load3d/components/modelThumbnail' import Button from '@/components/ui/button/Button.vue' import { findOutputAsset, @@ -18,9 +21,17 @@ import { import { useDialogStore } from '@/stores/dialogStore' import { cn } from '@comfyorg/tailwind-utils' -import type { ReplyAsset } from '@/workbench/extensions/agent/utils/replyAssets' -import { replyAssetResultItem } from '@/workbench/extensions/agent/utils/replyAssets' -import ReplyAudioCard from './ReplyAudioCard.vue' +import type { + ReplyAsset, + ReplyVisualAsset as ReplyVisualAssetData +} from '@/workbench/extensions/agent/utils/replyAssets' +import { + isReplyAudioAsset, + isReplyVisualAsset, + replyAssetResultItem +} from '@/workbench/extensions/agent/utils/replyAssets' +import ReplyAudioAssetGroup from './ReplyAudioAssetGroup.vue' +import ReplyVisualAsset from './ReplyVisualAsset.vue' const { assets } = defineProps<{ assets: ReplyAsset[] }>() @@ -29,8 +40,8 @@ const { t } = useI18n() /* Three rows of the four-column grid, per DES-530. */ const COLLAPSED_COUNT = 12 -const visual = computed(() => assets.filter((asset) => asset.kind !== 'audio')) -const audio = computed(() => assets.filter((asset) => asset.kind === 'audio')) +const visual = computed(() => assets.filter(isReplyVisualAsset)) +const audio = computed(() => assets.filter(isReplyAudioAsset)) const expanded = ref(false) const collapsible = computed(() => visual.value.length > COLLAPSED_COUNT) @@ -70,78 +81,267 @@ const galleryItems = computed(() => ) const galleryIndex = ref(-1) -const modelThumbnails = ref>({}) -const assetNames = ref>({}) -const thumbnailAbortController = new AbortController() -const thumbnailRetryTimers = new Set>() +// Preserve controller identity inside Vue's deeply reactive state record. +type ThumbnailState = + | { + phase: 'loading' + controller: AbortController + attempts: number + busyAttempts: number + } + | { + phase: 'retryPending' + timeout: ReturnType + attempts: number + busyAttempts: number + } + | { phase: 'paused'; attempts: number; busyAttempts: number } + | { phase: 'ready'; src: string } + | { phase: 'gaveUp' } + +type ThumbnailLoadOptions = { + attempts?: number + busyAttempts?: number + checkServerPreview?: boolean + generateOnPreviewMiss?: boolean +} + +type ActiveThumbnailLoad = { + url: string + filename: string + controller: AbortController + attempts: number + busyAttempts: number + generateOnPreviewMiss: boolean +} + +const MAX_THUMBNAIL_RETRIES = 2 const THUMBNAIL_RETRY_DELAY_MS = 2000 -const THUMBNAIL_MAX_ATTEMPTS = 30 +const MAX_THUMBNAIL_BUSY_RETRIES = 9 +const MAX_THUMBNAIL_BUSY_DELAY_MS = 8 * 60_000 + +const thumbnailState = ref>({}) +const assetNames = ref>({}) +let mounted = true + +function cancelThumbnailState(state: ThumbnailState | undefined): void { + if (state?.phase === 'loading') { + state.controller.abort() + } else if (state?.phase === 'retryPending') { + clearTimeout(state.timeout) + } +} onBeforeUnmount(() => { - thumbnailAbortController.abort() - for (const timer of thumbnailRetryTimers) clearTimeout(timer) - thumbnailRetryTimers.clear() + mounted = false + for (const state of Object.values(thumbnailState.value)) { + cancelThumbnailState(state) + } }) -async function generateOffscreenModelThumbnail( +/** + * Whether an offscreen render can resolve `url` at all. The Load3D loader + * resolves model data through the `filename` query, so a URL without one is + * terminally unrenderable rather than transiently failing — retrying it would + * spend the whole budget on work that cannot succeed. + */ +function canRenderOffscreen(url: string): boolean { + return new URL(url, window.location.origin).searchParams.has('filename') +} + +/** Whether `url`'s current entry is still the `loading` strand owned by `controller`. */ +function owns(url: string, controller: AbortController): boolean { + const state = thumbnailState.value[url] + return state?.phase === 'loading' && state.controller === controller +} + +function loadModelThumbnail( url: string, filename: string, - attempt = 1 + { + attempts = 0, + busyAttempts = 0, + checkServerPreview = true, + generateOnPreviewMiss = true + }: ThumbnailLoadOptions = {} +): void { + const controller = markRaw(new AbortController()) + thumbnailState.value[url] = { + phase: 'loading', + controller, + attempts, + busyAttempts + } + + const previewLookup = checkServerPreview + ? findServerPreviewUrl(filename) + : Promise.resolve(null) + void previewLookup + .then((preview) => + handlePreviewLookup( + { + url, + filename, + controller, + attempts, + busyAttempts, + generateOnPreviewMiss + }, + preview + ) + ) + .catch(() => { + if (!mounted || !owns(url, controller)) return + scheduleThumbnailRetry(url, filename, attempts, busyAttempts) + }) +} + +async function handlePreviewLookup( + load: ActiveThumbnailLoad, + preview: string | null ): Promise { - const { generateModelThumbnail } = - await import('@/extensions/core/load3d/components/modelThumbnail') - const generated = await generateModelThumbnail( - url, - filename, - thumbnailAbortController.signal - ) - if (generated.status === 'rendered') { - modelThumbnails.value[url] = generated.dataUrl + const { url, filename, controller, attempts, busyAttempts } = load + if (!mounted || !owns(url, controller)) return + if (preview) { + thumbnailState.value[url] = { phase: 'ready', src: preview } + return + } + if (!load.generateOnPreviewMiss) { + scheduleThumbnailLoad(url, filename, { + attempts, + busyAttempts, + delay: THUMBNAIL_RETRY_DELAY_MS + }) return } + if (!canRenderOffscreen(url)) { + thumbnailState.value[url] = { phase: 'gaveUp' } + return + } + const result = await generateModelThumbnail(url, filename, controller.signal) + if (!mounted || !owns(url, controller)) return + publishThumbnailResult(load, result) +} + +function publishThumbnailResult( + load: ActiveThumbnailLoad, + result: ModelThumbnailResult +): void { + const { url, filename, attempts, busyAttempts } = load + if (result.status === 'rendered') { + thumbnailState.value[url] = { phase: 'ready', src: result.dataUrl } + } else if (result.status === 'failed' || result.status === 'busy') { + scheduleThumbnailRetry(url, filename, attempts, busyAttempts, result.status) + } else { + thumbnailState.value[url] = { phase: 'gaveUp' } + } +} + +function scheduleThumbnailLoad( + url: string, + filename: string, + { + attempts, + busyAttempts, + delay, + checkServerPreview = true + }: { + attempts: number + busyAttempts: number + delay: number + checkServerPreview?: boolean + } +): void { + const timeout = setTimeout(() => { + if (!mounted) return + loadModelThumbnail(url, filename, { + attempts, + busyAttempts, + checkServerPreview + }) + }, delay) + thumbnailState.value[url] = { + phase: 'retryPending', + timeout: markRaw(timeout), + attempts, + busyAttempts + } +} + +function scheduleThumbnailRetry( + url: string, + filename: string, + attempts: number, + busyAttempts = 0, + status: 'failed' | 'busy' = 'failed' +): void { if ( - generated.status !== 'busy' || - thumbnailAbortController.signal.aborted || - attempt >= THUMBNAIL_MAX_ATTEMPTS + (status === 'failed' && attempts >= MAX_THUMBNAIL_RETRIES) || + (status === 'busy' && busyAttempts >= MAX_THUMBNAIL_BUSY_RETRIES) ) { + thumbnailState.value[url] = { phase: 'gaveUp' } return } - const timer = setTimeout(() => { - thumbnailRetryTimers.delete(timer) - void generateOffscreenModelThumbnail(url, filename, attempt + 1).catch( - () => {} - ) - }, THUMBNAIL_RETRY_DELAY_MS) - thumbnailRetryTimers.add(timer) + const nextAttempts = status === 'failed' ? attempts + 1 : attempts + const nextBusyAttempts = status === 'busy' ? busyAttempts + 1 : busyAttempts + const delay = + status === 'busy' + ? Math.min( + THUMBNAIL_RETRY_DELAY_MS * 2 ** busyAttempts, + MAX_THUMBNAIL_BUSY_DELAY_MS + ) + : THUMBNAIL_RETRY_DELAY_MS + scheduleThumbnailLoad(url, filename, { + attempts: nextAttempts, + busyAttempts: nextBusyAttempts, + delay, + checkServerPreview: status !== 'busy' + }) +} + +function hideThumbnail(url: string): void { + const state = thumbnailState.value[url] + if (state?.phase === 'ready' || state?.phase === 'gaveUp') return + cancelThumbnailState(state) + thumbnailState.value[url] = { + phase: 'paused', + attempts: state && 'attempts' in state ? state.attempts : 0, + busyAttempts: state && 'busyAttempts' in state ? state.busyAttempts : 0 + } +} + +function showThumbnail(url: string, filename: string): void { + const state = thumbnailState.value[url] + if (!state) loadModelThumbnail(url, filename) + else if (state.phase === 'paused') { + loadModelThumbnail(url, filename, { + attempts: state.attempts, + busyAttempts: state.busyAttempts + }) + } } watch( - () => assets.filter((asset) => asset.kind === '3D' || asset.kind === 'audio'), + () => [ + ...visibleVisual.value.filter((asset) => asset.kind === '3D'), + ...visibleAudio.value + ], (lookups) => { if (!isAssetPreviewSupported()) return - for (const { url, filename, kind } of lookups) { - if (kind === '3D' && !(url in modelThumbnails.value)) { - modelThumbnails.value[url] = '' - void findServerPreviewUrl(filename) - .then(async (preview) => { - if (preview) { - modelThumbnails.value[url] = preview - return - } - if ( - !new URL(url, window.location.origin).searchParams.has('filename') - ) { - return - } - await generateOffscreenModelThumbnail(url, filename) - }) - .catch(() => {}) - } + const visible3D = lookups.filter((asset) => asset.kind === '3D') + const visibleUrls = new Set(visible3D.map((asset) => asset.url)) + for (const url of Object.keys(thumbnailState.value)) { + if (!visibleUrls.has(url)) hideThumbnail(url) + } + for (const { url, filename } of visible3D) { + showThumbnail(url, filename) + } + for (const { url, filename } of lookups) { if (!(url in assetNames.value)) { assetNames.value[url] = '' void findOutputAsset(filename) .then((record) => { - if (record?.name) assetNames.value[url] = record.name + if (mounted && record?.name) assetNames.value[url] = record.name }) .catch(() => {}) } @@ -157,18 +357,21 @@ const MediaLightbox = defineAsyncComponent( () => import('@/components/sidebar/tabs/queue/MediaLightbox.vue') ) -function refreshModelThumbnail(asset: ReplyAsset, retry = true): void { - if (!isAssetPreviewSupported() || modelThumbnails.value[asset.url]) return - void findServerPreviewUrl(asset.filename).then((preview) => { - if (preview) { - modelThumbnails.value[asset.url] = preview - } else if (retry) { - setTimeout(() => refreshModelThumbnail(asset, false), 2000) - } +function refreshModelThumbnail(asset: ReplyAsset): void { + const state = thumbnailState.value[asset.url] + if (!mounted || !isAssetPreviewSupported() || state?.phase === 'ready') return + cancelThumbnailState(state) + loadModelThumbnail(asset.url, asset.filename, { + generateOnPreviewMiss: false }) } -function inspect(asset: ReplyAsset): void { +function modelThumbnailSrc(url: string): string { + const state = thumbnailState.value[url] + return state?.phase === 'ready' ? state.src : '' +} + +function inspect(asset: ReplyVisualAssetData): void { if (asset.kind === '3D') { useDialogStore().showDialog({ key: 'asset-3d-viewer', @@ -184,86 +387,19 @@ function inspect(asset: ReplyAsset): void { } galleryIndex.value = galleryAssets.value.indexOf(asset) } - -function playPreview(event: Event): void { - const video = event.target - if (video instanceof HTMLVideoElement) void video.play().catch(() => {}) -} - -function stopPreview(event: Event): void { - const video = event.target - if (video instanceof HTMLVideoElement) video.pause() -}