diff --git a/locales/en.json b/locales/en.json index b79621f53..d282e9ad8 100644 --- a/locales/en.json +++ b/locales/en.json @@ -57,6 +57,27 @@ "loadFailed": "Couldn't load assets", "download": "Download", "downloadFailed": "Couldn't download asset", + "inspect": "Inspect {name}", + "kindSingular": { + "image": "Image", + "video": "Video", + "audio": "Audio", + "3d": "3D", + "file": "File" + }, + "detail": { + "tags": "Tags", + "tagsHint": "Tags are edited in ComfyUI.", + "createdAt": "Created {date}", + "fit": "Fit", + "zoom": "Zoom", + "zoomLevel": "Zoom: {level}", + "generationInfo": "Generation info", + "workflow": "Workflow", + "available": "Available", + "notSaved": "Not saved", + "openWorkflow": "Open workflow" + }, "noMatches": "No matching assets", "noMatchesHint": "Try another type or search.", "filter": "Filter", diff --git a/locales/zh.json b/locales/zh.json index fec143b83..071b5331e 100644 --- a/locales/zh.json +++ b/locales/zh.json @@ -57,6 +57,27 @@ "loadFailed": "无法加载资产", "download": "下载", "downloadFailed": "无法下载资产", + "inspect": "查看 {name}", + "kindSingular": { + "image": "图片", + "video": "视频", + "audio": "音频", + "3d": "3D", + "file": "文件" + }, + "detail": { + "tags": "标签", + "tagsHint": "标签在 ComfyUI 中编辑。", + "createdAt": "创建于 {date}", + "fit": "适应", + "zoom": "缩放", + "zoomLevel": "缩放:{level}", + "generationInfo": "生成信息", + "workflow": "工作流", + "available": "可用", + "notSaved": "未保存", + "openWorkflow": "打开工作流" + }, "noMatches": "没有匹配的资产", "noMatchesHint": "试试其他类型或搜索词。", "filter": "筛选", diff --git a/src/main/apps/appsRun.test.ts b/src/main/apps/appsRun.test.ts index d2984ed67..811bdc326 100644 --- a/src/main/apps/appsRun.test.ts +++ b/src/main/apps/appsRun.test.ts @@ -16,6 +16,7 @@ const pages = vi.hoisted(() => { loadURL: () => Promise.resolve(), isDestroyed: () => false, getURL: () => wc.mainFrame.url, + focus: () => {}, close: () => {} } made.push(wc) @@ -42,6 +43,9 @@ vi.mock('electron', () => ({ setBorderRadius() {} setBounds() {} setVisible() {} + getVisible() { + return false + } } })) diff --git a/src/main/apps/appsView.test.ts b/src/main/apps/appsView.test.ts index fa89602c4..5db09b0c4 100644 --- a/src/main/apps/appsView.test.ts +++ b/src/main/apps/appsView.test.ts @@ -19,6 +19,7 @@ vi.mock('electron', () => { setBorderRadius = vi.fn() setBounds = vi.fn() setVisible = vi.fn() + getVisible = vi.fn(() => false) constructor() { createdViews.push(this) } diff --git a/src/main/apps/appsView.ts b/src/main/apps/appsView.ts index 489552c91..641e3995f 100644 --- a/src/main/apps/appsView.ts +++ b/src/main/apps/appsView.ts @@ -55,6 +55,7 @@ import { syncLocalSocket } from './localComfy' import { listCatalogApps } from './catalog' +import { outputAction, outputRequest } from './outputs' import { localBuilds, openInComfy } from './openInComfy' import { isAppsEnabled, onAppsEnabledChanged } from './appsSwitch' import { @@ -80,9 +81,15 @@ export const APPS_APP_OPEN_CHANNEL = 'comfy-apps:app-open' export const APPS_SETTLED_CHANNEL = 'comfy-apps:settled' export const APPS_PAGE_SETTLED_CHANNEL = 'comfy-apps:page-settled' export const APPS_CATALOG_CHANNEL = 'comfy-apps:catalog' +export const APPS_OUTPUT_CHANNEL = 'comfy-apps:output' +export const APPS_OUTPUT_ACTION_CHANNEL = 'comfy-apps:output-action' +const BACKDROP_QUALITY = 80 export const APPS_ENABLED_CHANNEL = 'comfy-apps:enabled' const appOpen = new Map() +const capturingOutputs = new Set() +const pageDocuments = new Map() +const liveOutputCounts = new Map() const awaitingCommit = new Set() const settledPages = new Set() @@ -181,6 +188,8 @@ function createAppsView(key: number, url: string): WebContentsView { if (isMainFrame && isPageLoadFailure(errorCode)) markLoadFailed(key, view, url) }) wc.on('did-navigate', (_event, url, httpResponseCode) => { + pageDocuments.set(key, (pageDocuments.get(key) ?? 0) + 1) + liveOutputCounts.delete(key) awaitingCommit.delete(key) settledPages.delete(key) if (isDocumentFailure(httpResponseCode)) markLoadFailed(key, view, url) @@ -242,6 +251,8 @@ function showApps(event: IpcMainEvent, rect: unknown): void { entry.window.once('closed', () => { views.delete(key) appOpen.delete(key) + liveOutputCounts.delete(key) + pageDocuments.delete(key) awaitingCommit.delete(key) settledPages.delete(key) if (!view!.webContents.isDestroyed()) view!.webContents.close() @@ -256,8 +267,12 @@ function showApps(event: IpcMainEvent, rect: unknown): void { height: Math.round(r.height!) }) entry.window.contentView.addChildView(view) + const uncovered = !view.getVisible() view.setVisible(true) - if (r.fresh !== true) return + if (r.fresh !== true) { + if (uncovered) view.webContents.focus() + return + } tellPanelAppOpen(key) if (settledPages.has(key)) tellPanelSettled(key) } @@ -285,6 +300,53 @@ function appsPageUrl(event: IpcMainInvokeEvent): string | null { return isAppsFrameUrl(frame.url, APPS_URL) ? frame.url : null } +function keyForAppsPage(sender: WebContents): number | null { + for (const [key, view] of views) if (view.webContents === sender) return key + return null +} + +async function captureBackdrop(page: WebContents): Promise { + try { + const jpeg = (await page.capturePage()).toJPEG(BACKDROP_QUALITY) + return jpeg.byteLength > 0 ? jpeg : null + } catch { + return null + } +} + +function isOnScreen(key: number): boolean { + return views.get(key)?.getVisible() === true +} + +async function showOutput(page: WebContents, request: unknown): Promise { + const outputs = outputRequest(request) + if (!outputs) throw new Error('These outputs cannot be shown.') + const key = keyForAppsPage(page) + const entry = key === null ? undefined : comfyWindows.get(key) + const panel = entry?.panelView?.webContents + if (key === null || !appOpen.get(key) || !panel || panel.isDestroyed()) return + if (!isOnScreen(key)) throw new Error('The app is not on screen.') + if (capturingOutputs.has(key)) throw new Error('Outputs are already opening.') + const pageDocument = pageDocuments.get(key) + capturingOutputs.add(key) + const backdrop = await captureBackdrop(page).finally(() => capturingOutputs.delete(key)) + if (panel.isDestroyed() || !appOpen.get(key) || pageDocuments.get(key) !== pageDocument) return + if (!isOnScreen(key)) throw new Error('The app is not on screen.') + liveOutputCounts.set(key, outputs.items.length) + if (entry?.window.isFocused()) panel.focus() + panel.send(APPS_OUTPUT_CHANNEL, { ...outputs, backdrop }) +} + +function relayOutputAction(event: IpcMainEvent, input: unknown): void { + const found = entryForPanel(event.sender) + const action = outputAction(input) + if (!found || !action) return + const count = liveOutputCounts.get(found[0]) ?? 0 + const page = views.get(found[0])?.webContents + if (action.index >= count || !page || page.isDestroyed()) return + page.send(APPS_OUTPUT_ACTION_CHANNEL, action) +} + function authState(): AppsAuthState { const status = getCloudSession().status() return status.signedIn ? { signedIn: true, email: status.email } : { signedIn: false } @@ -687,6 +749,16 @@ async function uploadImage( return { name: body.name } } +function handleFromAppsView( + channel: string, + fn: (page: WebContents, ...args: A) => Promise | R +): void { + ipcMain.handle(channel, async (event, ...args) => { + if (appsPageUrl(event) === null) throw new Error('Not allowed.') + return fn(event.sender, ...(args as A)) + }) +} + function handleFromAppsPage( channel: string, fn: (pageUrl: string, ...args: A) => Promise | R @@ -718,6 +790,7 @@ export function registerAppsIpc(): void { ipcMain.handle(APPS_CATALOG_CHANNEL, (event) => entryForPanel(event.sender) ? listCatalogApps(APPS_URL) : [] ) + ipcMain.on(APPS_OUTPUT_ACTION_CHANNEL, relayOutputAction) onAppsEnabledChanged(tellPanelsAppsEnabled) initLocalComfy(appsPages) @@ -750,5 +823,6 @@ export function registerAppsIpc(): void { localDownload(req, appOfPage(pageUrl, APPS_URL)) ) handleFromApps('comfy-apps:open-in-comfy', openInComfy) + handleFromAppsView('comfy-apps:show-output', showOutput) handleFromApps('comfy-apps:track', trackFromPage) } diff --git a/src/main/apps/outputs.test.ts b/src/main/apps/outputs.test.ts new file mode 100644 index 000000000..272ae4176 --- /dev/null +++ b/src/main/apps/outputs.test.ts @@ -0,0 +1,75 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest' + +import { MAX_OUTPUTS, MAX_OUTPUTS_BYTES, outputAction, outputRequest } from './outputs' + +const picture = (extra: Record = {}) => ({ + name: 'picture-1.png', + mimeType: 'image/png', + bytes: new Uint8Array([1, 2, 3]), + ...extra +}) + +describe('outputRequest', () => { + it('keeps the outputs and the one to show first', () => { + const request = outputRequest({ + items: [ + picture({ tags: ['This computer', 7, ' '], workflow: true }), + picture({ name: 'clip.mp4', mimeType: 'video/mp4' }) + ], + index: 1 + }) + expect(request?.index).toBe(1) + expect(request?.items[0]).toMatchObject({ + name: 'picture-1.png', + tags: ['This computer'], + workflow: true + }) + expect(request?.items[1]).toMatchObject({ workflow: false, tags: [] }) + }) + + it('refuses outputs it cannot show', () => { + expect(outputRequest(null)).toBeNull() + expect(outputRequest({ items: [], index: 0 })).toBeNull() + expect(outputRequest({ items: [picture()], index: 1 })).toBeNull() + expect(outputRequest({ items: [picture()], index: -1 })).toBeNull() + expect(outputRequest({ items: [picture({ mimeType: 'text/html' })], index: 0 })).toBeNull() + expect(outputRequest({ items: [picture({ mimeType: 'image/svg+xml' })], index: 0 })).toBeNull() + expect(outputRequest({ items: [picture({ bytes: 'abc' })], index: 0 })).toBeNull() + expect(outputRequest({ items: [picture({ name: ' ' })], index: 0 })).toBeNull() + expect( + outputRequest({ items: Array.from({ length: MAX_OUTPUTS + 1 }, () => picture()), index: 0 }) + ).toBeNull() + const half = new Uint8Array(MAX_OUTPUTS_BYTES / 2 + 1) + expect( + outputRequest({ items: [picture({ bytes: half }), picture({ bytes: half })], index: 0 }) + ).toBeNull() + }) + + it('refuses an items list with holes', () => { + const items = [picture(), picture()] + delete items[0] + expect(outputRequest({ items, index: 1 })).toBeNull() + }) + + it('counts the whole buffer behind each output', () => { + const view = new Uint8Array(new ArrayBuffer(MAX_OUTPUTS_BYTES + 1), 0, 1) + expect(outputRequest({ items: [picture({ bytes: view })], index: 0 })).toBeNull() + }) +}) + +describe('outputAction', () => { + it('passes on a download or an open-workflow for one output', () => { + expect(outputAction({ action: 'download', index: 2 })).toEqual({ action: 'download', index: 2 }) + expect(outputAction({ action: 'open-workflow', index: 0 })).toEqual({ + action: 'open-workflow', + index: 0 + }) + }) + + it('drops anything else', () => { + expect(outputAction({ action: 'delete', index: 0 })).toBeNull() + expect(outputAction({ action: 'download', index: 1.5 })).toBeNull() + expect(outputAction('download')).toBeNull() + }) +}) diff --git a/src/main/apps/outputs.ts b/src/main/apps/outputs.ts new file mode 100644 index 000000000..8d7ed0937 --- /dev/null +++ b/src/main/apps/outputs.ts @@ -0,0 +1,55 @@ +import type { AppOutputAction, AppOutputItem } from '../../types/ipc' + +export const MAX_OUTPUTS = 40 +export const MAX_OUTPUTS_BYTES = 100_000_000 +const MAX_TAGS = 8 +const MEDIA_TYPE = + /^(image\/(png|jpeg|webp|gif|avif)|video\/(mp4|webm|quicktime)|audio\/(mpeg|mp4|wav|x-wav|ogg|webm|flac|aac))$/ + +function text(value: unknown, max: number): string | null { + return typeof value === 'string' && value.trim() !== '' ? value.trim().slice(0, max) : null +} + +function outputItem(value: unknown): AppOutputItem | null { + if (!value || typeof value !== 'object') return null + const item = value as Record + const name = text(item['name'], 200) + const mimeType = item['mimeType'] + const bytes = item['bytes'] + if (!name || typeof mimeType !== 'string' || !MEDIA_TYPE.test(mimeType)) return null + if (!(bytes instanceof Uint8Array)) return null + const tags = Array.isArray(item['tags']) + ? item['tags'] + .slice(0, MAX_TAGS) + .map((tag) => text(tag, 40)) + .filter((tag): tag is string => tag !== null) + : [] + return { + name, + mimeType, + bytes: bytes as Uint8Array, + tags, + workflow: item['workflow'] === true + } +} + +export function outputRequest(input: unknown): { items: AppOutputItem[]; index: number } | null { + if (!input || typeof input !== 'object') return null + const { items, index } = input as { items?: unknown; index?: unknown } + if (!Array.isArray(items) || items.length === 0 || items.length > MAX_OUTPUTS) return null + const parsed = Array.from(items, outputItem) + if (parsed.some((item) => item === null)) return null + const outputs = parsed as AppOutputItem[] + const total = outputs.reduce((sum, item) => sum + item.bytes.buffer.byteLength, 0) + if (total > MAX_OUTPUTS_BYTES) return null + if (typeof index !== 'number' || !Number.isInteger(index) || index < 0) return null + return index < outputs.length ? { items: outputs, index } : null +} + +export function outputAction(input: unknown): AppOutputAction | null { + if (!input || typeof input !== 'object') return null + const { action, index } = input as { action?: unknown; index?: unknown } + if (action !== 'download' && action !== 'open-workflow') return null + if (typeof index !== 'number' || !Number.isInteger(index) || index < 0) return null + return { action, index } +} diff --git a/src/main/apps/wiring.test.ts b/src/main/apps/wiring.test.ts index 6c7daf124..2360716cd 100644 --- a/src/main/apps/wiring.test.ts +++ b/src/main/apps/wiring.test.ts @@ -16,7 +16,7 @@ const exposed = all( /^ {2}(\w+):/gm ) const invoked = all(preload, /ipcRenderer\.invoke\('([\w:-]+)'/g) -const handled = all(main, /handleFromApps(?:Page)?\('([\w:-]+)'/g) +const handled = all(main, /handleFromApps(?:Page|View)?\('([\w:-]+)'/g) describe('Apps bridge wiring', () => { it('getInfo names exactly the calls the page gets', () => { @@ -45,7 +45,9 @@ describe('Apps bridge wiring', () => { }) it('the only handler outside the apps-page check answers the panel alone', () => { - expect(all(main, /ipcMain\.handle\((\w+)/g)).toEqual(['channel', 'APPS_CATALOG_CHANNEL']) + expect(new Set(all(main, /ipcMain\.handle\((\w+)/g))).toEqual( + new Set(['channel', 'APPS_CATALOG_CHANNEL']) + ) expect(main).toMatch( /ipcMain\.handle\(APPS_CATALOG_CHANNEL, \(event\) =>\s+entryForPanel\(event\.sender\)/ ) @@ -57,6 +59,13 @@ describe('Apps bridge wiring', () => { expect(main).toContain('ipcMain.on(APPS_PAGE_SETTLED_CHANNEL, relayPageSettled)') }) + it('output actions reach the page on the channel the preload listens to', () => { + const action = main.match(/APPS_OUTPUT_ACTION_CHANNEL = '([\w:-]+)'/)![1] + expect(preload).toContain(`ipcRenderer.on('${action}'`) + expect(preload).toContain(`ipcRenderer.removeListener('${action}'`) + expect(main).toContain('ipcMain.on(APPS_OUTPUT_ACTION_CHANNEL, relayOutputAction)') + }) + it('a Cloud run takes only the model from the page and the app from its address', () => { expect(preload).toMatch(/const BRIDGE = 3\n/) expect(preload).toContain('run: (workflow: unknown, options?: { model?: string }) =>') diff --git a/src/preload/api.ts b/src/preload/api.ts index 30e759532..7119a9189 100644 --- a/src/preload/api.ts +++ b/src/preload/api.ts @@ -116,6 +116,13 @@ export function buildElectronApi(): ElectronApi { ipcRenderer.on('comfy-apps:app-open', handler) return () => ipcRenderer.removeListener('comfy-apps:app-open', handler) }, + onAppsOutput: (callback) => { + const handler = (_event: IpcRendererEvent, outputs: unknown) => + callback(outputs as Parameters[0]) + ipcRenderer.on('comfy-apps:output', handler) + return () => ipcRenderer.removeListener('comfy-apps:output', handler) + }, + appsOutputAction: (action) => ipcRenderer.send('comfy-apps:output-action', action), onAppsSettled: (callback) => { const handler = () => callback() ipcRenderer.on('comfy-apps:settled', handler) diff --git a/src/preload/appsPreload.ts b/src/preload/appsPreload.ts index 24996f67b..f2e8205ed 100644 --- a/src/preload/appsPreload.ts +++ b/src/preload/appsPreload.ts @@ -28,6 +28,8 @@ const CALLS = [ 'localDownload', 'cancelRun', 'openInComfy', + 'showOutput', + 'onOutputAction', 'track' ] @@ -73,6 +75,14 @@ contextBridge.exposeInMainWorld('comfyDesktopApps', { ipcRenderer.invoke('comfy-apps:local-fetch', req), localDownload: (req: { url: string; folder: string; name: string }) => ipcRenderer.invoke('comfy-apps:local-download', req), + showOutput: (request: unknown) => ipcRenderer.invoke('comfy-apps:show-output', request), + onOutputAction: (callback: (action: unknown) => void) => { + const listener = (_e: IpcRendererEvent, msg: unknown) => callback(msg) + ipcRenderer.on('comfy-apps:output-action', listener) + return () => { + ipcRenderer.removeListener('comfy-apps:output-action', listener) + } + }, track: (name: string, fields?: Record): void => { try { ipcRenderer.invoke('comfy-apps:track', name, fields).catch(() => {}) diff --git a/src/renderer/csp.test.ts b/src/renderer/csp.test.ts index 9c9e0d0a1..9603f3164 100644 --- a/src/renderer/csp.test.ts +++ b/src/renderer/csp.test.ts @@ -58,6 +58,11 @@ describe('Content-Security-Policy: panel.html', () => { it('allows GitHub-hosted starter-template thumbnails in img-src', () => { expect(csp['img-src']).toContain('https://raw.githubusercontent.com') }) + + it('shows app outputs that the panel holds as blob URLs', () => { + expect(csp['img-src']).toContain('blob:') + expect(csp['media-src']).toContain('blob:') + }) }) describe.each(TELEMETRY_RENDERER_HTMLS)( diff --git a/src/renderer/panel.html b/src/renderer/panel.html index 7cbc5d810..40445fbe3 100644 --- a/src/renderer/panel.html +++ b/src/renderer/panel.html @@ -5,7 +5,7 @@ Comfy Desktop — Panel