diff --git a/browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts b/browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts index 38f99ae7e8a..d284b59a80e 100644 --- a/browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts +++ b/browser_tests/tests/agent/agentAttachFileTypeParity.spec.ts @@ -42,11 +42,11 @@ test.describe( await expect(agentPanel.fileInput).toHaveAttribute( 'accept', - /(^|,)video\/\*(,|$)/ + /(^|,)\.mp4(,|$)/ ) await expect(agentPanel.fileInput).toHaveAttribute( 'accept', - /(^|,)audio\/\*(,|$)/ + /(^|,)\.wav(,|$)/ ) await agentPanel.fileInput.setInputFiles([ @@ -123,8 +123,6 @@ test.describe( ) expect(dropped).not.toBe('ignored') - - test.fail() expect( dropped, `the file browser answered "${picked}"; a drop of the same file must answer the same` diff --git a/browser_tests/tests/agent/agentAttachFileTypes.spec.ts b/browser_tests/tests/agent/agentAttachFileTypes.spec.ts new file mode 100644 index 00000000000..4c6859768e4 --- /dev/null +++ b/browser_tests/tests/agent/agentAttachFileTypes.spec.ts @@ -0,0 +1,289 @@ +import { expect } from '@playwright/test' + +import type { UploadImageResponse } from '@comfyorg/ingest-types' + +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { + AGENT_ATTACH_ACCEPT, + agentAttachCapability +} from '@/workbench/extensions/agent/utils/attachableFiles' +import type { AgentAttachCapability } from '@/workbench/extensions/agent/utils/attachableFiles' + +import { agentTest as test } from '@e2e/tests/agent/agentPanelMocks' +import { assetPath } from '@e2e/fixtures/utils/paths' +import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' + +const ACCEPTED: ReadonlyArray<{ + tier: AgentAttachCapability + file: string +}> = [ + { tier: 'view', file: 'photo.png' }, + { tier: 'probe', file: 'clip.mp4' }, + { tier: 'read', file: 'notes.md' }, + { tier: 'reference', file: 'mesh.glb' }, + { tier: 'retain', file: 'photo.avif' } +] + +const REJECTED = [ + 'page.html', + 'bundle.js', + 'style.css', + 'doc.pdf', + 'clip.wmv', + 'clip.flv', + 'scene.usdz', + 'env.hdr', + 'track.aac', + 'track.aiff', + 'track.wma', + 'plate.exr' +] as const + +const VIDEO_HEADERS: Partial> = { + mp4: Buffer.from('\0\0\0\x18ftypisom', 'latin1') +} + +function syntheticFile(name: string) { + const extension = name.slice(name.lastIndexOf('.') + 1) + return { + name, + mimeType: 'application/octet-stream', + buffer: VIDEO_HEADERS[extension] ?? Buffer.from(name) + } +} + +function unsupportedTypeMessage(name: string): string { + return enMessages.agent.attachmentUnsupportedType.replace('{name}', name) +} + +test.describe( + 'Agent composer accepted file types', + { tag: ['@cloud', '@ui'] }, + () => { + test.beforeEach(async ({ comfyPage }) => { + await comfyPage.page.route('**/api/upload/image', (route) => { + const response: UploadImageResponse = { + name: 'uploaded', + subfolder: '', + type: 'input' + } + return route.fulfill(jsonRoute(response)) + }) + await comfyPage.nodeOps.clearGraph() + await expect.poll(() => comfyPage.nodeOps.getGraphNodesCount()).toBe(0) + }) + + test('advertises extensions only, so the OS picker cannot offer a rejected type', async ({ + agentPanel + }) => { + await agentPanel.open() + + await expect(agentPanel.fileInput).toHaveAttribute( + 'accept', + AGENT_ATTACH_ACCEPT + ) + expect(AGENT_ATTACH_ACCEPT).not.toContain('/*') + for (const rejected of REJECTED) { + expect( + AGENT_ATTACH_ACCEPT.split(','), + `${rejected} must not be offered by the picker` + ).not.toContain(rejected.slice(rejected.lastIndexOf('.'))) + } + }) + + for (const { tier, file } of ACCEPTED) { + test(`uploads a ${tier}-tier file (${file}) and leaves no status on its chip`, async ({ + agentPanel, + comfyPage + }) => { + await agentPanel.open() + const upload = comfyPage.page.waitForRequest( + (request) => + request.method() === 'POST' && + request.url().endsWith('/api/upload/image') + ) + + await agentPanel.fileInput.setInputFiles(syntheticFile(file)) + + const chip = agentPanel.attachmentChip(file) + await expect(chip).toBeVisible() + await expect( + chip.getByRole('status'), + `${file} must finish uploading with no validation status` + ).toHaveCount(0) + expect((await upload).postDataBuffer()?.toString('latin1')).toContain( + `filename="${file}"` + ) + }) + } + + test('stages every rejected format as an invalid chip that is never uploaded', async ({ + agentPanel, + comfyPage + }) => { + const uploads: string[] = [] + comfyPage.page.on('request', (request) => { + if (request.url().includes('/api/upload/image')) + uploads.push(request.url()) + }) + await agentPanel.open() + + for (const file of REJECTED) { + expect( + agentAttachCapability(file), + `${file} must not be on any tier` + ).toBeUndefined() + + await agentPanel.fileInput.setInputFiles(syntheticFile(file)) + + const chip = agentPanel.attachmentChip(file) + await expect( + chip.getByRole('status', { name: unsupportedTypeMessage(file) }), + `${file} must be staged as an invalid chip` + ).toBeVisible() + await expect(comfyPage.toast.withText(file)).toHaveCount(0) + await chip.hover() + await chip + .getByRole('button', { + name: enMessages.agent.removeAsset.replace('{name}', file), + exact: true + }) + .click() + await expect(chip).toHaveCount(0) + } + expect(uploads).toEqual([]) + }) + + test('blocks Send for refused files from any path until they are removed', async ({ + agentPanel, + comfyPage + }) => { + const page = comfyPage.page + await agentPanel.open() + await agentPanel.selectWorkflow() + await agentPanel.composer.fill('Describe these files') + await expect(agentPanel.sendButton).toBeEnabled() + + const dataTransfer = await page.evaluateHandle(() => { + const transfer = new DataTransfer() + transfer.items.add( + new File(['%PDF'], 'doc.pdf', { type: 'application/pdf' }) + ) + return transfer + }) + try { + await agentPanel.root.dispatchEvent('drop', { dataTransfer }) + } finally { + await dataTransfer.dispose() + } + await expect( + agentPanel + .attachmentChip('doc.pdf') + .getByRole('status', { name: unsupportedTypeMessage('doc.pdf') }) + ).toBeVisible() + await agentPanel.sendButton.hover() + await expect( + page.getByRole('tooltip', { + name: enMessages.agent.sendBlockedByUnsupportedType.replace( + '{name}', + 'doc.pdf' + ), + exact: true + }) + ).toBeVisible() + + await agentPanel.fileInput.setInputFiles({ + name: 'renamed.mp4', + mimeType: 'video/mp4', + buffer: Buffer.from('plain text') + }) + await expect( + agentPanel.attachmentChip('renamed.mp4').getByRole('status') + ).toBeVisible() + await page.mouse.move(0, 0) + await agentPanel.sendButton.hover() + await expect( + page.getByRole('tooltip', { + name: enMessages.agent.sendBlockedByInvalidAttachments.replace( + '{count}', + '2' + ), + exact: true + }) + ).toBeVisible() + await agentPanel.composer.press('Enter') + await expect(agentPanel.composer).toHaveText('Describe these files') + + await agentPanel.close() + await agentPanel.open() + await expect(agentPanel.attachmentChip('doc.pdf')).toBeVisible() + await expect(agentPanel.composer).toHaveText('Describe these files') + await expect(agentPanel.sendButton).toBeDisabled() + await expect(comfyPage.toast.withText('doc.pdf')).toHaveCount(0) + + await agentPanel.removeAllAttachments() + await expect(agentPanel.sendButton).toBeEnabled() + }) + + test('attaches a .json dropped on the panel instead of opening it as a workflow', async ({ + agentPanel, + comfyPage + }) => { + await agentPanel.open() + await expect(agentPanel.composerAssetSection).toHaveCount(0) + + await agentPanel.dropFile(comfyPage.dragDrop, 'default.json') + + await expect(agentPanel.attachmentChip('default.json')).toBeVisible() + await expect.poll(() => comfyPage.nodeOps.getGraphNodesCount()).toBe(0) + }) + + test('still opens a .json dropped on the CANVAS as a workflow', async ({ + agentPanel, + comfyPage + }) => { + await agentPanel.open() + await expect(agentPanel.composerAssetSection).toHaveCount(0) + + await comfyPage.dragDrop.dragAndDropFile('default.json', { + preserveNativePropagation: true + }) + + await expect.poll(() => comfyPage.nodeOps.getGraphNodesCount()).toBe(7) + expect( + await comfyPage.nodeOps.getNodeRefsByType('KSampler') + ).toHaveLength(1) + await expect( + agentPanel.composerAssetSection, + 'a canvas drop must not attach to the chat' + ).toHaveCount(0) + }) + + test('attaches the real fixtures that exist on disk through the picker', async ({ + agentPanel + }) => { + await agentPanel.open() + + for (const file of [ + 'default.json', + 'animated_triangle.glb', + 'cube.obj' + ]) { + await agentPanel.fileInput.setInputFiles(assetPath(file)) + const chip = agentPanel.attachmentChip(file) + await expect(chip, `${file} must attach`).toBeVisible() + await expect( + chip.getByRole('status', { name: enMessages.agent.uploading }) + ).toHaveCount(0) + await chip.hover() + await chip + .getByRole('button', { + name: enMessages.agent.removeAsset.replace('{name}', file), + exact: true + }) + .click() + await expect(chip).toHaveCount(0) + } + }) + } +) diff --git a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts b/browser_tests/tests/agent/agentAttachJsonFile.spec.ts deleted file mode 100644 index 456529ec19c..00000000000 --- a/browser_tests/tests/agent/agentAttachJsonFile.spec.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { expect } from '@playwright/test' - -import type { UploadImageResponse } from '@comfyorg/ingest-types' - -import enMessages from '@/locales/en/main.json' with { type: 'json' } -import { AGENT_ATTACH_ACCEPT } from '@/workbench/extensions/agent/utils/attachableFiles' - -import { agentTest as test } from '@e2e/tests/agent/agentPanelMocks' -import { assetPath } from '@e2e/fixtures/utils/paths' -import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' - -test.describe( - 'Agent panel .json attachment', - { tag: ['@cloud', '@ui'] }, - () => { - test.beforeEach(async ({ comfyPage }) => { - const response: UploadImageResponse = { - name: 'default.json', - subfolder: '', - type: 'input' - } - await comfyPage.page.route('**/api/upload/image', (route) => - route.fulfill(jsonRoute(response)) - ) - await comfyPage.nodeOps.clearGraph() - await expect.poll(() => comfyPage.nodeOps.getGraphNodesCount()).toBe(0) - }) - - test('advertises .json in the attach accept list and accepts a .json file through the file browser', async ({ - agentPanel - }) => { - await agentPanel.open() - const { fileInput, composerAssetSection: assetSection } = agentPanel - - // The OS picker filters on this attribute; a missing `.json`/ - // `application/json` entry is exactly what hid JSON files from it. - await expect(fileInput).toHaveAttribute('accept', AGENT_ATTACH_ACCEPT) - - // Browse flow: picking a .json file through the input lands it in the - // composer instead of being silently ignored. This is the actual bug - // that was reported (the OS picker hid .json files) and the actual fix - // (AGENT_ATTACH_ACCEPT now lists .json/application/json). - await fileInput.setInputFiles(assetPath('default.json')) - const attachment = assetSection.getByRole('group', { - name: 'default.json', - exact: true - }) - await expect(attachment).toBeVisible() - await expect( - assetSection.locator(`[aria-label="${enMessages.agent.uploading}"]`) - ).toHaveCount(0) - - await attachment.hover() - await attachment - .getByRole('button', { name: 'Remove default.json', exact: true }) - .click() - await expect(assetSection).toHaveCount(0) - }) - - test('leaves a drag-and-dropped .json file unclaimed for the graph loader', async ({ - agentPanel, - comfyPage - }) => { - await agentPanel.open() - const { root: panel, composerAssetSection: assetSection } = agentPanel - await expect(assetSection).toHaveCount(0) - - const panelBox = await panel.boundingBox() - if (!panelBox) throw new Error('Agent panel is not visible') - await comfyPage.dragDrop.dragAndDropFile('default.json', { - preserveNativePropagation: true, - dropPosition: { - x: panelBox.x + panelBox.width / 2, - y: panelBox.y + panelBox.height / 2 - } - }) - - await expect.poll(() => comfyPage.nodeOps.getGraphNodesCount()).toBe(7) - expect( - await comfyPage.nodeOps.getNodeRefsByType('KSampler') - ).toHaveLength(1) - expect( - await comfyPage.nodeOps.getNodeRefsByType('CLIPTextEncode') - ).toHaveLength(2) - await expect(assetSection).toHaveCount(0) - }) - } -) diff --git a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts index c2dfd903f4b..00dc2eb5e47 100644 --- a/browser_tests/tests/agent/agentAttachmentUpload.spec.ts +++ b/browser_tests/tests/agent/agentAttachmentUpload.spec.ts @@ -94,9 +94,10 @@ test.describe('Agent attachment validation', { tag: ['@cloud', '@ui'] }, () => { await agentPanel.sendButton.hover() await expect( comfyPage.page.getByRole('tooltip', { - name: enMessages.agent.sendBlockedByInvalidVideo - .split(' | ')[0] - .replace('{name}', 'renamed.mp4'), + name: enMessages.agent.sendBlockedByInvalidVideo.replace( + '{name}', + 'renamed.mp4' + ), exact: true }) ).toBeVisible() diff --git a/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts index 53fd750fbc0..059458a3cf8 100644 --- a/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts +++ b/browser_tests/tests/agent/agentBulkAttachmentComposer.spec.ts @@ -4,7 +4,7 @@ import { expect } from '@playwright/test' import { jsonRoute } from '@e2e/fixtures/utils/jsonRoute' import { agentTest as test } from '@e2e/tests/agent/agentPanelMocks' -const ATTACHMENT_COUNT = 36 +const ATTACHMENT_COUNT = 25 // Regression: https://linear.app/comfyorg/issue/FE-3202 test.describe('Agent composer bulk attachments', { tag: '@cloud' }, () => { @@ -14,9 +14,6 @@ test.describe('Agent composer bulk attachments', { tag: '@cloud' }, () => { postedMessages }) => { const page = comfyPage.page - // Each attachment ref is the upload response name, so a shared name would - // make the POST assertion below pass on one ref echoed 36 times. Unique - // names are what let it distinguish that from 36 refs actually travelling. const uploadedRefs: string[] = [] await page.route('**/api/upload/image', (route) => { const name = `reference-${uploadedRefs.length + 1}.png` diff --git a/packages/ingest-types/src/index.ts b/packages/ingest-types/src/index.ts index 038da7e07eb..8eebf2d5db3 100644 --- a/packages/ingest-types/src/index.ts +++ b/packages/ingest-types/src/index.ts @@ -34,6 +34,10 @@ export type { AgentAskResolvedFrame, AgentAskServerFrame, AgentAskStatus, + AgentAttachmentBoundsExceeded, + AgentAttachmentError, + AgentAttachmentPolicy, + AgentAttachmentRejected, AgentCancelAccepted, AgentCancelMessageData, AgentCancelMessageError, @@ -115,6 +119,7 @@ export type { AgentPostMessageRequest, AgentPostMessageResponse, AgentPostMessageResponses, + AgentProbeableAttachmentExtension, AgentPublishSkillData, AgentPublishSkillError, AgentPublishSkillErrors, @@ -125,6 +130,9 @@ export type { AgentPutRunModeErrors, AgentPutRunModeResponse, AgentPutRunModeResponses, + AgentReadableAttachmentExtension, + AgentReferenceAttachmentExtension, + AgentRetainedAttachmentExtension, AgentRunApprovalContext, AgentRunMode, AgentRunModePutRequest, @@ -136,6 +144,7 @@ export type { AgentThreadListResponse, AgentThreadSummary, AgentTurnAccepted, + AgentViewableAttachmentExtension, Asset, AssetCreated, AssetCreatedWritable, diff --git a/packages/ingest-types/src/types.gen.ts b/packages/ingest-types/src/types.gen.ts index 38df864a305..eca280a75b0 100644 --- a/packages/ingest-types/src/types.gen.ts +++ b/packages/ingest-types/src/types.gen.ts @@ -5297,6 +5297,25 @@ export type AssetCreated = Asset & { created_new: boolean } +/** + * Attachment extensions whose content the model can see: the raster formats Go's image + * package decodes, plus HEIC/HEIF stills (iPhone photos), which view_asset converts to JPEG + * first. view_asset always re-encodes to PNG or JPEG, so the provider's own four-type image + * allowlist is satisfied by construction rather than by matching this list. + * + */ +export type AgentViewableAttachmentExtension = + | '.png' + | '.jpg' + | '.jpeg' + | '.gif' + | '.webp' + | '.bmp' + | '.tif' + | '.tiff' + | '.heic' + | '.heif' + /** * Acknowledgement that a turn was accepted. The agent runs asynchronously; output streams over the WebSocket (agent_message_delta, agent_tool_call, draft_patch, agent_message_done). */ @@ -5477,6 +5496,78 @@ export type AgentRunApprovalContext = { workflow_name?: string } +/** + * Attachment extensions that are accepted so a user can keep them beside a workflow, and + * nothing more — the agent can neither read the contents nor wire the file in. + * + * .avif is the only one: it is an image the turn cannot decode (no registry entry, so + * view_asset would fail) and it is not text, so read_asset would return mojibake. It stays + * accepted because it was attachable before the policy existed. + * + */ +export type AgentRetainedAttachmentExtension = '.avif' + +/** + * Attachment extensions the agent recognizes as 3D models and wires into Load3D's model_file + * widget. They are exactly the set Load3D accepts (comfy_extras/nodes_load_3d.py); .usdz is + * absent from both lists, so it is not an exclusion this tier makes. + * + */ +export type AgentReferenceAttachmentExtension = + | '.glb' + | '.obj' + | '.fbx' + | '.gltf' + | '.stl' + | '.ply' + | '.spz' + | '.splat' + | '.ksplat' + +/** + * Attachment extensions whose contents the agent reads verbatim, with read_asset. The bytes + * are decoded as UTF-8 and returned as text, bounded and truncation-marked. + * + * This tier describes CONTENT ACCESS only and says nothing about whether the file can be + * wired into a graph. Those are separate questions, so a format becoming a node input later + * does not move it out of this tier. + * + * .svg is here rather than with the images because it is XML: view_asset cannot decode it, + * but its markup is exactly what a caller would want read. + * + */ +export type AgentReadableAttachmentExtension = + | '.md' + | '.markdown' + | '.txt' + | '.json' + | '.csv' + | '.yaml' + | '.yml' + | '.xml' + | '.log' + | '.svg' + +/** + * Attachment extensions the agent can only describe from metadata. A subset of the containers + * ingest sorts into a load node's video/audio lists: that categoriser also accepts .wmv, .flv, + * .aac and .wma, which the agreed attachment list rejects. + * + */ +export type AgentProbeableAttachmentExtension = + | '.mp4' + | '.webm' + | '.mov' + | '.m4v' + | '.avi' + | '.mkv' + | '.mp3' + | '.wav' + | '.ogg' + | '.opus' + | '.flac' + | '.m4a' + /** * A user turn posted to the agent. */ @@ -5486,7 +5577,24 @@ export type AgentPostMessageRequest = { */ ask_kinds?: Array /** - * Optional input filenames the client already uploaded to the ComfyUI input namespace (via /api/upload/image, which returns the {name, subfolder, type} reference). Images, video and audio are all accepted. The agent wires them into the workflow by filename — it never receives file bytes here, and reads an attachment's contents through its own asset tools when a request depends on them. + * Optional input filenames the client already uploaded to the ComfyUI input namespace (via + * /api/upload/image, which returns the {name, subfolder, type} reference). The agent never + * receives file bytes here; what it can read or wire depends on the policy tier. + * + * Each entry's extension must appear in AgentAttachmentPolicy; a reference outside it is + * rejected with 422 (AgentAttachmentRejected). A reference with NO extension is the one + * exception: not every reference is a filename — an asset hash may be a bare hex digest or + * "blake3:" — so one with nothing to judge is admitted here and checked again once it + * resolves to a stored name. A refusal there costs the attachment its asset id and its + * kind, so the agent gets no tool for it and the turn is not refused; the reference itself + * is still echoed back by GET .../messages. Which tier of that policy an extension lands + * in says what the agent can do with it, and the tiers differ sharply — a client that + * presents them as one flat list misleads the user. + * + * The constraint is expressed as extension enums rather than on this array because the array + * carries FILENAMES, not content types: an extension is what both ends can actually agree + * on, and the upload route this list references stores no MIME type for what it accepts. + * */ attachments?: Array /** @@ -5779,6 +5887,82 @@ export type AgentCancelAccepted = { status: 'cancelling' } +/** + * 422 body when a posted attachment's extension is outside the accepted list. It carries the + * policy rather than only an error string so a client never has to hardcode the list to + * explain the refusal — the same reason the list is expressed as enums the frontend generates + * its types from. + * + */ +export type AgentAttachmentRejected = { + accepted: AgentAttachmentPolicy + error: string + /** + * The offending references, in the order they were posted. Capped, so it can be shorter than rejected_count — a client that reports "N were refused" must read that field rather than this array's length. + */ + rejected: Array + /** + * How many references were refused in total, before the list above was capped. + */ + rejected_count: number + type: 'ATTACHMENT_TYPE_NOT_ACCEPTED' +} + +/** + * Every extension the agent accepts as an attachment, grouped by what it can actually DO with + * one. The grouping is the point: the tiers differ sharply, and a client that renders them as + * one flat "supported files" list tells the user something false (PM-1855). + * + * - `view`: the model sees the content. view_asset decodes the still (converting HEIC/HEIF to + * JPEG first) and re-encodes it as PNG/JPEG into the turn, so the model reasons about actual + * pixels. + * - `probe`: metadata only. probe_media reports duration, resolution, frame rate and codecs; + * nothing in the model's input can carry a clip, so the CONTENT stays unreadable. Seeing a + * video means cutting a frame out with process_media and viewing that. + * - `reference`: the agent knows the file is a 3D model, with its kind and id, and wires its + * filename into a Load3D node's model_file widget. No tool opens the mesh itself: the cloud + * turn registers neither a shell nor a file-read tool, by design, as it is a shared + * multi-tenant pod. + * - `read`: the agent reads the contents verbatim with read_asset — UTF-8 decoded, bounded + * and truncation-marked. The tier describes CONTENT ACCESS only; whether the file can also + * be wired into a graph is a separate axis, so a format gaining an upload-backed node input + * later stays in this tier. + * - `retain`: the file is attached and nothing more — neither readable nor wirable. Only + * .avif, which the turn cannot decode as an image and is not text. + * + * The union of the five arrays is the accepted list. Anything absent is rejected. + * + */ +export type AgentAttachmentPolicy = { + probe: Array + read: Array + reference: Array + retain: Array + view: Array +} + +export type AgentAttachmentError = + | ({ + type: 'ATTACHMENT_TYPE_NOT_ACCEPTED' + } & AgentAttachmentRejected) + | ({ + type: 'ATTACHMENT_BOUNDS_EXCEEDED' + } & AgentAttachmentBoundsExceeded) + +export type AgentAttachmentBoundsExceeded = { + error: string + limits: { + max_item_length: number + max_items: number + } + /** + * A bounded example of the first overlong reference; empty for count-only violations. + */ + overlong_ref: string + received_count: number + type: 'ATTACHMENT_BOUNDS_EXCEEDED' +} + /** * How an ask left pending, as stored on the ask row. `answered` means an answer was committed and delivered to the turn; `cancelled` means the turn ended, was stopped, or could not take the answer; `expired` means the ask timed out. There is no "unknown" status: when an answer request fails with a 5xx the client does not know the outcome and should wait for agent_ask_resolved or refetch the thread. */ @@ -7078,6 +7262,10 @@ export type AgentPostMessageErrors = { * Forbidden (workflow or thread not owned by the caller). An ingest refusal, such as the web session's, is ErrorResponse; the agent service's is AgentError. */ 403: ErrorResponse | AgentError + /** + * One or more attachments name a file type the agent does not accept. The body names the offending references and carries the full accepted policy, so the client can explain the refusal without hardcoding the list. + */ + 422: AgentAttachmentError /** * Internal server error (ingest-raised failures use the standard ErrorResponse shape instead) */ diff --git a/packages/ingest-types/src/zod.gen.ts b/packages/ingest-types/src/zod.gen.ts index cb4c299e55b..d63498988a3 100644 --- a/packages/ingest-types/src/zod.gen.ts +++ b/packages/ingest-types/src/zod.gen.ts @@ -3148,6 +3148,26 @@ export const zAssetCreated = zAsset.and( }) ) +/** + * Attachment extensions whose content the model can see: the raster formats Go's image + * package decodes, plus HEIC/HEIF stills (iPhone photos), which view_asset converts to JPEG + * first. view_asset always re-encodes to PNG or JPEG, so the provider's own four-type image + * allowlist is satisfied by construction rather than by matching this list. + * + */ +export const zAgentViewableAttachmentExtension = z.enum([ + '.png', + '.jpg', + '.jpeg', + '.gif', + '.webp', + '.bmp', + '.tif', + '.tiff', + '.heic', + '.heif' +]) + /** * Acknowledgement that a turn was accepted. The agent runs asynchronously; output streams over the WebSocket (agent_message_delta, agent_tool_call, draft_patch, agent_message_done). */ @@ -3250,6 +3270,81 @@ export const zAgentRunApprovalContext = z.object({ workflow_name: z.string().optional() }) +/** + * Attachment extensions that are accepted so a user can keep them beside a workflow, and + * nothing more — the agent can neither read the contents nor wire the file in. + * + * .avif is the only one: it is an image the turn cannot decode (no registry entry, so + * view_asset would fail) and it is not text, so read_asset would return mojibake. It stays + * accepted because it was attachable before the policy existed. + * + */ +export const zAgentRetainedAttachmentExtension = z.enum(['.avif']) + +/** + * Attachment extensions the agent recognizes as 3D models and wires into Load3D's model_file + * widget. They are exactly the set Load3D accepts (comfy_extras/nodes_load_3d.py); .usdz is + * absent from both lists, so it is not an exclusion this tier makes. + * + */ +export const zAgentReferenceAttachmentExtension = z.enum([ + '.glb', + '.obj', + '.fbx', + '.gltf', + '.stl', + '.ply', + '.spz', + '.splat', + '.ksplat' +]) + +/** + * Attachment extensions whose contents the agent reads verbatim, with read_asset. The bytes + * are decoded as UTF-8 and returned as text, bounded and truncation-marked. + * + * This tier describes CONTENT ACCESS only and says nothing about whether the file can be + * wired into a graph. Those are separate questions, so a format becoming a node input later + * does not move it out of this tier. + * + * .svg is here rather than with the images because it is XML: view_asset cannot decode it, + * but its markup is exactly what a caller would want read. + * + */ +export const zAgentReadableAttachmentExtension = z.enum([ + '.md', + '.markdown', + '.txt', + '.json', + '.csv', + '.yaml', + '.yml', + '.xml', + '.log', + '.svg' +]) + +/** + * Attachment extensions the agent can only describe from metadata. A subset of the containers + * ingest sorts into a load node's video/audio lists: that categoriser also accepts .wmv, .flv, + * .aac and .wma, which the agreed attachment list rejects. + * + */ +export const zAgentProbeableAttachmentExtension = z.enum([ + '.mp4', + '.webm', + '.mov', + '.m4v', + '.avi', + '.mkv', + '.mp3', + '.wav', + '.ogg', + '.opus', + '.flac', + '.m4a' +]) + /** * Which card renders an ask. `ask_user` is the generic prompt raised by the ask_user tool. `run_approval` is the run consent card raised when the caller's ask_approval run mode gates a run: exactly the Run and Cancel options, never free text. `paused` is the admission pause card: one Resume option, never free text. `delete_approval` is the delete consent card for nodes the user made: exactly the Delete and Keep options, never free text. The same value is the agent_ask event's `kind`, the message list's `pending_ask.kind`, and an entry of `ask_kinds` on the message POST. */ @@ -3265,7 +3360,7 @@ export const zAgentAskKind = z.enum([ */ export const zAgentPostMessageRequest = z.object({ ask_kinds: z.array(zAgentAskKind).optional(), - attachments: z.array(z.string()).optional(), + attachments: z.array(z.string().max(256)).max(25).optional(), client_id: z.string().optional(), client_message_id: z.string().max(128).optional(), content: z.string(), @@ -3476,6 +3571,74 @@ export const zAgentCancelAccepted = z.object({ status: z.enum(['cancelling']) }) +/** + * Every extension the agent accepts as an attachment, grouped by what it can actually DO with + * one. The grouping is the point: the tiers differ sharply, and a client that renders them as + * one flat "supported files" list tells the user something false (PM-1855). + * + * - `view`: the model sees the content. view_asset decodes the still (converting HEIC/HEIF to + * JPEG first) and re-encodes it as PNG/JPEG into the turn, so the model reasons about actual + * pixels. + * - `probe`: metadata only. probe_media reports duration, resolution, frame rate and codecs; + * nothing in the model's input can carry a clip, so the CONTENT stays unreadable. Seeing a + * video means cutting a frame out with process_media and viewing that. + * - `reference`: the agent knows the file is a 3D model, with its kind and id, and wires its + * filename into a Load3D node's model_file widget. No tool opens the mesh itself: the cloud + * turn registers neither a shell nor a file-read tool, by design, as it is a shared + * multi-tenant pod. + * - `read`: the agent reads the contents verbatim with read_asset — UTF-8 decoded, bounded + * and truncation-marked. The tier describes CONTENT ACCESS only; whether the file can also + * be wired into a graph is a separate axis, so a format gaining an upload-backed node input + * later stays in this tier. + * - `retain`: the file is attached and nothing more — neither readable nor wirable. Only + * .avif, which the turn cannot decode as an image and is not text. + * + * The union of the five arrays is the accepted list. Anything absent is rejected. + * + */ +export const zAgentAttachmentPolicy = z.object({ + probe: z.array(zAgentProbeableAttachmentExtension), + read: z.array(zAgentReadableAttachmentExtension), + reference: z.array(zAgentReferenceAttachmentExtension), + retain: z.array(zAgentRetainedAttachmentExtension), + view: z.array(zAgentViewableAttachmentExtension) +}) + +/** + * 422 body when a posted attachment's extension is outside the accepted list. It carries the + * policy rather than only an error string so a client never has to hardcode the list to + * explain the refusal — the same reason the list is expressed as enums the frontend generates + * its types from. + * + */ +export const zAgentAttachmentRejected = z.object({ + accepted: zAgentAttachmentPolicy, + error: z.string(), + rejected: z.array(z.string()), + rejected_count: z.int(), + type: z.enum(['ATTACHMENT_TYPE_NOT_ACCEPTED']) +}) + +export const zAgentAttachmentBoundsExceeded = z.object({ + error: z.string(), + limits: z.object({ + max_item_length: z.int(), + max_items: z.int() + }), + overlong_ref: z.string(), + received_count: z.int(), + type: z.enum(['ATTACHMENT_BOUNDS_EXCEEDED']) +}) + +export const zAgentAttachmentError = z.discriminatedUnion('type', [ + zAgentAttachmentRejected.extend({ + type: z.literal('ATTACHMENT_TYPE_NOT_ACCEPTED') + }), + zAgentAttachmentBoundsExceeded.extend({ + type: z.literal('ATTACHMENT_BOUNDS_EXCEEDED') + }) +]) + /** * How an ask left pending, as stored on the ask row. `answered` means an answer was committed and delivered to the turn; `cancelled` means the turn ended, was stopped, or could not take the answer; `expired` means the ask timed out. There is no "unknown" status: when an answer request fails with a 5xx the client does not know the outcome and should wait for agent_ask_resolved or refetch the thread. */ diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 47e427dafd4..ccc7f0f0b63 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5709,9 +5709,14 @@ "selectNode": "Select node" }, "addFromAssets": "Drag in asset from asset panel", + "assetFetchFailed": "{name} could not be retrieved", "assetNotAttachable": "Images, video, audio, 3D models, and text files can be attached", + "attachmentCountExceeded": "Only {count} files can be attached to one message", "attachmentInvalidVideo": "{name} isn’t a valid video file. Remove it and try another file.", + "attachmentNamesOverflow": "{names} and {count} more", "attachmentTooLarge": "{name} is larger than {limit}", + "attachmentTypeNotAccepted": "{name} is not a file type the agent accepts | {name} are not file types the agent accepts", + "attachmentUnsupportedType": "{name} isn’t a file type the agent accepts. Remove it and try another file.", "attachmentUploadFailed": "{name} could not be uploaded", "attachmentsAlreadyAdded": "{name} is already in the asset tray | Skipped {count} duplicate assets", "mention": "Mention a node", @@ -5744,7 +5749,9 @@ "runModeLimitDescription": "Comfy will only confirm when a workflow run exceeds your credit limit", "runModeSaveFailed": "Could not save run permissions. Please try again.", "send": "Send", - "sendBlockedByInvalidVideo": "{name} isn’t a valid video. Remove it to send. | {count} files aren’t valid videos. Remove them to send.", + "sendBlockedByInvalidAttachments": "{count} files can’t be sent. Remove them to send.", + "sendBlockedByInvalidVideo": "{name} isn’t a valid video. Remove it to send.", + "sendBlockedByUnsupportedType": "{name} isn’t a file type the agent accepts. Remove it to send.", "stop": "Stop", "updatingGraph": "The agent is updating the graph", "editWhileWorking": "You can keep editing while it works", diff --git a/src/utils/eventUtils.test.ts b/src/utils/eventUtils.test.ts index 8b72d427108..b0e1a2f56b0 100644 --- a/src/utils/eventUtils.test.ts +++ b/src/utils/eventUtils.test.ts @@ -1,7 +1,8 @@ import { fromAny } from '@total-typescript/shoehorn' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { extractFilesFromDragEvent, + fetchDroppedAsset, hasAudioType, hasImageType, hasVideoType, @@ -89,3 +90,97 @@ describe('extractFilesFromDragEvent', () => { expect(hasImageType(file)).toBe(true) }) }) + +describe('fetchDroppedAsset', () => { + function stubFetch(resolvedUrl: string) { + const response = new Response(new Blob(['x'], { type: 'image/png' })) + Object.defineProperty(response, 'url', { value: resolvedUrl }) + vi.mocked(fetch).mockResolvedValue(response) + } + + it.for([ + [ + 'a dotted content hash must not outrank the ref', + 'https://storage/bucket/0123456789abcdef0123456789abcdef.png', + 'cat.png' + ], + [ + 'nor must an extensionless storage key', + 'https://storage/bucket/0123456789abcdef0123456789abcdef', + 'cat.png' + ] + ] as const)('%s', async ([, resolvedUrl, want]) => { + stubFetch(resolvedUrl) + const file = await fetchDroppedAsset( + { + name: 'display label', + uri: 'https://api/assets/abc/content', + ref: 'cat.png' + }, + { nameFromSource: true } + ) + expect(file?.name).toBe(want) + }) + + it('still prefers an explicit filename parameter over everything', async () => { + stubFetch('https://storage/bucket/deadbeef.png?filename=real%20name.png') + const file = await fetchDroppedAsset( + { + name: 'display label', + uri: 'https://api/assets/abc/content', + ref: 'cat.png' + }, + { nameFromSource: true } + ) + expect(file?.name).toBe('real name.png') + }) + + it('keeps the file when the storage path has a malformed escape', async () => { + stubFetch('https://storage/bucket/%E0%A4%A') + const file = await fetchDroppedAsset( + { + name: 'display label', + uri: 'https://api/assets/abc/content', + ref: 'cat.png' + }, + { nameFromSource: true } + ) + expect(file?.name).toBe('cat.png') + }) + + it('still yields a file when only the malformed segment could name it', async () => { + stubFetch('https://storage/bucket/%E0%A4%A.png') + const file = await fetchDroppedAsset( + { + name: 'fallback.bin', + uri: 'https://api/assets/abc/content', + ref: 'bare-ref' + }, + { nameFromSource: true } + ) + expect(file?.name).toBe('%E0%A4%A.png') + }) + + it('falls back to the display label when nothing else carries an extension', async () => { + stubFetch('https://storage/bucket/deadbeef') + const file = await fetchDroppedAsset( + { + name: 'fallback.png', + uri: 'https://api/assets/abc/content', + ref: 'bare-ref' + }, + { nameFromSource: true } + ) + expect(file?.name).toBe('fallback.png') + }) + + it('keeps the display label as the name by default', async () => { + stubFetch('https://storage/bucket/deadbeef.png?filename=real%20name.png') + const file = await fetchDroppedAsset({ + name: 'display label', + uri: 'https://api/assets/abc/content', + ref: 'cat.png' + }) + expect(file?.name).toBe('display label') + }) +}) diff --git a/src/utils/eventUtils.ts b/src/utils/eventUtils.ts index adc6e5c94ae..c1b9a287924 100644 --- a/src/utils/eventUtils.ts +++ b/src/utils/eventUtils.ts @@ -35,16 +35,49 @@ export function getDroppedAsset( : undefined } -export async function fetchDroppedAsset({ - name, - uri -}: DroppedAsset): Promise { +function decodedBasename(pathname: string): string { + const segment = pathname.split('/').pop() || '' + try { + return decodeURIComponent(segment) + } catch { + return segment + } +} + +function sourceFileName( + { name, ref }: DroppedAsset, + uri: string, + responseUrl: string +): string { + const resolvedUrl = new URL(responseUrl || uri, 'http://localhost') + const named = [ + resolvedUrl.searchParams.get('filename'), + ref, + decodedBasename(resolvedUrl.pathname), + name + ].find((candidate) => candidate && /\.[a-z0-9]{1,8}$/i.test(candidate)) + return named || name +} + +interface FetchDroppedAssetOptions { + signal?: AbortSignal + nameFromSource?: boolean +} + +export async function fetchDroppedAsset( + asset: DroppedAsset, + { signal, nameFromSource = false }: FetchDroppedAssetOptions = {} +): Promise { + const { name, uri } = asset if (!uri) return undefined try { - const response = await fetch(uri) + const response = await fetch(uri, { signal }) if (!response.ok) return undefined const blob = await response.blob() - return new File([blob], name, { type: blob.type }) + const fileName = nameFromSource + ? sourceFileName(asset, uri, response.url) + : name + return new File([blob], fileName, { type: blob.type }) } catch { return undefined } diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index f2dc91f6f9e..4a817177bb7 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -262,7 +262,10 @@ vi.mock(import('@/platform/workspace/composables/useBillingCapabilities'), { import type { AgentMessages, TurnId } from './schemas/agentApiSchema' import { toTurnId, zAgentWsEvent } from './schemas/agentApiSchema' -import { MAX_ATTACHMENT_BYTES } from './composables/agent/useAttachment' +import { + MAX_ATTACHMENT_BYTES, + MAX_TURN_ATTACHMENTS +} from './composables/agent/useAttachment' import { useBillingIdentity } from './composables/useBillingIdentity' import type { AgentChatEvent } from './services/agent/agentEventTransport' import { useAgentChatHistoryStore } from './stores/agent/agentChatHistoryStore' @@ -2381,6 +2384,10 @@ function dispatchDrag( const mp4Header = '\0\0\0\x18ftyp' +function unsupportedTypeMessage(name: string): string { + return i18n.global.t('agent.attachmentUnsupportedType', { name }) +} + function fileOfSize(name: string, size: number, type: string): File { const file = new File([mp4Header], name, { type }) Object.defineProperty(file, 'size', { value: size }) @@ -3289,6 +3296,52 @@ describe('AgentPanelRoot attach flow', () => { expect(telemetry.trackAgentAttachButtonClicked).not.toHaveBeenCalled() }) + it.for(['picker', 'drop', 'paste'] as const)( + 'stages a refused type from the %s as an invalid chip and blocks Send', + async (path) => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const user = userEvent.setup({ applyAccept: false }) + const doc = new File(['x'], 'doc.pdf', { type: 'application/pdf' }) + const store = useAgentComposerStore() + store.setText('Keep this draft') + const target = screen.getByRole('textbox') + + if (path === 'picker') { + await openAddMenu() + await user.click( + await screen.findByRole('menuitem', { + name: i18n.global.t('agent.attachFiles') + }) + ) + await user.upload( + screen.getByTestId('agent-file-input'), + doc + ) + } + if (path === 'drop') dispatchDrag(target, 'drop', { files: [doc] }) + if (path === 'paste') { + const clipboard = new DataTransfer() + clipboard.items.add(doc) + await user.click(target) + await user.paste(clipboard) + } + + expect( + await screen.findByRole('status', { + name: unsupportedTypeMessage('doc.pdf') + }) + ).toBeVisible() + expect( + screen.getByRole('button', { name: i18n.global.t('agent.send') }) + ).toHaveAttribute('aria-disabled', 'true') + expect(store.draft).toBe('Keep this draft') + expect(uploaded).toEqual([]) + expect(useToast().toasts).toEqual([]) + } + ) + it('refuses a text file renamed to mp4 from the picker', async () => { const uploaded = stubUploadFetch() renderWithSelectedTarget() @@ -3922,6 +3975,39 @@ describe('AgentPanelRoot attach flow', () => { ) }) + it('uses the fetched asset filename when its display label has no type', async () => { + const uploads: File[] = [] + vi.mocked(fetch).mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (url.includes('/api/view')) + return new Response(new Blob(['asset'], { type: 'image/png' })) + if (url.endsWith('/api/upload/image')) { + if (!(init?.body instanceof FormData)) + throw new Error('Expected upload form data') + const image = init.body.get('image') + if (!(image instanceof File)) throw new Error('Expected upload file') + uploads.push(image) + return json(200, { name: 'uploaded_gen.png' }) + } + return json(200, agentThreadList()) + } + ) + renderWithSelectedTarget() + await nextTick() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ display_name: 'My renamed asset', type: 'input' }) + : 'http://localhost/api/view?filename=gen.png' + }) + + await vi.waitFor(() => expect(uploads).toHaveLength(1)) + expect(uploads[0].name).toBe('gen.png') + }) + it('does not warn after closing the panel during a deferred asset fetch', async () => { vi.mocked(fetch).mockImplementation((input) => { const url = String(input) @@ -3962,9 +4048,271 @@ describe('AgentPanelRoot attach flow', () => { } }) - it('attaches dropped assets and leaves other files to the graph loader', async () => { - // The graph loader only opens a dropped workflow while the drop is - // unclaimed, so the panel must not claim files it cannot attach. + it('aborts a timed-out asset fetch and reports the retrieval failure', async () => { + let assetSignal: AbortSignal | undefined + vi.mocked(fetch).mockImplementation((input, init) => { + const url = String(input) + if (url.includes('/api/view')) { + assetSignal = init?.signal ?? undefined + return new Promise(() => {}) + } + if (url.includes('/assets')) + return Promise.resolve( + json(200, { assets: [], total: 0, has_more: false }) + ) + if (url.includes('/workflows')) + return Promise.resolve( + json(200, { data: [], total: 0, has_more: false }) + ) + return Promise.resolve(json(200, agentThreadList())) + }) + renderWithSelectedTarget() + await nextTick() + vi.useFakeTimers() + try { + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ filename: 'gen.png', type: 'input' }) + : 'http://localhost/api/view?filename=gen.png' + }) + await nextTick() + await vi.advanceTimersByTimeAsync(60_000) + + expect(assetSignal?.aborted).toBe(true) + expect( + useToast().toasts.some(({ title }) => + title.includes('gen.png could not be retrieved') + ) + ).toBe(true) + expect( + useToast().toasts.some(({ title }) => + title.includes('could not be uploaded') + ) + ).toBe(false) + } finally { + vi.useRealTimers() + } + }) + + it('stages a rejected asset card as an invalid chip without fetching or uploading it', async () => { + const uploaded = stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + const store = useAgentComposerStore() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ + filename: 'scene.usdz', + type: 'input', + attachment_ref: 'stored_scene.usdz', + media_kind: '3D' + }) + : 'http://localhost/api/view?filename=stored_scene.usdz' + }) + + expect( + await screen.findByRole('status', { + name: unsupportedTypeMessage('scene.usdz') + }) + ).toBeVisible() + expect(store.attachments).toEqual([ + expect.objectContaining({ + name: 'scene.usdz', + ref: '', + validationError: 'unsupported-type' + }) + ]) + expect( + vi + .mocked(fetch) + .mock.calls.some(([input]) => String(input).includes('/api/view')) + ).toBe(false) + expect(uploaded).toEqual([]) + expect(useToast().toasts).toEqual([]) + }) + + it('preserves an opaque asset ref instead of uploading it', async () => { + const uploads: File[] = [] + vi.mocked(fetch).mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (url.includes('/api/view')) + return new Response(new Blob(['x'], { type: 'image/png' })) + if (url.endsWith('/api/upload/image')) { + if (init?.body instanceof FormData) { + const image = init.body.get('image') + if (image instanceof File) uploads.push(image) + } + return json(200, { name: 'uploaded.png' }) + } + return json(200, agentThreadList()) + } + ) + renderWithSelectedTarget() + await nextTick() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ + filename: 'photo.png', + type: 'input', + attachment_ref: 'blake3:abcdef0123456789', + media_kind: 'image' + }) + : 'http://localhost/api/view?filename=real.png' + }) + + await vi.waitFor(() => + expect(useAgentComposerStore().attachments).toEqual([ + expect.objectContaining({ + name: 'photo.png', + ref: 'blake3:abcdef0123456789' + }) + ]) + ) + expect(screen.queryByRole('status')).not.toBeInTheDocument() + expect(uploads).toHaveLength(0) + expect(useToast().toasts).toEqual([]) + }) + + it('stages an opaque asset card with a rejected label as an invalid chip', async () => { + stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info'], + getData: () => + JSON.stringify({ + filename: 'scene.usdz', + type: 'input', + attachment_ref: 'blake3:abcdef0123456789', + media_kind: '3D' + }) + }) + + expect( + await screen.findByRole('status', { + name: unsupportedTypeMessage('scene.usdz') + }) + ).toBeVisible() + expect(useToast().toasts).toEqual([]) + }) + + it('keeps a fetched asset of an unaccepted type as an invalid chip under its stored name', async () => { + const uploads: File[] = [] + vi.mocked(fetch).mockImplementation( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (url.includes('/api/view')) + return new Response(new Blob(['%PDF'], { type: 'application/pdf' })) + if (url.endsWith('/api/upload/image')) { + if (init?.body instanceof FormData) { + const image = init.body.get('image') + if (image instanceof File) uploads.push(image) + } + return json(200, { name: 'uploaded.pdf' }) + } + return json(200, agentThreadList()) + } + ) + renderWithSelectedTarget() + await nextTick() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info', 'text/uri-list'], + getData: (type: string) => + type === 'application/x-comfy-asset-info' + ? JSON.stringify({ + filename: 'My scan', + type: 'input' + }) + : 'http://localhost/api/view?filename=scan.pdf' + }) + + expect( + await screen.findByRole('status', { + name: unsupportedTypeMessage('scan.pdf') + }) + ).toBeVisible() + expect(uploads).toEqual([]) + expect(useToast().toasts).toEqual([]) + }) + + it('refuses an asset card once the turn is already full', async () => { + stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + + const card = (index: number) => ({ + types: ['application/x-comfy-asset-info'], + getData: () => + JSON.stringify({ + filename: `photo${index}.png`, + type: 'input', + attachment_ref: `stored_photo${index}.png`, + media_kind: 'image' + }) + }) + const target = screen.getByRole('textbox') + for (let index = 0; index < MAX_TURN_ATTACHMENTS; index++) { + dispatchDrag(target, 'drop', card(index)) + } + await vi.waitFor(() => + expect( + within(screen.getByTestId('composer-asset-section')).getAllByTestId( + 'agent-attachment-chip' + ) + ).toHaveLength(MAX_TURN_ATTACHMENTS) + ) + + dispatchDrag(target, 'drop', card(MAX_TURN_ATTACHMENTS)) + + await vi.waitFor(() => + expect(useToast().toasts.map(({ title }) => title)).toContain( + 'Only 25 files can be attached to one message' + ) + ) + expect( + within(screen.getByTestId('composer-asset-section')).getAllByTestId( + 'agent-attachment-chip' + ) + ).toHaveLength(MAX_TURN_ATTACHMENTS) + }) + + it('judges an asset by its stored filename rather than its display label', async () => { + stubUploadFetch() + renderWithSelectedTarget() + await nextTick() + + dispatchDrag(screen.getByRole('textbox'), 'drop', { + types: ['application/x-comfy-asset-info'], + getData: () => + JSON.stringify({ + filename: 'renamed.usdz', + type: 'input', + attachment_ref: 'stored.png', + media_kind: 'image' + }) + }) + + expect( + within(await screen.findByTestId('composer-asset-section')).getByRole( + 'group', + { name: 'renamed.usdz' } + ) + ).toBeInTheDocument() + expect(useToast().toasts).toEqual([]) + }) + + it('attaches a dropped workflow json to the chat instead of the graph loader', async () => { stubUploadFetch() renderWithSelectedTarget() await nextTick() @@ -3975,17 +4323,11 @@ describe('AgentPanelRoot attach flow', () => { const workflow = new File(['{}'], 'flow.json', { type: 'application/json' }) - expect(dispatchDrag(target, 'drop', { files: [workflow] })).toBe(false) - expect( - screen.queryByRole('group', { name: 'flow.json' }) - ).not.toBeInTheDocument() - - const asset = new File(['x'], 'cat.png', { type: 'image/png' }) - expect(dispatchDrag(target, 'drop', { files: [asset] })).toBe(true) + expect(dispatchDrag(target, 'drop', { files: [workflow] })).toBe(true) expect( within(await screen.findByTestId('composer-asset-section')).getByRole( 'group', - { name: 'cat.png' } + { name: 'flow.json' } ) ).toBeInTheDocument() }) @@ -4039,17 +4381,13 @@ describe('AgentPanelRoot attach flow', () => { expect(store.prompt).toEqual({ text: 'Keep this draft', references: [] }) }) - it('attaches only the assets out of a mixed drop', async () => { + it('attaches the accepted files out of a mixed drop and keeps the rest as invalid chips', async () => { const uploaded = stubUploadFetch() renderWithSelectedTarget() await nextTick() - // The non-attachable file comes first: addFiles uploads sequentially, so a - // regression that forwards the whole drop would upload flow.json before - // cat.png and the settled assertion below could never latch a lucky - // intermediate state. const files = [ - new File(['{}'], 'flow.json', { type: 'application/json' }), + new File(['x'], 'doc.pdf', { type: 'application/pdf' }), new File(['x'], 'cat.png', { type: 'image/png' }) ] dispatchDrag(screen.getByRole('textbox'), 'drop', { files }) @@ -4061,11 +4399,12 @@ describe('AgentPanelRoot attach flow', () => { ) ).toBeInTheDocument() await vi.waitFor(() => expect(uploaded).toEqual(['cat.png'])) + expect( + screen.getByRole('status', { name: unsupportedTypeMessage('doc.pdf') }) + ).toBeVisible() + expect(useToast().toasts).toEqual([]) }) - // Falsifiers for the committed-drop gates: an implementation that emits - // before the deferred fetch or upload settles, or ignores their results, - // fails these. it.for(['failed', 'cancelled'] as const)( 'emits no drop telemetry for a %s deferred asset', async (outcome) => { diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index 0233ae19a52..ed387952f1b 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -35,12 +35,15 @@ import type { LGraphNode } from '@/lib/litegraph/src/litegraph' import { useAppMode } from '@/composables/useAppMode' import { MIME_ASSET_INFO } from '@/platform/assets/schemas/mediaAssetSchema' import { fetchDroppedAsset, getDroppedAsset } from '@/utils/eventUtils' +import type { DroppedAsset } from '@/utils/eventUtils' import { useAssetsStore } from '@/stores/assetsStore' import { AGENT_ATTACH_ACCEPT, - isAgentAttachable, + agentAttachRefVerdict, + agentAttachVerdict, isValidAgentAttachment } from './utils/attachableFiles' +import type { AgentAttachVerdict } from './utils/attachableFiles' import { getNodeByLocatorId } from '@/utils/graphTraversalUtil' // oxlint-disable-next-line comfy/no-restricted-paths import { useCanvasStore } from '@/renderer/core/canvas/canvasStore' @@ -88,6 +91,7 @@ import AgentGraphActivityBar from './components/AgentGraphActivityBar.vue' import OnboardingCoach from './components/agent/OnboardingCoach.vue' import { MAX_ATTACHMENT_BYTES, + MAX_TURN_ATTACHMENTS, useAttachment } from './composables/agent/useAttachment' import type { ActiveTab } from './types/activeTab' @@ -1720,10 +1724,9 @@ const attachment = useAttachment({ api.getServerFeature( 'max_upload_size' ) ?? MAX_ATTACHMENT_BYTES, - // A rejected file is the user's problem to fix, not an agent failure, so it - // must not raise the server-error overlay. - onError: (message) => toast.warning(message, { duration: 5000 }), + onError: warnAttachment, onDuplicate: notifyDuplicateAttachments, + staged: () => composerStore.attachments, stage: composerStore.addAttachment, update: composerStore.updateAttachment, remove: composerStore.removeAttachment @@ -1834,42 +1837,66 @@ function onPanelDragLeave(): void { if (assetDragDepth === 0) assetDragActive.value = false } +function warnAttachment(message: string): void { + toast.warning(message, { duration: 5000 }) +} + +function droppedAssetVerdict(asset: DroppedAsset): AgentAttachVerdict { + const fromRef = asset.ref ? agentAttachRefVerdict(asset.ref) : 'unknown' + return fromRef === 'unknown' ? agentAttachVerdict(asset.name) : fromRef +} + +function stageAcceptedAssetCard( + asset: DroppedAsset, + ref: string, + sourceKey: string +): boolean { + if (composerStore.attachments.some((item) => item.ref === ref)) { + notifyDuplicateAttachments([asset.name]) + return false + } + if (composerStore.attachments.length >= MAX_TURN_ATTACHMENTS) { + warnAttachment( + t('agent.attachmentCountExceeded', { count: MAX_TURN_ATTACHMENTS }) + ) + return false + } + const added = composerStore.addAttachment({ + id: `asset:${crypto.randomUUID()}`, + name: asset.name, + ref, + sourceKey, + previewUrl: asset.previewUrl, + mediaUrl: asset.mediaUrl, + mediaKind: asset.kind + }) + if (!added) notifyDuplicateAttachments([asset.name]) + return added +} + async function attachDroppedAsset(event: DragEvent): Promise { const asset = event.dataTransfer && getDroppedAsset(event.dataTransfer) if (!asset) { - toast.warning(t('agent.assetNotAttachable'), { duration: 5000 }) + warnAttachment(t('agent.assetNotAttachable')) return false } const sourceKey = asset.ref ? `asset:${asset.ref}` : `uri:${asset.uri}` + if (droppedAssetVerdict(asset) === 'rejected') { + attachment.addUnsupportedAsset(asset.name, sourceKey) + return false + } if (asset.ref && asset.kind !== 'other') { - if (composerStore.attachments.some((item) => item.ref === asset.ref)) { - notifyDuplicateAttachments([asset.name]) - return false - } - const added = composerStore.addAttachment({ - id: `asset:${crypto.randomUUID()}`, - name: asset.name, - ref: asset.ref, - sourceKey, - previewUrl: asset.previewUrl, - mediaUrl: asset.mediaUrl, - mediaKind: asset.kind - }) - if (!added) notifyDuplicateAttachments([asset.name]) - return added + return stageAcceptedAssetCard(asset, asset.ref, sourceKey) } const result = await attachment.addDeferredFile( asset.name, - async () => { - const file = await fetchDroppedAsset(asset) - return file && isAgentAttachable(file) ? file : undefined - }, + (signal) => fetchDroppedAsset(asset, { signal, nameFromSource: true }), sourceKey ) - if (result === 'unsupported') - toast.warning(t('agent.assetNotAttachable'), { duration: 5000 }) + if (result === 'fetch_failed') + warnAttachment(t('agent.assetFetchFailed', { name: asset.name })) return result === 'uploaded' } @@ -1879,8 +1906,6 @@ function onPanelDragOver(event: DragEvent): void { async function onPanelDrop(event: DragEvent): Promise { clearAssetDrag() - // A dropped asset card carries a URI, not a File, so the claim must happen - // before the async fetch resolves it into one. if ((event.dataTransfer?.files.length ?? 0) === 0 && isAssetDrag(event)) { event.preventDefault() void attachDroppedAsset(event).then((attached) => { @@ -1889,11 +1914,7 @@ async function onPanelDrop(event: DragEvent): Promise { }) return } - // Anything the composer cannot attach still belongs to the graph loader, which - // only runs while the drop is unclaimed, so claim the attachable files alone. - const files = Array.from(event.dataTransfer?.files ?? []).filter( - isAgentAttachable - ) + const files = Array.from(event.dataTransfer?.files ?? []) if (files.length === 0) return event.preventDefault() if (await attachment.addFiles(files)) diff --git a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts index b04c490b5e9..b1c0f3cd684 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.assetTray.test.ts @@ -300,14 +300,17 @@ describe('composer asset tray', () => { store.addAttachment({ id: 'worse', - name: 'worse.mp4', + name: 'worse.pdf', ref: '', - validationError: 'invalid-video' + validationError: 'unsupported-type' }) await user.unhover(sendButton) sendButton.focus() expect(await screen.findByRole('tooltip')).toHaveTextContent( - '2 files aren’t valid videos. Remove them to send.' + '2 files can’t be sent. Remove them to send.' + ) + expect(sendButton).toHaveAccessibleDescription( + '2 files can’t be sent. Remove them to send.' ) store.removeAttachment('bad') @@ -318,6 +321,37 @@ describe('composer asset tray', () => { expect(send).toHaveBeenCalledExactlyOnceWith('Keep my draft', []) }) + it('names an unaccepted file type as the reason Send is blocked until it is removed', async () => { + const user = userEvent.setup() + const { store, editor, send } = setup() + store.addAttachment(asset) + store.addAttachment({ + id: 'doc', + name: 'doc.pdf', + ref: '', + validationError: 'unsupported-type' + }) + await user.type(editor, 'Keep my draft') + const sendButton = screen.getByRole('button', { name: 'Send' }) + const reason = + 'doc.pdf isn’t a file type the agent accepts. Remove it to send.' + + expect(sendButton).toHaveAttribute('aria-disabled', 'true') + expect(sendButton).toHaveAccessibleDescription(reason) + await user.hover(sendButton) + expect(await screen.findByRole('tooltip')).toHaveTextContent(reason) + await user.click(editor) + await user.keyboard('{Enter}') + expect(send).not.toHaveBeenCalled() + + store.removeAttachment('doc') + await waitFor(() => expect(sendButton).not.toHaveAttribute('aria-disabled')) + await user.click(sendButton) + expect(send).toHaveBeenCalledExactlyOnceWith('Keep my draft', [ + expect.objectContaining({ id: asset.id }) + ]) + }) + it('keeps Stop usable without the blocked-Send explanation while the agent runs', async () => { const user = userEvent.setup() const { store } = setup({ streaming: true }) @@ -516,7 +550,7 @@ describe('composer asset tray', () => { ) }) - it('leaves invalid videos out of @ asset mentions', async () => { + it('leaves invalid videos and unaccepted types out of @ asset mentions', async () => { const { store, editor } = setup() store.addAttachment(asset) store.addAttachment({ @@ -525,11 +559,20 @@ describe('composer asset tray', () => { ref: '', validationError: 'invalid-video' }) + store.addAttachment({ + id: 'doc', + name: 'doc.pdf', + ref: '', + validationError: 'unsupported-type' + }) await userEvent.type(editor, '@') expect(screen.getByRole('menuitem', { name: asset.name })).toBeVisible() expect( screen.queryByRole('menuitem', { name: 'bad.mov' }) ).not.toBeInTheDocument() + expect( + screen.queryByRole('menuitem', { name: 'doc.pdf' }) + ).not.toBeInTheDocument() await userEvent.keyboard('bad') expect( screen.queryByRole('menuitem', { name: 'bad.mov' }) diff --git a/src/workbench/extensions/agent/components/agent/Composer.pasteAttachment.test.ts b/src/workbench/extensions/agent/components/agent/Composer.pasteAttachment.test.ts index a9741c81a62..76ebcf12172 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.pasteAttachment.test.ts +++ b/src/workbench/extensions/agent/components/agent/Composer.pasteAttachment.test.ts @@ -34,7 +34,7 @@ describe('pasting files into the composer', () => { expect(view.emitted().attachFiles).toEqual([[[screenshot]]]) }) - it('attaches every attachable file in one paste', async () => { + it('forwards every pasted file, leaving the accept decision to the composer', async () => { const user = userEvent.setup() const view = mount() @@ -48,7 +48,11 @@ describe('pasting files into the composer', () => { ) const [[attached]] = view.emitted<[File[]]>().attachFiles - expect(attached.map((file) => file.name)).toEqual(['first.png', 'notes.md']) + expect(attached.map((file) => file.name)).toEqual([ + 'first.png', + 'notes.md', + 'archive.zip' + ]) }) it('attaches the file and still pastes the text a mixed clipboard carries', async () => { @@ -83,22 +87,6 @@ describe('pasting files into the composer', () => { expect(editor).toHaveTextContent('keep me') }) - it('keeps the selected text when only an unattachable file is pasted', async () => { - const user = userEvent.setup() - const view = mount() - const editor = screen.getByRole('textbox') - - await user.click(editor) - await user.paste('keep me') - await user.keyboard('{Control>}a{/Control}') - await user.paste( - clipboardOf(new File(['x'], 'archive.zip', { type: 'application/zip' })) - ) - - expect(view.emitted().attachFiles).toBeUndefined() - expect(editor).toHaveTextContent('keep me') - }) - it.for(['plain clipboard text', ''])( 'still pastes text and attaches nothing for %o', async (text) => { @@ -113,19 +101,4 @@ describe('pasting files into the composer', () => { expect(editor).toHaveTextContent(text) } ) - - it('leaves the paste to the editor when no clipboard file is attachable', async () => { - const user = userEvent.setup() - const view = mount() - const clipboard = clipboardOf( - new File(['x'], 'archive.zip', { type: 'application/zip' }) - ) - clipboard.setData('text/plain', 'fallback text') - - await user.click(screen.getByRole('textbox')) - await user.paste(clipboard) - - expect(view.emitted().attachFiles).toBeUndefined() - expect(screen.getByRole('textbox')).toHaveTextContent('fallback text') - }) }) diff --git a/src/workbench/extensions/agent/components/agent/Composer.vue b/src/workbench/extensions/agent/components/agent/Composer.vue index 422c14fddbb..c697b605152 100644 --- a/src/workbench/extensions/agent/components/agent/Composer.vue +++ b/src/workbench/extensions/agent/components/agent/Composer.vue @@ -21,7 +21,10 @@ import type { AgentStarterPromptAttribution } from '../../utils/starterPrompts' import { useAgentMentionPicker } from '../../composables/agent/useAgentMentionPicker' import { useSkillMenuDescription } from '../../composables/agent/useSkillMenuDescription' import { useWorkflowReferencePicker } from '../../composables/agent/useWorkflowReferencePicker' -import type { ComposerAttachment } from '../../types/composerAttachment' +import type { + AttachmentValidationError, + ComposerAttachment +} from '../../types/composerAttachment' import type { MentionSection } from '../../composables/agent/mentionPickerState' import { useComposer } from '../../composables/agent/useComposer' import type { SelectedNode } from '../../composables/agent/useCanvasSelection' @@ -362,16 +365,23 @@ const primaryActionLabel = computed(() => const primaryActionVariant = computed(() => running.value ? 'secondary' : 'inverted' ) +const SEND_BLOCKED_MESSAGES = { + 'invalid-video': 'agent.sendBlockedByInvalidVideo', + 'unsupported-type': 'agent.sendBlockedByUnsupportedType' +} as const satisfies Record + const sendBlockedReason = computed(() => { - const invalid = composer.attachments.value.filter( - (item) => item.validationError + const invalid = composer.attachments.value.flatMap( + ({ name, validationError }) => + validationError ? [{ name, validationError }] : [] ) if (running.value || invalid.length === 0) return undefined - return t( - 'agent.sendBlockedByInvalidVideo', - { name: invalid[0].name, count: invalid.length }, - invalid.length - ) + if (invalid.length > 1) + return t('agent.sendBlockedByInvalidAttachments', { + count: invalid.length + }) + const [{ name, validationError }] = invalid + return t(SEND_BLOCKED_MESSAGES[validationError], { name }) }) const primaryActionTooltip = computed( () => sendBlockedReason.value ?? primaryActionLabel.value diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.stories.ts b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.stories.ts index 1a9f06c9859..3ebf47f178c 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.stories.ts +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.stories.ts @@ -50,3 +50,10 @@ export const InvalidVideo: Story = { validationError: 'invalid-video' } } + +export const UnsupportedType: Story = { + args: { + name: 'report.pdf', + validationError: 'unsupported-type' + } +} diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts index 40a2366af3f..a45870cef2c 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.test.ts @@ -12,6 +12,7 @@ import { nextTick } from 'vue' import type { MediaKind } from '@/platform/assets/schemas/mediaAssetSchema' import { i18n } from '@/i18n' +import type { AttachmentValidationError } from '../../../types/composerAttachment' import AttachmentChip from './AttachmentChip.vue' function renderChip(props: { @@ -20,7 +21,7 @@ function renderChip(props: { mediaUrl?: string mediaKind?: MediaKind uploading?: boolean - validationError?: 'invalid-video' + validationError?: AttachmentValidationError }) { const rendered = render(AttachmentChip, { props, @@ -443,4 +444,26 @@ describe('AttachmentChip', () => { await user.keyboard('{Enter}') expect(emitted('remove')).toHaveLength(1) }) + + it.for([ + { name: 'doc.pdf', mediaKind: undefined }, + { name: 'clip.wmv', mediaKind: 'video' as const } + ])( + 'shows $name of an unaccepted type as a generic file with the type explanation', + ({ name, mediaKind }) => { + renderChip({ + name, + mediaKind, + previewUrl: 'blob:preview', + validationError: 'unsupported-type' + }) + const message = `${name} isn’t a file type the agent accepts. Remove it and try another file.` + + expect(screen.getByRole('status')).toHaveAccessibleName(message) + expect(screen.queryByRole('img')).not.toBeInTheDocument() + expect( + screen.queryByRole('button', { name: `Preview ${name}` }) + ).not.toBeInTheDocument() + } + ) }) diff --git a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue index 777b2bddec0..cce86424115 100644 --- a/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue +++ b/src/workbench/extensions/agent/components/agent/composer/AttachmentChip.vue @@ -1,5 +1,6 @@