diff --git a/packages/ingest-types/src/index.ts b/packages/ingest-types/src/index.ts index 0c9a69727e6..a677935b768 100644 --- a/packages/ingest-types/src/index.ts +++ b/packages/ingest-types/src/index.ts @@ -281,6 +281,11 @@ export type { ClearWorkspaceDeploymentResponses, ClientOptions, CloudNodeCompatibility, + CompleteModelUploadData, + CompleteModelUploadError, + CompleteModelUploadErrors, + CompleteModelUploadResponse, + CompleteModelUploadResponses, ContentCatalogProjection, ContentCatalogRecord, CreateAssetData, @@ -1228,6 +1233,7 @@ export type { ModelDownloadFrame, ModelFile, ModelFolder, + ModelUploadStarted, MoveUserdataFileData, MoveUserdataFileError, MoveUserdataFileErrors, @@ -1476,6 +1482,11 @@ export type { StageCccChangesErrors, StageCccChangesResponse, StageCccChangesResponses, + StartModelUploadData, + StartModelUploadError, + StartModelUploadErrors, + StartModelUploadResponse, + StartModelUploadResponses, StartSsoConfirmedData, StartSsoConfirmedError, StartSsoConfirmedErrors, diff --git a/packages/ingest-types/src/types.gen.ts b/packages/ingest-types/src/types.gen.ts index fb955b23409..50af5b69eae 100644 --- a/packages/ingest-types/src/types.gen.ts +++ b/packages/ingest-types/src/types.gen.ts @@ -473,6 +473,35 @@ export type ValidationResult = { warnings?: Array } +/** + * Where and how the browser sends a model file's bytes (POST /api/assets/uploads). + */ +export type ModelUploadStarted = { + /** + * Names this upload in `POST /api/assets/uploads/{upload_id}/complete`. + */ + upload_id: string + /** + * Signed URL for exactly one staged object; works until `expires_at`. + */ + upload_url: string + method: 'PUT' + /** + * Every header the PUT must send, with exactly these values (`Content-Type`, `x-goog-content-length-range` and `x-goog-if-generation-match` set to `0`, which makes storage refuse a second write to the object). + */ + headers: { + [key: string]: string + } + /** + * When `upload_url` stops working (15 minutes after the answer). + */ + expires_at: string + /** + * The largest file an upload accepts, 2147483648 (2 GiB). + */ + max_size_bytes: number +} + /** * Details of a single validation error encountered during asset operations. */ @@ -10286,6 +10315,123 @@ export type CreateAssetDownloadResponses = { export type CreateAssetDownloadResponse = CreateAssetDownloadResponses[keyof CreateAssetDownloadResponses] +export type StartModelUploadData = { + body: { + /** + * A bare file name ending in `.safetensors` (any case), at most 255 bytes, no path, no leading dot. + */ + filename: string + /** + * The file's size, 1 to 2147483648 (2 GiB). + */ + size_bytes: number + /** + * Exactly `models` plus one model folder from `model_upload_folders` in GET /api/features (`loras`, `checkpoints`, `vae`, `embeddings`, ...); another folder is `400 UNSUPPORTED_MODEL_TYPE`. + */ + tags: Array + } + path?: never + query?: never + url: '/api/assets/uploads' +} + +export type StartModelUploadErrors = { + /** + * `INVALID_BODY` (bad filename, size or tag list), `UNSUPPORTED_FILE_TYPE` (not `.safetensors`: a pickle format such as `.ckpt`, `.pt` or `.bin` can run code when loaded), `UNSUPPORTED_MODEL_TYPE` (`models` plus a folder not in `model_upload_folders` of GET /api/features) or `FILE_TOO_LARGE` (over 2 GiB; the message is "Files can be up to 2 GB.") + */ + 400: ErrorResponse + /** + * Unauthorized + */ + 401: ErrorResponse + /** + * `FEATURE_DISABLED` when `runtime_library_models_enabled` is off for this workspace, `PRIVATE_MODELS_DISABLED` when the workspace's plan does not include private models (the link import's check), or a refused session write (see `SessionWriteForbidden`) + */ + 403: ErrorResponse + /** + * Internal server error + */ + 500: ErrorResponse + /** + * Private-model eligibility is temporarily unavailable (`PRIVATE_MODELS_UNAVAILABLE`); retry without inferring plan ineligibility. + */ + 503: ErrorResponse +} + +export type StartModelUploadError = + StartModelUploadErrors[keyof StartModelUploadErrors] + +export type StartModelUploadResponses = { + /** + * Signed upload URL for one staged object + */ + 201: ModelUploadStarted +} + +export type StartModelUploadResponse = + StartModelUploadResponses[keyof StartModelUploadResponses] + +export type CompleteModelUploadData = { + body: { + /** + * The `filename` sent to start the upload. + */ + filename: string + /** + * The `tags` sent to start the upload. + */ + tags: Array + } + path: { + /** + * The `upload_id` from `POST /api/assets/uploads`. + */ + upload_id: string + } + query?: never + url: '/api/assets/uploads/{upload_id}/complete' +} + +export type CompleteModelUploadErrors = { + /** + * `INVALID_BODY` (bad filename or tag list), `UNSUPPORTED_FILE_TYPE` (not `.safetensors`), `UNSUPPORTED_MODEL_TYPE` (a folder not in `model_upload_folders`) or `FILE_TOO_LARGE` (the stored object is over 2 GiB; the message is "Files can be up to 2 GB.") + */ + 400: ErrorResponse + /** + * Unauthorized + */ + 401: ErrorResponse + /** + * `FEATURE_DISABLED` when `runtime_library_models_enabled` is off for this workspace, `PRIVATE_MODELS_DISABLED` when the workspace's plan does not include private models (the link import's check), or a refused session write (see `SessionWriteForbidden`) + */ + 403: ErrorResponse + /** + * `UPLOAD_NOT_FOUND`: no uploaded object for this `upload_id` under this caller and workspace (not uploaded yet, cleaned up, or someone else's) + */ + 404: ErrorResponse + /** + * Internal server error + */ + 500: ErrorResponse + /** + * Private-model eligibility is temporarily unavailable (`PRIVATE_MODELS_UNAVAILABLE`); retry without inferring plan ineligibility. + */ + 503: ErrorResponse +} + +export type CompleteModelUploadError = + CompleteModelUploadErrors[keyof CompleteModelUploadErrors] + +export type CompleteModelUploadResponses = { + /** + * Accepted; a task stores the file as an asset + */ + 202: AssetDownloadResponse +} + +export type CompleteModelUploadResponse = + CompleteModelUploadResponses[keyof CompleteModelUploadResponses] + export type CreateAssetExportData = { body: { /** @@ -13243,6 +13389,10 @@ export type GetFeaturesResponses = { * Maximum upload size in bytes */ max_upload_size?: number + /** + * The model types (folder names, sorted) a model file uploaded from disk may be: the list POST /api/assets/uploads checks, which answers any other with 400 UNSUPPORTED_MODEL_TYPE. Present only while runtime_library_models_enabled is on for the caller. + */ + model_upload_folders?: Array /** * Whether new free-tier subscriptions are enabled for this caller. Current servers always emit a boolean; clients should tolerate absence when talking to older servers that predate this declared field. */ diff --git a/packages/ingest-types/src/zod.gen.ts b/packages/ingest-types/src/zod.gen.ts index 3b6342952f3..625c8888ec6 100644 --- a/packages/ingest-types/src/zod.gen.ts +++ b/packages/ingest-types/src/zod.gen.ts @@ -4396,6 +4396,25 @@ export const zAssetMetadataResponse = z.object({ validation: zValidationResult.optional() }) +/** + * Where and how the browser sends a model file's bytes (POST /api/assets/uploads). + */ +export const zModelUploadStarted = z.object({ + upload_id: z.uuid(), + upload_url: z.string(), + method: z.enum(['PUT']), + headers: z.record(z.string(), z.string()), + expires_at: z.iso.datetime(), + max_size_bytes: z.coerce + .bigint() + .min(BigInt('-9223372036854775808'), { + error: 'Invalid value: Expected int64 to be >= -9223372036854775808' + }) + .max(BigInt('9223372036854775807'), { + error: 'Invalid value: Expected int64 to be <= 9223372036854775807' + }) +}) + /** * Acknowledgement of an async asset download task; clients poll GET /api/tasks/{task_id} for status. */ @@ -5460,6 +5479,38 @@ export const zCreateAssetDownloadResponse = z.union([ zAssetDownloadResponse ]) +export const zStartModelUploadBody = z.object({ + filename: z.string(), + size_bytes: z.coerce + .bigint() + .min(BigInt('-9223372036854775808'), { + error: 'Invalid value: Expected int64 to be >= -9223372036854775808' + }) + .max(BigInt('9223372036854775807'), { + error: 'Invalid value: Expected int64 to be <= 9223372036854775807' + }), + tags: z.array(z.string()) +}) + +/** + * Signed upload URL for one staged object + */ +export const zStartModelUploadResponse = zModelUploadStarted + +export const zCompleteModelUploadBody = z.object({ + filename: z.string(), + tags: z.array(z.string()) +}) + +export const zCompleteModelUploadPath = z.object({ + upload_id: z.string() +}) + +/** + * Accepted; a task stores the file as an asset + */ +export const zCompleteModelUploadResponse = zAssetDownloadResponse + export const zCreateAssetExportBody = z.object({ asset_ids: z.array(z.string()).optional(), include_previews: z.boolean().optional().default(false), @@ -6002,6 +6053,7 @@ export const zGetFeaturesResponse = z.object({ }) .optional(), max_upload_size: z.int().optional(), + model_upload_folders: z.array(z.string()).optional(), new_free_tier_subscriptions: z.boolean().optional(), private_workflow_links_enabled: z.boolean().optional(), sso_enabled: z.boolean().optional(), diff --git a/src/composables/__mocks__/useFeatureFlags.ts b/src/composables/__mocks__/useFeatureFlags.ts index ae0e4116890..67e790cbf72 100644 --- a/src/composables/__mocks__/useFeatureFlags.ts +++ b/src/composables/__mocks__/useFeatureFlags.ts @@ -29,6 +29,7 @@ const defaultFlags: FeatureFlags = { deploymentCompatibilityMarksEnabled: false, lateCustomNodesEnabled: false, runtimeLibraryModelsEnabled: false, + modelUploadFolders: [], hostedBillingDestination: 'stripe', hostedBillingWebEnabled: false, showSignInButton: undefined, diff --git a/src/composables/useFeatureFlags.test.ts b/src/composables/useFeatureFlags.test.ts index 3e139098eb3..a266b519686 100644 --- a/src/composables/useFeatureFlags.test.ts +++ b/src/composables/useFeatureFlags.test.ts @@ -642,6 +642,30 @@ describe('useFeatureFlags', () => { }) }) + describe('modelUploadFolders', () => { + afterEach(() => { + remoteConfig.value = {} + }) + + it('lists the folders the server accepts uploads into', () => { + remoteConfig.value = { model_upload_folders: ['checkpoints', 'loras'] } + + const { flags } = useFeatureFlags() + + expect(flags.modelUploadFolders).toEqual(['checkpoints', 'loras']) + }) + + it('is empty when the server sends no list', () => { + vi.mocked(api.getServerFeature).mockImplementation( + (_path, defaultValue) => defaultValue + ) + + const { flags } = useFeatureFlags() + + expect(flags.modelUploadFolders).toEqual([]) + }) + }) + describe('dev override via localStorage', () => { it('resolveFlag returns localStorage override over remoteConfig and server value', () => { vi.mocked(api.getServerFeature).mockReturnValue(false) diff --git a/src/composables/useFeatureFlags.ts b/src/composables/useFeatureFlags.ts index 00403090fee..ec12fc0994f 100644 --- a/src/composables/useFeatureFlags.ts +++ b/src/composables/useFeatureFlags.ts @@ -245,6 +245,14 @@ export function useFeatureFlags() { false ) }, + /** The model folders an uploaded model file may go into; empty when uploads are off. */ + get modelUploadFolders(): readonly string[] { + return resolveFlag( + ServerFeatureFlag.MODEL_UPLOAD_FOLDERS, + remoteConfig.value.model_upload_folders, + [] + ) + }, get lateCustomNodesEnabled() { return resolveFlag( ServerFeatureFlag.LATE_CUSTOM_NODES_ENABLED, diff --git a/src/locales/en/main.json b/src/locales/en/main.json index bc809a0b813..1c4dbb082a7 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -4530,6 +4530,14 @@ "fileName": "File Name", "fileSize": "File Size", "filterBy": "Filter by", + "fileUpload": { + "cancelled": "Upload canceled.", + "failed": "Upload failed. Try again.", + "notSafetensors": "Only .safetensors files can be uploaded.", + "planUnavailable": "Couldn't check your plan. Try again.", + "privateModelsDisabled": "Your plan doesn't include private models.", + "unsupportedType": "Cloud can't store this model type yet. Choose another type." + }, "findInLibrary": "Find it in the {type} section of the models library.", "finish": "Finish", "importAnother": "Import Another", diff --git a/src/platform/remoteConfig/serverFeatureFlag.ts b/src/platform/remoteConfig/serverFeatureFlag.ts index d16253423d9..e4b983bd9ac 100644 --- a/src/platform/remoteConfig/serverFeatureFlag.ts +++ b/src/platform/remoteConfig/serverFeatureFlag.ts @@ -22,6 +22,7 @@ export enum ServerFeatureFlag { DEPLOYMENT_COMPATIBILITY_MARKS_ENABLED = 'deployment_compatibility_marks_enabled', LATE_CUSTOM_NODES_ENABLED = 'late_custom_nodes_enabled', RUNTIME_LIBRARY_MODELS_ENABLED = 'runtime_library_models_enabled', + MODEL_UPLOAD_FOLDERS = 'model_upload_folders', HOSTED_BILLING_DESTINATION = 'hosted_billing_destination', SHOW_SIGNIN_BUTTON = 'show_signin_button', UNIFIED_CLOUD_AUTH = 'unified_cloud_auth', diff --git a/src/platform/workspace/api/modelFileUploadApi.test.ts b/src/platform/workspace/api/modelFileUploadApi.test.ts new file mode 100644 index 00000000000..4af33a6857d --- /dev/null +++ b/src/platform/workspace/api/modelFileUploadApi.test.ts @@ -0,0 +1,312 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { api } from '@/scripts/api' + +import { modelFileUploadApi } from './modelFileUploadApi' + +vi.mock(import('@/scripts/api')) + +const UPLOAD_ID = '2f4b8c1e-6a0d-4e7b-9c3a-1d5e8f7a9b20' +const TASK_ID = '8c2d4e6f-1a3b-4c5d-9e7f-0a1b2c3d4e5f' +const SIGNED_URL = 'https://storage.example.com/temp/model-uploads/x?sig=1' +const SIGNED_HEADERS = { + 'Content-Type': 'application/octet-stream', + 'x-goog-content-length-range': '0,12', + 'x-goog-if-generation-match': '0' +} + +class StorageRequest { + static sent: StorageRequest[] = [] + method = '' + url = '' + headers: Record = {} + body: unknown + status = 0 + aborted = false + upload: { onprogress: ((event: { loaded: number }) => void) | null } = { + onprogress: null + } + onload: (() => void) | null = null + onerror: (() => void) | null = null + onabort: (() => void) | null = null + onloadend: (() => void) | null = null + + open(method: string, url: string) { + this.method = method + this.url = url + } + setRequestHeader(name: string, value: string) { + this.headers[name] = value + } + send(body: unknown) { + this.body = body + StorageRequest.sent.push(this) + } + abort() { + this.aborted = true + this.onabort?.() + this.onloadend?.() + } + progress(loaded: number) { + this.upload.onprogress?.({ loaded }) + } + finish(status: number) { + this.status = status + this.onload?.() + this.onloadend?.() + } + fail() { + this.onerror?.() + this.onloadend?.() + } +} + +function json(status: number, body: unknown) { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' } + }) +} + +const started = { + upload_id: UPLOAD_ID, + upload_url: SIGNED_URL, + method: 'PUT', + headers: SIGNED_HEADERS, + expires_at: '2026-10-29T15:30:00Z', + max_size_bytes: 2147483648 +} + +function serverAnswers(...answers: Array) { + const calls: Array<{ route: string; method?: string; body: unknown }> = [] + vi.mocked(api.fetchApi).mockImplementation((route, options) => { + calls.push({ + route, + method: options?.method, + body: JSON.parse(String(options?.body)) + }) + const answer = + answers.shift() ?? json(500, { code: 'INTERNAL', message: 'x' }) + if (answer !== 'never') return Promise.resolve(answer) + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => + reject(new DOMException('Aborted', 'AbortError')) + ) + }) + }) + return calls +} + +async function storageRequest(nth = 1) { + await vi.waitFor(() => expect(StorageRequest.sent).toHaveLength(nth)) + return StorageRequest.sent[nth - 1] +} + +const file = new File(['twelve bytes'], 'my_style_v2.safetensors') +const tags = ['models', 'loras'] + +describe('modelFileUploadApi.uploadModelFile', () => { + beforeEach(() => { + StorageRequest.sent = [] + vi.stubGlobal('XMLHttpRequest', StorageRequest) + }) + + function upload(signal = new AbortController().signal) { + const onProgress = vi.fn() + const result = modelFileUploadApi.uploadModelFile(file, { + tags, + signal, + onProgress + }) + return { result, onProgress } + } + + it('starts, puts the bytes straight to storage with the signed headers, then completes', async () => { + const calls = serverAnswers( + json(201, started), + json(202, { task_id: TASK_ID, status: 'created' }) + ) + const { result, onProgress } = upload() + + const put = await storageRequest() + expect(put.method).toBe('PUT') + expect(put.url).toBe(SIGNED_URL) + expect(put.headers).toEqual(SIGNED_HEADERS) + expect(put.body).toBe(file) + put.progress(5) + put.finish(200) + + await expect(result).resolves.toEqual({ ok: true, taskId: TASK_ID }) + expect(onProgress).toHaveBeenCalledWith(5) + expect(calls).toEqual([ + { + route: '/assets/uploads', + method: 'POST', + body: { filename: file.name, size_bytes: file.size, tags } + }, + { + route: `/assets/uploads/${UPLOAD_ID}/complete`, + method: 'POST', + body: { filename: file.name, tags } + } + ]) + }) + + it.for([ + { + refusal: 'the file type', + status: 400, + body: { code: 'UNSUPPORTED_FILE_TYPE', message: 'not safetensors' }, + expected: { ok: false, reason: 'unsupportedFile' } + }, + { + refusal: 'the file size, with its limit message', + status: 400, + body: { code: 'FILE_TOO_LARGE', message: 'Files can be up to 2 GB.' }, + expected: { + ok: false, + reason: 'tooLarge', + message: 'Files can be up to 2 GB.' + } + }, + { + refusal: 'the model folder', + status: 400, + body: { code: 'UNSUPPORTED_MODEL_TYPE', message: 'not stored' }, + expected: { ok: false, reason: 'unsupportedType' } + }, + { + refusal: 'a plan without private models', + status: 403, + body: { code: 'PRIVATE_MODELS_DISABLED', message: 'from the server' }, + expected: { ok: false, reason: 'privateModelsDisabled' } + }, + { + refusal: 'a plan it could not read', + status: 503, + body: { code: 'PRIVATE_MODELS_UNAVAILABLE', message: 'from the server' }, + expected: { ok: false, reason: 'planUnavailable' } + } + ])( + 'sends no bytes when the start call refuses $refusal', + async ({ status, body, expected }) => { + serverAnswers(json(status, body)) + + await expect(upload().result).resolves.toEqual(expected) + expect(StorageRequest.sent).toHaveLength(0) + } + ) + + it('says the model type is refused when the finish call refuses it', async () => { + serverAnswers( + json(201, started), + json(400, { code: 'UNSUPPORTED_MODEL_TYPE', message: 'not stored' }) + ) + const { result } = upload() + + ;(await storageRequest()).finish(200) + + await expect(result).resolves.toEqual({ + ok: false, + reason: 'unsupportedType' + }) + }) + + it.for([ + [403, 'PRIVATE_MODELS_DISABLED', 'privateModelsDisabled'], + [503, 'PRIVATE_MODELS_UNAVAILABLE', 'planUnavailable'], + [400, 'UNSUPPORTED_FILE_TYPE', 'unsupportedFile'] + ] as const)( + 'stops when the finish call answers %s %s', + async ([status, code, reason]) => { + serverAnswers(json(201, started), json(status, { code, message: 'x' })) + const { result } = upload() + + ;(await storageRequest()).finish(200) + + await expect(result).resolves.toEqual({ ok: false, reason }) + } + ) + + it('finishes the same upload when storage already holds its object', async () => { + const calls = serverAnswers( + json(201, started), + json(202, { task_id: TASK_ID, status: 'created' }) + ) + const { result } = upload() + + ;(await storageRequest()).finish(412) + + await expect(result).resolves.toEqual({ ok: true, taskId: TASK_ID }) + expect(StorageRequest.sent).toHaveLength(1) + expect(calls.map((call) => call.route)).toEqual([ + '/assets/uploads', + `/assets/uploads/${UPLOAD_ID}/complete` + ]) + }) + + it('fails without completing when storage refuses the bytes', async () => { + const calls = serverAnswers(json(201, started)) + const { result } = upload() + + ;(await storageRequest()).finish(403) + + await expect(result).resolves.toMatchObject({ ok: false, reason: 'failed' }) + expect(calls).toHaveLength(1) + }) + + it('fails when the connection drops during the upload', async () => { + serverAnswers(json(201, started)) + const { result } = upload() + + ;(await storageRequest()).fail() + + await expect(result).resolves.toMatchObject({ ok: false, reason: 'failed' }) + }) + + it('stops the transfer and never completes when cancelled', async () => { + const calls = serverAnswers(json(201, started)) + const controller = new AbortController() + const { result } = upload(controller.signal) + + const put = await storageRequest() + controller.abort() + + await expect(result).resolves.toEqual({ ok: false, reason: 'cancelled' }) + expect(put.aborted).toBe(true) + expect(calls).toHaveLength(1) + }) + + it('stops waiting for the upload URL when cancelled', async () => { + serverAnswers('never') + const controller = new AbortController() + const { result } = upload(controller.signal) + await vi.waitFor(() => expect(api.fetchApi).toHaveBeenCalledOnce()) + + controller.abort() + + await expect(result).resolves.toEqual({ ok: false, reason: 'cancelled' }) + expect(StorageRequest.sent).toHaveLength(0) + }) + + it('stops waiting for the finish answer when cancelled', async () => { + serverAnswers(json(201, started), 'never') + const controller = new AbortController() + const { result } = upload(controller.signal) + ;(await storageRequest()).finish(200) + await vi.waitFor(() => expect(api.fetchApi).toHaveBeenCalledTimes(2)) + + controller.abort() + + await expect(result).resolves.toEqual({ ok: false, reason: 'cancelled' }) + }) + + it('fails when the finish answer is not a task', async () => { + serverAnswers(json(201, started), json(202, { status: 'created' })) + const { result } = upload() + + ;(await storageRequest()).finish(200) + + await expect(result).resolves.toMatchObject({ ok: false, reason: 'failed' }) + }) +}) diff --git a/src/platform/workspace/api/modelFileUploadApi.ts b/src/platform/workspace/api/modelFileUploadApi.ts new file mode 100644 index 00000000000..5b6f4adb815 --- /dev/null +++ b/src/platform/workspace/api/modelFileUploadApi.ts @@ -0,0 +1,155 @@ +import type { + CompleteModelUploadData, + ModelUploadStarted, + StartModelUploadData +} from '@comfyorg/ingest-types' +import { + zCompleteModelUploadResponse, + zErrorResponse, + zStartModelUploadResponse +} from '@comfyorg/ingest-types/zod' + +import { api } from '@/scripts/api' + +const UPLOADS_ENDPOINT = '/assets/uploads' +const STORAGE_OBJECT_ALREADY_WRITTEN = 412 + +type ModelFileUploadFailure = + | 'unsupportedFile' + | 'unsupportedType' + | 'tooLarge' + | 'privateModelsDisabled' + | 'planUnavailable' + | 'cancelled' + | 'failed' + +const FAILURE_BY_CODE: Partial> = { + UNSUPPORTED_FILE_TYPE: 'unsupportedFile', + UNSUPPORTED_MODEL_TYPE: 'unsupportedType', + PRIVATE_MODELS_DISABLED: 'privateModelsDisabled', + PRIVATE_MODELS_UNAVAILABLE: 'planUnavailable' +} + +interface ModelFileUploadStopped { + ok: false + reason: ModelFileUploadFailure + message?: string + cause?: unknown +} + +/** The outcome of sending one model file: the server task that stores it, or why it stopped. */ +export type ModelFileUploadResult = + | { ok: true; taskId: string } + | ModelFileUploadStopped + +interface ModelFileUploadOptions { + tags: string[] + signal: AbortSignal + onProgress: (sentBytes: number) => void +} + +async function readFailure(res: Response): Promise { + const body = zErrorResponse.safeParse(await res.json().catch(() => null)) + const code = body.success ? body.data.code : undefined + const reason = code ? FAILURE_BY_CODE[code] : undefined + if (reason) return { ok: false, reason } + if (code === 'FILE_TOO_LARGE' && body.success) { + return { ok: false, reason: 'tooLarge', message: body.data.message } + } + return { ok: false, reason: 'failed', cause: `${res.status} ${code ?? ''}` } +} + +function putFile( + start: Pick, + file: File, + { signal, onProgress }: ModelFileUploadOptions +): Promise { + return new Promise((resolve) => { + if (signal.aborted) { + resolve({ ok: false, reason: 'cancelled' }) + return + } + const request = new XMLHttpRequest() + request.open(start.method, start.upload_url) + for (const [name, value] of Object.entries(start.headers)) { + request.setRequestHeader(name, value) + } + request.upload.onprogress = (event) => onProgress(event.loaded) + request.onload = () => resolve(readStorageAnswer(request.status)) + request.onerror = () => + resolve({ ok: false, reason: 'failed', cause: 'storage network error' }) + request.onabort = () => resolve({ ok: false, reason: 'cancelled' }) + const abort = () => request.abort() + request.onloadend = () => signal.removeEventListener('abort', abort) + signal.addEventListener('abort', abort, { once: true }) + request.send(file) + }) +} + +function readStorageAnswer(status: number): ModelFileUploadStopped | undefined { + if (status >= 200 && status < 300) return undefined + if (status === STORAGE_OBJECT_ALREADY_WRITTEN) return undefined + return { ok: false, reason: 'failed', cause: `storage ${status}` } +} + +async function postJson(route: string, body: unknown, signal: AbortSignal) { + return api.fetchApi(route, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal + }) +} + +async function stageFile( + file: File, + options: ModelFileUploadOptions +): Promise { + const startRes = await postJson( + UPLOADS_ENDPOINT, + { + filename: file.name, + size_bytes: file.size, + tags: options.tags + } satisfies StartModelUploadData['body'], + options.signal + ) + if (!startRes.ok) return await readFailure(startRes) + const start = zStartModelUploadResponse.safeParse(await startRes.json()) + if (!start.success) { + return { ok: false, reason: 'failed', cause: start.error } + } + return (await putFile(start.data, file, options)) ?? start.data.upload_id +} + +async function uploadModelFile( + file: File, + options: ModelFileUploadOptions +): Promise { + const { tags, signal } = options + try { + const staged = await stageFile(file, options) + if (typeof staged !== 'string') return staged + + const completeRes = await postJson( + `${UPLOADS_ENDPOINT}/${encodeURIComponent(staged)}/complete`, + { filename: file.name, tags } satisfies CompleteModelUploadData['body'], + signal + ) + if (!completeRes.ok) return await readFailure(completeRes) + const task = zCompleteModelUploadResponse.safeParse( + await completeRes.json() + ) + if (!task.success) { + return { ok: false, reason: 'failed', cause: task.error } + } + return { ok: true, taskId: task.data.task_id } + } catch (cause) { + return signal.aborted + ? { ok: false, reason: 'cancelled' } + : { ok: false, reason: 'failed', cause } + } +} + +/** Sends a model file from the computer into the workspace library; never throws. */ +export const modelFileUploadApi = { uploadModelFile } diff --git a/src/platform/workspace/composables/useUploadModelFile.test.ts b/src/platform/workspace/composables/useUploadModelFile.test.ts new file mode 100644 index 00000000000..07c4c97f631 --- /dev/null +++ b/src/platform/workspace/composables/useUploadModelFile.test.ts @@ -0,0 +1,519 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createApp, nextTick } from 'vue' +import type { App } from 'vue' +import { createI18n } from 'vue-i18n' + +import { useFeatureFlags } from '@/composables/useFeatureFlags' +import enMessages from '@/locales/en/main.json' with { type: 'json' } +import { modelFileUploadApi } from '@/platform/workspace/api/modelFileUploadApi' +import type { ModelFileUploadResult } from '@/platform/workspace/api/modelFileUploadApi' +import type { AssetDownloadWsMessage } from '@/platform/remote/comfyui/execution/types' +import { + TaskNotFoundError, + taskService +} from '@/platform/tasks/services/taskService' +import { reportError } from '@comfyorg/telemetry/reportError' +import { api } from '@/scripts/api' +import { useAssetDownloadStore } from '@/stores/assetDownloadStore' + +import { useUploadModelFile } from './useUploadModelFile' + +vi.mock(import('@/scripts/api')) +vi.mock(import('@/composables/useFeatureFlags')) +vi.mock(import('@comfyorg/telemetry/reportError'), { spy: true }) + +const TASK_ID = '8c2d4e6f-1a3b-4c5d-9e7f-0a1b2c3d4e5f' +const lora = new File(['0123456789'], 'my_style_v2.safetensors') + +type UploadOptions = Parameters[1] + +const mountedApps: App[] = [] + +function setup() { + let upload: ReturnType | undefined + const app = createApp({ + setup() { + upload = useUploadModelFile() + return () => null + } + }) + app.use( + createI18n({ legacy: false, locale: 'en', messages: { en: enMessages } }) + ) + app.mount(document.createElement('div')) + mountedApps.push(app) + if (!upload) throw new Error('composable did not run') + return { upload, app } +} + +function pendingUpload() { + let options: UploadOptions | undefined + let finish!: (result: ModelFileUploadResult) => void + vi.spyOn(modelFileUploadApi, 'uploadModelFile').mockImplementation( + (_file, opts) => { + options = opts + return new Promise((resolve) => { + finish = resolve + }) + } + ) + return { + options: () => { + if (!options) throw new Error('upload not started') + return options + }, + finish: (result: ModelFileUploadResult) => finish(result) + } +} + +function taskUpdate( + status: AssetDownloadWsMessage['status'], + error?: string, + taskId = TASK_ID +) { + const listener = vi + .mocked(api.addEventListener) + .mock.calls.findLast(([type]) => type === 'asset_download')?.[1] + if (typeof listener !== 'function') throw new Error('no download listener') + listener( + new CustomEvent('asset_download', { + detail: { + task_id: taskId, + asset_id: 'asset-1', + asset_name: lora.name, + bytes_total: lora.size, + bytes_downloaded: lora.size, + progress: 1, + status, + error + } + }) + ) +} + +describe('useUploadModelFile', () => { + beforeEach(() => { + vi.mocked(reportError).mockImplementation(() => undefined) + const flags = vi.mocked(useFeatureFlags().flags) + flags.modelUploadFolders = ['checkpoints', 'loras'] + }) + + afterEach(() => { + for (const app of mountedApps.splice(0)) app.unmount() + }) + + it('refuses a file that is not .safetensors before any upload', () => { + const { upload } = setup() + const spy = vi.spyOn(modelFileUploadApi, 'uploadModelFile') + + upload.choose(new File(['x'], 'notes.txt')) + + expect(upload.state.value).toEqual({ + status: 'idle', + error: 'Only .safetensors files can be uploaded.' + }) + upload.choose(new File(['x'], 'STYLE.SAFETENSORS')) + expect(upload.state.value.status).toBe('chosen') + expect(spy).not.toHaveBeenCalled() + }) + + it('shows bytes sent, then saving, then done once the library task finishes', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + expect(server.options().tags).toEqual(['models', 'loras']) + server.options().onProgress(4) + expect(upload.state.value).toEqual({ + status: 'uploading', + file: lora, + sent: 4 + }) + + server.finish({ ok: true, taskId: TASK_ID }) + await expect(result).resolves.toEqual({ + filename: lora.name, + modelType: 'loras', + taskId: TASK_ID, + status: 'processing' + }) + expect(upload.state.value.status).toBe('saving') + + taskUpdate('completed') + await nextTick() + expect(upload.state.value).toEqual({ + status: 'done', + file: lora, + modelType: 'loras' + }) + expect(useAssetDownloadStore().lastCompletedDownload).toMatchObject({ + taskId: TASK_ID, + modelType: 'loras' + }) + }) + + it('uploads into the folder the person picked', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + upload.modelType.value = 'checkpoints' + + const result = upload.upload() + + expect(server.options().tags).toEqual(['models', 'checkpoints']) + server.finish({ ok: true, taskId: TASK_ID }) + await expect(result).resolves.toMatchObject({ modelType: 'checkpoints' }) + }) + + it('stops the transfer on cancel and says so, without reporting an error', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + upload.cancel() + expect(server.options().signal.aborted).toBe(true) + server.finish({ ok: false, reason: 'cancelled' }) + + await expect(result).resolves.toBeNull() + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: 'Upload canceled.' + }) + expect(reportError).not.toHaveBeenCalled() + }) + + it('reports a failed upload and offers to try again', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: false, reason: 'failed', cause: 'storage 403' }) + + await expect(result).resolves.toBeNull() + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: 'Upload failed. Try again.' + }) + expect(reportError).toHaveBeenCalledWith( + 'storage 403', + expect.objectContaining({ errorType: 'error_uploading_model_file' }) + ) + + const retry = pendingUpload() + const retried = upload.upload() + expect(retry.options().tags).toEqual(['models', 'loras']) + retry.finish({ ok: true, taskId: TASK_ID }) + await expect(retried).resolves.toMatchObject({ taskId: TASK_ID }) + }) + + it("shows the server's size limit when the file is too large", async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ + ok: false, + reason: 'tooLarge', + message: 'Files can be up to 2 GB.' + }) + + await result + expect(upload.state.value).toMatchObject({ + status: 'failed', + error: 'Files can be up to 2 GB.' + }) + }) + + it('asks for another type when the server cannot store the chosen one', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: false, reason: 'unsupportedType' }) + + await result + expect(upload.state.value).toMatchObject({ + status: 'failed', + error: "Cloud can't store this model type yet. Choose another type." + }) + expect(reportError).not.toHaveBeenCalled() + }) + + it.for([ + ['privateModelsDisabled', "Your plan doesn't include private models."], + ['planUnavailable', "Couldn't check your plan. Try again."], + ['unsupportedFile', 'Only .safetensors files can be uploaded.'] + ] as const)( + 'says what the server refused when it answers %s', + async ([reason, copy]) => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: false, reason }) + + await result + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: copy + }) + expect(reportError).not.toHaveBeenCalled() + } + ) + + it('offers only the folders the server accepts, LoRA first when listed', () => { + const { upload } = setup() + + upload.choose(lora) + + expect(upload.uploadFolders.value).toEqual(['checkpoints', 'loras']) + expect(upload.modelType.value).toBe('loras') + }) + + it('starts on the first accepted folder when LoRA is not one of them', () => { + vi.mocked(useFeatureFlags().flags).modelUploadFolders = [ + 'checkpoints', + 'vae' + ] + const { upload } = setup() + + upload.choose(lora) + + expect(upload.modelType.value).toBe('checkpoints') + }) + + it('starts each newly chosen file on the default type again', () => { + const { upload } = setup() + upload.choose(lora) + upload.modelType.value = 'checkpoints' + + upload.choose(new File(['x'], 'portrait.safetensors')) + + expect(upload.modelType.value).toBe('loras') + }) + + it('is offered exactly while the server lists upload folders', () => { + const flags = vi.mocked(useFeatureFlags().flags) + flags.runtimeLibraryModelsEnabled = false + const { upload } = setup() + expect(upload.isAvailable.value).toBe(true) + + flags.modelUploadFolders = [] + expect(upload.isAvailable.value).toBe(false) + }) + + it('shows why the library task failed after the bytes arrived', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + + taskUpdate('failed', 'This file is not a valid .safetensors file.') + await nextTick() + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: 'This file is not a valid .safetensors file.' + }) + }) + + it('refreshes the model lists when the library task finishes before the upload call answers', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + taskUpdate('completed') + server.finish({ ok: true, taskId: TASK_ID }) + await result + await nextTick() + + expect(upload.state.value).toEqual({ + status: 'done', + file: lora, + modelType: 'loras' + }) + expect(useAssetDownloadStore().lastCompletedDownload).toMatchObject({ + taskId: TASK_ID, + modelType: 'loras' + }) + }) + + it('stops saving when the server no longer has the library task', async () => { + vi.useFakeTimers({ shouldAdvanceTime: false }) + vi.spyOn(taskService, 'getTask').mockResolvedValue({ + ok: false, + error: new TaskNotFoundError(TASK_ID) + }) + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + await vi.advanceTimersByTimeAsync(30_000) + + expect(useAssetDownloadStore().downloadList[0]?.status).toBe('failed') + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: enMessages.progressToast.taskUnavailable + }) + }) + + it('says the upload was canceled when the library task is canceled', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + + taskUpdate('cancelled') + await nextTick() + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: 'Upload canceled.' + }) + }) + + it('says the upload was canceled when the cancel is never confirmed, and finishes if the task completes after its row is dismissed', async () => { + vi.useFakeTimers({ shouldAdvanceTime: false }) + vi.spyOn(taskService, 'cancelTask').mockResolvedValue({ + ok: true, + value: 'cancelling' + }) + vi.spyOn(taskService, 'getTask').mockResolvedValue({ + ok: true, + value: { + id: TASK_ID, + idempotency_key: 'upload-1', + task_name: 'task:download_file', + payload: {}, + status: 'running', + create_time: new Date().toISOString(), + update_time: new Date().toISOString() + } + }) + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + taskUpdate('running') + + const downloads = useAssetDownloadStore() + await downloads.cancelDownload(TASK_ID) + await vi.advanceTimersByTimeAsync(120_000) + expect(downloads.downloadList[0]?.status).toBe('cancellation_unconfirmed') + expect(upload.state.value).toEqual({ + status: 'failed', + file: lora, + error: 'Upload canceled.' + }) + + downloads.clearDismissibleDownloads() + expect(downloads.downloadList).toEqual([]) + taskUpdate('completed') + await nextTick() + expect(upload.state.value).toEqual({ + status: 'done', + file: lora, + modelType: 'loras' + }) + }) + + it('keeps saving when a different import finishes', async () => { + const otherImport = '0f9e8d7c-6b5a-4c3d-8e2f-1a0b9c8d7e6f' + const { upload } = setup() + taskUpdate('running', undefined, otherImport) + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + + taskUpdate('completed', undefined, otherImport) + await nextTick() + expect(upload.state.value).toEqual({ status: 'saving', file: lora }) + }) + + it('stops the earlier transfer when another file is chosen', async () => { + const otherFile = new File(['01234'], 'teammate_style.safetensors') + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + upload.choose(otherFile) + server.options().onProgress(4) + server.finish({ ok: true, taskId: TASK_ID }) + + await expect(result).resolves.toBeNull() + expect(server.options().signal.aborted).toBe(true) + expect(upload.state.value).toEqual({ status: 'chosen', file: otherFile }) + }) + + it('stops following the earlier library task when another file is chosen', async () => { + const otherFile = new File(['01234'], 'teammate_style.safetensors') + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + const result = upload.upload() + server.finish({ ok: true, taskId: TASK_ID }) + await result + + upload.choose(otherFile) + taskUpdate('completed') + await nextTick() + + expect(upload.state.value).toEqual({ status: 'chosen', file: otherFile }) + }) + + it('ignores an upload that answers after the person went back', async () => { + const { upload } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + const signal = server.options().signal + upload.reset() + server.finish({ ok: true, taskId: TASK_ID }) + + await expect(result).resolves.toBeNull() + expect(signal.aborted).toBe(true) + expect(upload.state.value).toEqual({ status: 'idle' }) + }) + + it('stops the transfer when the dialog closes', async () => { + const { upload, app } = setup() + const server = pendingUpload() + upload.choose(lora) + + const result = upload.upload() + app.unmount() + mountedApps.splice(0) + + expect(server.options().signal.aborted).toBe(true) + server.finish({ ok: false, reason: 'cancelled' }) + await expect(result).resolves.toBeNull() + }) +}) diff --git a/src/platform/workspace/composables/useUploadModelFile.ts b/src/platform/workspace/composables/useUploadModelFile.ts new file mode 100644 index 00000000000..fee9be1d124 --- /dev/null +++ b/src/platform/workspace/composables/useUploadModelFile.ts @@ -0,0 +1,192 @@ +import { computed, onScopeDispose, ref, shallowRef, watch } from 'vue' +import type { WatchStopHandle } from 'vue' +import { useI18n } from 'vue-i18n' + +import { useFeatureFlags } from '@/composables/useFeatureFlags' +import type { UploadModelSuccess } from '@/platform/assets/composables/useUploadModelWizard' +import { modelFileUploadApi } from '@/platform/workspace/api/modelFileUploadApi' +import type { ModelFileUploadResult } from '@/platform/workspace/api/modelFileUploadApi' +import { reportError } from '@comfyorg/telemetry/reportError' +import { + isDownloadCancelled, + useAssetDownloadStore +} from '@/stores/assetDownloadStore' + +const MODEL_FILE_EXTENSION = '.safetensors' +const DEFAULT_MODEL_TYPE = 'loras' + +type UploadStopped = Extract + +const FAILURE_COPY: Partial> = { + cancelled: 'assetBrowser.fileUpload.cancelled', + unsupportedFile: 'assetBrowser.fileUpload.notSafetensors', + unsupportedType: 'assetBrowser.fileUpload.unsupportedType', + privateModelsDisabled: 'assetBrowser.fileUpload.privateModelsDisabled', + planUnavailable: 'assetBrowser.fileUpload.planUnavailable' +} + +type ModelFileUpload = + | { status: 'idle'; error?: string } + | { status: 'chosen'; file: File } + | { status: 'uploading'; file: File; sent: number } + | { status: 'saving'; file: File } + | { status: 'done'; file: File; modelType: string } + | { status: 'failed'; file: File; error: string } + +/** + * One model file chosen from the computer and uploaded into the workspace + * library: choose it, pick its type, upload with progress or cancel. + */ +export function useUploadModelFile() { + const { t } = useI18n() + const { flags } = useFeatureFlags() + const downloads = useAssetDownloadStore() + const uploadFolders = computed(() => flags.modelUploadFolders) + const isAvailable = computed(() => uploadFolders.value.length > 0) + + function defaultModelType() { + return uploadFolders.value.includes(DEFAULT_MODEL_TYPE) + ? DEFAULT_MODEL_TYPE + : (uploadFolders.value[0] ?? '') + } + + const state = shallowRef({ status: 'idle' }) + const modelType = ref(defaultModelType()) + let controller: AbortController | undefined + let stopTaskWatch: WatchStopHandle | undefined + let generation = 0 + + function stop() { + generation++ + controller?.abort() + stopTaskWatch?.() + stopTaskWatch = undefined + } + + onScopeDispose(stop) + + function choose(file: File) { + stop() + if (!file.name.toLowerCase().endsWith(MODEL_FILE_EXTENSION)) { + state.value = { + status: 'idle', + error: t('assetBrowser.fileUpload.notSafetensors') + } + return + } + modelType.value = defaultModelType() + state.value = { status: 'chosen', file } + } + + function failureText(failure: UploadStopped) { + const copy = FAILURE_COPY[failure.reason] + if (copy) return t(copy) + return failure.message ?? t('assetBrowser.fileUpload.failed') + } + + function taskOutcome( + taskId: string, + file: File, + type: string + ): ModelFileUpload | undefined { + if (downloads.lastCompletedDownload?.taskId === taskId) { + return { status: 'done', file, modelType: type } + } + const download = downloads.downloadList.find((d) => d.taskId === taskId) + if (!download) return undefined + if (download.status === 'completed') { + return { status: 'done', file, modelType: type } + } + if (download.status === 'failed') { + return { + status: 'failed', + file, + error: download.error || t('assetBrowser.fileUpload.failed') + } + } + if (isDownloadCancelled(download.status)) { + return { + status: 'failed', + file, + error: t('assetBrowser.fileUpload.cancelled') + } + } + return undefined + } + + function followTask(taskId: string, file: File, type: string) { + const run = generation + downloads.trackDownload(taskId, type, file.name) + state.value = { status: 'saving', file } + stopTaskWatch = watch( + () => taskOutcome(taskId, file, type), + (outcome) => { + if (run === generation && outcome) state.value = outcome + }, + { immediate: true } + ) + } + + async function upload(): Promise { + const current = state.value + if (current.status !== 'chosen' && current.status !== 'failed') return null + + stop() + const run = generation + const { file } = current + const type = modelType.value + controller = new AbortController() + state.value = { status: 'uploading', file, sent: 0 } + + const result = await modelFileUploadApi.uploadModelFile(file, { + tags: ['models', type], + signal: controller.signal, + onProgress: (sent) => { + if (run === generation) + state.value = { status: 'uploading', file, sent } + } + }) + if (run !== generation) return null + + if (!result.ok) { + if (result.reason === 'failed') { + reportError(result.cause, { + surface: 'assets', + errorType: 'error_uploading_model_file', + level: 'warning' + }) + } + state.value = { status: 'failed', file, error: failureText(result) } + return null + } + + followTask(result.taskId, file, type) + return { + filename: file.name, + modelType: type, + taskId: result.taskId, + status: 'processing' + } + } + + function cancel() { + controller?.abort() + } + + function reset() { + stop() + state.value = { status: 'idle' } + modelType.value = defaultModelType() + } + + return { + isAvailable, + uploadFolders, + state, + modelType, + choose, + upload, + cancel, + reset + } +} diff --git a/src/stores/assetDownloadStore.ts b/src/stores/assetDownloadStore.ts index 7060cba5bea..716409b6a95 100644 --- a/src/stores/assetDownloadStore.ts +++ b/src/stores/assetDownloadStore.ts @@ -309,7 +309,19 @@ export const useAssetDownloadStore = defineStore('assetDownload', () => { } function trackDownload(taskId: TaskId, modelType: string, assetName: string) { - if (downloads.value.has(taskId)) return + const existing = downloads.value.get(taskId) + if (existing) { + if (existing.modelType) return + existing.modelType = modelType + if (existing.status === 'completed') { + lastCompletedDownload.value = { + taskId, + modelType, + timestamp: Date.now() + } + } + return + } dismissedPendingDownloads.delete(taskId) taskNotFoundAttempts.delete(taskId)