diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 64348039c54..ce0a8b261f2 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5481,6 +5481,13 @@ "toastTitle": "Generation failed", "toastMessage": "Not enough GPU memory. Try reducing image resolution or batch size and run again." }, + "server_out_of_memory": { + "title": "Ran out of memory", + "message": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.", + "itemLabel": "{nodeName}", + "toastTitle": "Ran out of memory", + "toastMessage": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again." + }, "content_blocked": { "title": "Content blocked", "message": "This request was blocked by the content moderation system. Try changing the prompt or inputs.", diff --git a/src/platform/workflowErrors/errorCatalog/catalogIds.ts b/src/platform/workflowErrors/errorCatalog/catalogIds.ts index c187365d544..ada3d5c1703 100644 --- a/src/platform/workflowErrors/errorCatalog/catalogIds.ts +++ b/src/platform/workflowErrors/errorCatalog/catalogIds.ts @@ -6,6 +6,7 @@ export const UNKNOWN_VALIDATION_ERROR_CATALOG_ID = 'unknown_validation_error' export const EXECUTION_FAILED_CATALOG_ID = 'execution_failed' export const IMAGE_NOT_LOADED_CATALOG_ID = 'image_not_loaded' export const OUT_OF_MEMORY_CATALOG_ID = 'out_of_memory' +export const SERVER_OUT_OF_MEMORY_CATALOG_ID = 'server_out_of_memory' export const CONTENT_BLOCKED_CATALOG_ID = 'content_blocked' export const ACCESS_REQUIRED_CATALOG_ID = 'access_required' export const MODEL_ACCESS_ERROR_CATALOG_ID = 'model_access_error' diff --git a/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts b/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts index 8dc2a217d72..6b10b5d6f32 100644 --- a/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts +++ b/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts @@ -975,6 +975,41 @@ describe('errorMessageResolver', () => { }) }) + it('tells a Cloud server memory kill apart from GPU out of memory', () => { + const serverOomMessage = + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.' + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'Motion Transfer', + error: executionError('OOMError', serverOomMessage) + }) + ).toMatchObject({ + catalogId: 'server_out_of_memory', + displayTitle: 'Ran out of memory', + displayMessage: + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.', + displayItemLabel: 'Motion Transfer' + }) + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'Motion Transfer', + error: executionError('OOMError', ` ${serverOomMessage}\n`) + }) + ).toMatchObject({ catalogId: 'server_out_of_memory' }) + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'KSampler', + error: executionError('OOMError', 'OOMError: Workflow execution failed') + }) + ).toMatchObject({ catalogId: 'out_of_memory' }) + }) + it.for([ { type: 'InsufficientFundsError', diff --git a/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts b/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts index 8e79e58a076..72bfb890ef1 100644 --- a/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts +++ b/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts @@ -18,6 +18,7 @@ import { RUN_START_FAILED_CATALOG_ID, SERVER_BUSY_CATALOG_ID, SERVER_CRASHED_CATALOG_ID, + SERVER_OUT_OF_MEMORY_CATALOG_ID, SIGN_IN_REQUIRED_CATALOG_ID, SUBSCRIPTION_REQUIRED_CATALOG_ID, SUBSCRIPTION_UPGRADE_REQUIRED_CATALOG_ID, @@ -92,6 +93,8 @@ const CORE_OOM_TIP = 'This error means you ran out of memory on your GPU.' const CORE_OOM_ALLOCATION_PREFIX = 'Allocation on device' const CLOUD_OOM_PREFIX = 'Workflow execution failed due to insufficient memory (OOM).' +const SERVER_OOM_MESSAGE = + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.' const ERRNO_DIRECTORY_MESSAGE = '[Errno 21] Is a directory:' const INVALID_CLIP_INPUT_PREFIX = 'ERROR: clip input is invalid: None' const PROMPT_TOO_SHORT_MESSAGE = @@ -227,6 +230,11 @@ const RUNTIME_MATCH_RULES: RuntimeMatchRule[] = [ resolve: (_info, message) => catalogMatchWithMessageFallback(IMAGE_NOT_LOADED_CATALOG_ID, message) }, + { + matches: (_info, message) => message.trim() === SERVER_OOM_MESSAGE, + resolve: (_info, message) => + catalogMatchWithMessageFallback(SERVER_OUT_OF_MEMORY_CATALOG_ID, message) + }, { matches: isOutOfMemoryError, resolve: (_info, message) =>