Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/locales/en/main.json
Original file line number Diff line number Diff line change
Expand Up @@ -5481,6 +5481,13 @@
"toastTitle": "Generation failed",
"toastMessage": "Not enough GPU memory. Try reducing image resolution or batch size and run again."
},
"server_out_of_memory": {
"title": "Ran out of memory",
"message": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.",
"itemLabel": "{nodeName}",
"toastTitle": "Ran out of memory",
"toastMessage": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again."
},
"content_blocked": {
"title": "Content blocked",
"message": "This request was blocked by the content moderation system. Try changing the prompt or inputs.",
Expand Down
1 change: 1 addition & 0 deletions src/platform/workflowErrors/errorCatalog/catalogIds.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export const UNKNOWN_VALIDATION_ERROR_CATALOG_ID = 'unknown_validation_error'
export const EXECUTION_FAILED_CATALOG_ID = 'execution_failed'
export const IMAGE_NOT_LOADED_CATALOG_ID = 'image_not_loaded'
export const OUT_OF_MEMORY_CATALOG_ID = 'out_of_memory'
export const SERVER_OUT_OF_MEMORY_CATALOG_ID = 'server_out_of_memory'
export const CONTENT_BLOCKED_CATALOG_ID = 'content_blocked'
export const ACCESS_REQUIRED_CATALOG_ID = 'access_required'
export const MODEL_ACCESS_ERROR_CATALOG_ID = 'model_access_error'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -975,6 +975,41 @@ describe('errorMessageResolver', () => {
})
})

it('tells a Cloud server memory kill apart from GPU out of memory', () => {
const serverOomMessage =
'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.'

expect(
resolveRunErrorMessage({
kind: 'execution',
nodeDisplayName: 'Motion Transfer',
error: executionError('OOMError', serverOomMessage)
})
).toMatchObject({
catalogId: 'server_out_of_memory',
displayTitle: 'Ran out of memory',
displayMessage:
'This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.',
displayItemLabel: 'Motion Transfer'
})

expect(
resolveRunErrorMessage({
kind: 'execution',
nodeDisplayName: 'Motion Transfer',
error: executionError('OOMError', ` ${serverOomMessage}\n`)
})
).toMatchObject({ catalogId: 'server_out_of_memory' })

expect(
resolveRunErrorMessage({
kind: 'execution',
nodeDisplayName: 'KSampler',
error: executionError('OOMError', 'OOMError: Workflow execution failed')
})
).toMatchObject({ catalogId: 'out_of_memory' })
})

it.for([
{
type: 'InsufficientFundsError',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
RUN_START_FAILED_CATALOG_ID,
SERVER_BUSY_CATALOG_ID,
SERVER_CRASHED_CATALOG_ID,
SERVER_OUT_OF_MEMORY_CATALOG_ID,
SIGN_IN_REQUIRED_CATALOG_ID,
SUBSCRIPTION_REQUIRED_CATALOG_ID,
SUBSCRIPTION_UPGRADE_REQUIRED_CATALOG_ID,
Expand Down Expand Up @@ -92,6 +93,8 @@ const CORE_OOM_TIP = 'This error means you ran out of memory on your GPU.'
const CORE_OOM_ALLOCATION_PREFIX = 'Allocation on device'
const CLOUD_OOM_PREFIX =
'Workflow execution failed due to insufficient memory (OOM).'
const SERVER_OOM_MESSAGE =
'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.'
const ERRNO_DIRECTORY_MESSAGE = '[Errno 21] Is a directory:'
const INVALID_CLIP_INPUT_PREFIX = 'ERROR: clip input is invalid: None'
const PROMPT_TOO_SHORT_MESSAGE =
Expand Down Expand Up @@ -227,6 +230,11 @@ const RUNTIME_MATCH_RULES: RuntimeMatchRule[] = [
resolve: (_info, message) =>
catalogMatchWithMessageFallback(IMAGE_NOT_LOADED_CATALOG_ID, message)
},
{
matches: (_info, message) => message.trim() === SERVER_OOM_MESSAGE,
resolve: (_info, message) =>
catalogMatchWithMessageFallback(SERVER_OUT_OF_MEMORY_CATALOG_ID, message)
},
{
matches: isOutOfMemoryError,
resolve: (_info, message) =>
Expand Down
Loading