From 3a8c6820999acd93a0042b9afb9ffef85a857c10 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Thu, 5 Nov 2026 15:06:43 +0000 Subject: [PATCH] fix(errors): say the server ran out of memory when Cloud kills the ComfyUI container (#23006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary When Cloud runs out of memory on the server (not the GPU), the error now says so, instead of telling the user to free up GPU memory. Fixes [BE-24775](https://linear.app/comfyorg/issue/BE-24775) ## Changes - **What**: Since cloud v0.528.0, when the ComfyUI container is OOM killed, Cloud fails the job with `exception_type` `OOMError` and the fixed message "This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch." ([source](https://github.com/Comfy-Org/cloud/blob/main/services/inference/gateways/comfy/container_exit.go)). The `OOMError` rule sent that to `out_of_memory`, whose copy says "Not enough GPU memory…", which is the wrong advice for host memory. This adds a `server_out_of_memory` catalog entry, matched on the exact (trimmed) message before the generic OOM rule. - Only the `en` locale is edited; other locales fall back as usual. ## Review Focus - Rule order: the new rule sits right before `isOutOfMemoryError`, after `isImageNotLoadedError`. - How often: in production PostHog, `execution_error` with this message fired 67 / 98 / 104 / 88 times on 11-01 to 11-04, so users see the wrong copy about 100 times a day today. ## Testing - New test in `errorMessageResolver.test.ts`: the exact message (and with surrounding whitespace) resolves to `server_out_of_memory` with the new title and message; a plain `OOMError` still resolves to `out_of_memory`. It failed before the fix (`catalogId: 'out_of_memory'`) and passes after. - `vitest` on `src/platform/workflowErrors`: all `errorCatalog` tests pass. Two unrelated files (`missingMediaScan`, `missingModelScan`) hit a 10 s hook timeout on a busy host; CI is the gate for them. - `oxfmt --check`, `oxlint --type-aware` and `eslint` pass on the changed files. - Not checked in a running app: the copy is resolved by the same path as every other catalog entry, and the test asserts the resolved strings. --- src/locales/en/main.json | 7 ++++ .../workflowErrors/errorCatalog/catalogIds.ts | 1 + .../errorCatalog/errorMessageResolver.test.ts | 35 +++++++++++++++++++ .../errorCatalog/runtimeErrorMatcher.ts | 8 +++++ 4 files changed, 51 insertions(+) diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 64348039c54..ce0a8b261f2 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5481,6 +5481,13 @@ "toastTitle": "Generation failed", "toastMessage": "Not enough GPU memory. Try reducing image resolution or batch size and run again." }, + "server_out_of_memory": { + "title": "Ran out of memory", + "message": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.", + "itemLabel": "{nodeName}", + "toastTitle": "Ran out of memory", + "toastMessage": "This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again." + }, "content_blocked": { "title": "Content blocked", "message": "This request was blocked by the content moderation system. Try changing the prompt or inputs.", diff --git a/src/platform/workflowErrors/errorCatalog/catalogIds.ts b/src/platform/workflowErrors/errorCatalog/catalogIds.ts index c187365d544..ada3d5c1703 100644 --- a/src/platform/workflowErrors/errorCatalog/catalogIds.ts +++ b/src/platform/workflowErrors/errorCatalog/catalogIds.ts @@ -6,6 +6,7 @@ export const UNKNOWN_VALIDATION_ERROR_CATALOG_ID = 'unknown_validation_error' export const EXECUTION_FAILED_CATALOG_ID = 'execution_failed' export const IMAGE_NOT_LOADED_CATALOG_ID = 'image_not_loaded' export const OUT_OF_MEMORY_CATALOG_ID = 'out_of_memory' +export const SERVER_OUT_OF_MEMORY_CATALOG_ID = 'server_out_of_memory' export const CONTENT_BLOCKED_CATALOG_ID = 'content_blocked' export const ACCESS_REQUIRED_CATALOG_ID = 'access_required' export const MODEL_ACCESS_ERROR_CATALOG_ID = 'model_access_error' diff --git a/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts b/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts index 8dc2a217d72..6b10b5d6f32 100644 --- a/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts +++ b/src/platform/workflowErrors/errorCatalog/errorMessageResolver.test.ts @@ -975,6 +975,41 @@ describe('errorMessageResolver', () => { }) }) + it('tells a Cloud server memory kill apart from GPU out of memory', () => { + const serverOomMessage = + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.' + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'Motion Transfer', + error: executionError('OOMError', serverOomMessage) + }) + ).toMatchObject({ + catalogId: 'server_out_of_memory', + displayTitle: 'Ran out of memory', + displayMessage: + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or a smaller batch, then run it again.', + displayItemLabel: 'Motion Transfer' + }) + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'Motion Transfer', + error: executionError('OOMError', ` ${serverOomMessage}\n`) + }) + ).toMatchObject({ catalogId: 'server_out_of_memory' }) + + expect( + resolveRunErrorMessage({ + kind: 'execution', + nodeDisplayName: 'KSampler', + error: executionError('OOMError', 'OOMError: Workflow execution failed') + }) + ).toMatchObject({ catalogId: 'out_of_memory' }) + }) + it.for([ { type: 'InsufficientFundsError', diff --git a/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts b/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts index 8e79e58a076..72bfb890ef1 100644 --- a/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts +++ b/src/platform/workflowErrors/errorCatalog/runtimeErrorMatcher.ts @@ -18,6 +18,7 @@ import { RUN_START_FAILED_CATALOG_ID, SERVER_BUSY_CATALOG_ID, SERVER_CRASHED_CATALOG_ID, + SERVER_OUT_OF_MEMORY_CATALOG_ID, SIGN_IN_REQUIRED_CATALOG_ID, SUBSCRIPTION_REQUIRED_CATALOG_ID, SUBSCRIPTION_UPGRADE_REQUIRED_CATALOG_ID, @@ -92,6 +93,8 @@ const CORE_OOM_TIP = 'This error means you ran out of memory on your GPU.' const CORE_OOM_ALLOCATION_PREFIX = 'Allocation on device' const CLOUD_OOM_PREFIX = 'Workflow execution failed due to insufficient memory (OOM).' +const SERVER_OOM_MESSAGE = + 'This workflow ran out of memory on the server. Try a shorter video, lower resolution or smaller batch.' const ERRNO_DIRECTORY_MESSAGE = '[Errno 21] Is a directory:' const INVALID_CLIP_INPUT_PREFIX = 'ERROR: clip input is invalid: None' const PROMPT_TOO_SHORT_MESSAGE = @@ -227,6 +230,11 @@ const RUNTIME_MATCH_RULES: RuntimeMatchRule[] = [ resolve: (_info, message) => catalogMatchWithMessageFallback(IMAGE_NOT_LOADED_CATALOG_ID, message) }, + { + matches: (_info, message) => message.trim() === SERVER_OOM_MESSAGE, + resolve: (_info, message) => + catalogMatchWithMessageFallback(SERVER_OUT_OF_MEMORY_CATALOG_ID, message) + }, { matches: isOutOfMemoryError, resolve: (_info, message) =>