From 47a8209635a8b2a15a7ccedba10e7e0268dde41d Mon Sep 17 00:00:00 2001 From: Dan Stotts Date: Thu, 8 Oct 2026 10:23:15 -0600 Subject: [PATCH] fix: cap MAX_NUM_SEQS at 32 so vLLM starts on a 48 GB GPU The Hub test of launch-20261008 crashed during engine startup: "max_num_seqs (256) exceeds available Mamba cache blocks (125)". Holo4-35B-A3B is a hybrid Mamba model, and on a 48 GB card the fp8 weights take 33.8 GiB, which leaves 2.53 GiB of cache. vLLM's default of 256 sequences doesn't fit there. 32 is just above MAX_CONCURRENCY (30), so the worker loses no throughput, and it is well under 125 on every GPU in the ADA_48_PRO pool. It is set in tests.json and as the preset default and an advanced env input in hub.json, because every user's deploy would hit the same crash. Co-Authored-By: Claude Opus 5.5 --- .runpod/hub.json | 14 +++++++++++++- .runpod/tests.json | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.runpod/hub.json b/.runpod/hub.json index 671e489..279140b 100644 --- a/.runpod/hub.json +++ b/.runpod/hub.json @@ -23,7 +23,8 @@ "TENSOR_PARALLEL_SIZE": "1", "GPU_MEMORY_UTILIZATION": "0.90", "MAX_CONCURRENCY": "30", - "QUANTIZATION": "fp8" + "QUANTIZATION": "fp8", + "MAX_NUM_SEQS": "32" } } ], @@ -61,6 +62,17 @@ "advanced": false } }, + { + "key": "MAX_NUM_SEQS", + "input": { + "name": "Max Num Seqs", + "type": "string", + "description": "Most sequences vLLM schedules at once; Holo4's Mamba cache holds 125 on a 48 GB GPU", + "default": "32", + "required": false, + "advanced": true + } + }, { "key": "MODEL_NAME", "input": { diff --git a/.runpod/tests.json b/.runpod/tests.json index 3ef0e83..9faeb72 100644 --- a/.runpod/tests.json +++ b/.runpod/tests.json @@ -33,6 +33,10 @@ "key": "MAX_MODEL_LEN", "value": "8192" }, + { + "key": "MAX_NUM_SEQS", + "value": "32" + }, { "key": "MODEL_NAME", "value": "Hcompany/Holo4-35B-A3B"