diff --git a/.runpod/hub.json b/.runpod/hub.json index 671e489..279140b 100644 --- a/.runpod/hub.json +++ b/.runpod/hub.json @@ -23,7 +23,8 @@ "TENSOR_PARALLEL_SIZE": "1", "GPU_MEMORY_UTILIZATION": "0.90", "MAX_CONCURRENCY": "30", - "QUANTIZATION": "fp8" + "QUANTIZATION": "fp8", + "MAX_NUM_SEQS": "32" } } ], @@ -61,6 +62,17 @@ "advanced": false } }, + { + "key": "MAX_NUM_SEQS", + "input": { + "name": "Max Num Seqs", + "type": "string", + "description": "Most sequences vLLM schedules at once; Holo4's Mamba cache holds 125 on a 48 GB GPU", + "default": "32", + "required": false, + "advanced": true + } + }, { "key": "MODEL_NAME", "input": { diff --git a/.runpod/tests.json b/.runpod/tests.json index 3ef0e83..9faeb72 100644 --- a/.runpod/tests.json +++ b/.runpod/tests.json @@ -33,6 +33,10 @@ "key": "MAX_MODEL_LEN", "value": "8192" }, + { + "key": "MAX_NUM_SEQS", + "value": "32" + }, { "key": "MODEL_NAME", "value": "Hcompany/Holo4-35B-A3B"