From 0f1385c15f945d16d5242c68db8e7cc97cdbf97b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 21:40:19 +0000 Subject: [PATCH 1/4] fix: answer jobs when a model's own vLLM code rejects the setup A model's vLLM implementation can refuse a configuration outright, and it fails the same way on every start. RecursiveSI/flint on an H200 (vLLM 0.32.0rc1 day-0 image) did it twice: "AssertionError: Flint requires FlashInfer attention" with the default backend, then "RuntimeError: Flint BF16 decode MoE requires the FlashInfer TRTLLM BF16 MoE backend" with ATTENTION_BACKEND=FLASHINFER (a Blackwell-only kernel). Both exited and restarted into the same failure, so jobs got no answer. classify() now treats an AssertionError, RuntimeError, NotImplementedError, ValueError or TypeError whose innermost frame is in model code (vllm/model_executor/models/ or vllm/models/) as a startup_error, quoting the root cause rather than the API server's wrapper. CUDA/NCCL errors and anything raised outside model code still exit for a platform restart. Checked last, so every specific match keeps priority. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- README.md | 2 +- src/startup_errors.py | 45 ++++++++++++++++++ tests/test_startup_errors.py | 91 ++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a3db1ddf..ebea10da 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ The worker recognises these and stays up instead, answering every job with the c } ``` -The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. +The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. The one broad rule: when the traceback ends inside a model's own vLLM implementation (`vllm/model_executor/models/` or `vllm/models/`), for example a model that requires the FlashInfer attention backend or a newer GPU generation, the failure is treated as permanent too, unless it is a CUDA or NCCL error. ## Option 2: Build Docker Image with Model Inside diff --git a/src/startup_errors.py b/src/startup_errors.py index 669abac9..478301e9 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -57,6 +57,41 @@ # --- Disk ------------------------------------------------------------------- _NO_SPACE = re.compile(r"No space left on device|ENOSPC|errno 28|Disk quota exceeded|EDQUOT|errno 122", re.I) +# vLLM's log prefix on EngineCore lines, e.g. +# "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] ". +_LOG_PREFIX = re.compile(r"^\([^)]*\) [A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?") +_FRAME = re.compile(r'^\s+File "([^"]+)", line \d+') +_EXCEPTION = re.compile(r"^(AssertionError|RuntimeError|NotImplementedError|ValueError|TypeError)(?:: (.*))?$") +# Model implementations: vllm/model_executor/models/.py, and the +# per-model packages newer day-0 images ship under vllm/models//. +_MODEL_CODE = re.compile(r"/vllm/(?:model_executor/)?models/") +# Raised from model code but about the host, not the model: a different host +# can succeed, so these stay with the platform restart. +_HOST_FAULT = re.compile(r"CUDA error|cudaError|NCCL|CUBLAS_STATUS|illegal memory access|uncorrectable ECC", re.I) + + +def model_code_error(output: str) -> Optional[str]: + """The exception line of the first traceback whose innermost frame is model code. + + A model's own vLLM implementation can refuse a setup outright: flint's + asserts the FlashInfer attention backend and its BF16 MoE needs a + Blackwell-only kernel. Those fail identically on every start. + """ + last_file = None + for raw in output.splitlines(): + line = _LOG_PREFIX.sub("", raw.rstrip()) + frame = _FRAME.match(line) + if frame: + last_file = frame.group(1) + continue + exc = _EXCEPTION.match(line) + if exc and last_file: + if _MODEL_CODE.search(last_file) and not _HOST_FAULT.search(line): + return line[:300] + last_file = None + return None + + # The model-access messages are shared with model_preflight.py, which detects # the same failures before the cold start; whether the user hits the fast path @@ -235,4 +270,14 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False f"weights are cached there instead." ) + raised = model_code_error(output) + if raised: + return ( + f"{named} failed inside vLLM's implementation of the model during " + f"startup: {raised.rstrip('.')}. A restart will not change this. A model's vLLM " + f"code can require a specific attention backend (set ATTENTION_BACKEND), " + f"GPU generation or dtype: check the model card and vLLM's recipe for " + f"the model, then change the endpoint's settings or GPU type." + ) + return None diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 35ba86fd..6d786e60 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -230,3 +230,94 @@ def test_non_memory_failures_do_not_match(self): assert not memory_shortfall(REVISION_NOT_FOUND) assert not memory_shortfall(GATED) assert not memory_shortfall("Connection reset by peer while downloading") + + +# Tails of real worker logs (RecursiveSI/flint on an H200, vLLM 0.32.0rc1 day-0 +# image), trimmed to the frames that matter. EngineCore prints the root cause; +# the API server's own traceback follows it and ends in a generic RuntimeError. +_EC = "(EngineCore pid=1079) ERROR 10-21 20:16:36 [core.py:1543] " +API_SERVER_TAIL = """Traceback (most recent call last): + File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", line 1102, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +""" +FLINT_NEEDS_FLASHINFER = "".join(_EC + line + "\n" for line in [ + "EngineCore failed to start.", + "Traceback (most recent call last):", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 1515, in run_engine_core', + " engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/model.py", line 479, in __init__', + " self.self_attn = FlintAttention(", + " ^^^^^^^^^^^^^^^", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/model.py", line 316, in __init__', + ' assert isinstance(impl, FlashInferImpl), "Flint requires FlashInfer attention"', + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", + "AssertionError: Flint requires FlashInfer attention", +]) + API_SERVER_TAIL +FLINT_MOE_NEEDS_BLACKWELL = "".join(_EC + line + "\n" for line in [ + "EngineCore failed to start.", + "Traceback (most recent call last):", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/warmup/kernel_warmup.py", line 294, in kernel_warmup', + " mixed_batch_attention_warmup(worker)", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/ops/moe.py", line 806, in forward', + " self._check(routed_experts)", + ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/ops/moe.py", line 792, in _check', + " raise RuntimeError(", + "RuntimeError: Flint BF16 decode MoE requires the FlashInfer TRTLLM BF16 MoE backend " + "without padding (got UnquantizedMoeBackend.TRITON)", +]) + API_SERVER_TAIL + + +def _traceback(innermost: str, exception: str) -> str: + return ( + "Traceback (most recent call last):\n" + ' File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 157, in __init__\n' + " kv_cache_config = self._initialize_kv_caches(vllm_config)\n" + f' File "/usr/local/lib/python3.12/dist-packages/{innermost}", line 42, in forward\n' + " raise\n" + f"{exception}\n" + ) + + +class TestModelCodeErrors: + @pytest.mark.parametrize( + "output,cause", + [ + (FLINT_NEEDS_FLASHINFER, "AssertionError: Flint requires FlashInfer attention"), + (FLINT_MOE_NEEDS_BLACKWELL, "requires the FlashInfer TRTLLM BF16 MoE backend"), + ], + ) + def test_quote_the_root_cause_not_the_api_server_wrapper(self, output, cause): + message = classify(output, model="RecursiveSI/flint") + + assert message.startswith("RecursiveSI/flint failed inside vLLM's implementation") + assert cause in message + assert "Engine core initialization failed" not in message + assert "ATTENTION_BACKEND" in message + + def test_classic_model_executor_layout(self): + output = _traceback("vllm/model_executor/models/foo.py", "NotImplementedError: FooAttention needs sm90") + + assert "NotImplementedError: FooAttention needs sm90" in classify(output) + + @pytest.mark.parametrize( + "innermost,exception", + [ + # Same exception types outside model code: could be anything. + ("vllm/v1/engine/core.py", "RuntimeError: something nobody has seen before"), + ("safetensors/torch.py", "ValueError: truncated file"), + # Raised in model code but about the host: another host can work. + ("vllm/models/flint/ops/moe.py", "RuntimeError: CUDA error: an illegal memory access was encountered"), + ("vllm/model_executor/models/llama.py", "RuntimeError: NCCL error: unhandled system error"), + ], + ) + def test_everything_else_still_restarts(self, innermost, exception): + assert classify(_traceback(innermost, exception)) is None + + def test_oom_raised_in_model_code_keeps_the_memory_advice(self): + output = _traceback( + "vllm/model_executor/models/llama.py", + "torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.00 GiB.", + ) + + assert "ran out of GPU memory" in classify(output) From cec0dc249472c2b5ac0746a1cb58b02027d23d49 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 23:07:25 +0000 Subject: [PATCH 2/4] model-code errors: match stated requirements only, never with a host fault Review found the location rule too broad: a CUDA driver fault, an NVML internal assert or a dropped connection inside load_weights also ends in model code, and would have been answered forever. Now a model-code exception counts only when its message (all lines) states a requirement ("requires", "only supports", "not supported on ..."), and any CUDA, NCCL, NVML, cuBLAS/cuDNN, ECC, Xid or connection/timeout error anywhere in the failed launch's output sends it back to the platform restart, including a CUDA cause wrapped by a model-code ValueError. Both flint logs, the H100 log and the K2 logs classify as before. 161 tests (17 new). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- README.md | 2 +- src/startup_errors.py | 62 ++++++++++++++++++++++++++---------- tests/test_startup_errors.py | 59 ++++++++++++++++++++++++++-------- 3 files changed, 91 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ebea10da..480f1c29 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ The worker recognises these and stays up instead, answering every job with the c } ``` -The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. The one broad rule: when the traceback ends inside a model's own vLLM implementation (`vllm/model_executor/models/` or `vllm/models/`), for example a model that requires the FlashInfer attention backend or a newer GPU generation, the failure is treated as permanent too, unless it is a CUDA or NCCL error. +The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. One more case is treated as permanent: an error raised inside a model's own vLLM implementation (`vllm/model_executor/models/` or `vllm/models/`) that states a requirement the setup doesn't meet, for example a model that requires the FlashInfer attention backend or a newer GPU generation. Any other error from model code, and any failed start with a CUDA, NCCL or NVML error in its output, still restarts. ## Option 2: Build Docker Image with Model Inside diff --git a/src/startup_errors.py b/src/startup_errors.py index 478301e9..e78cd10a 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -61,34 +61,61 @@ # "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] ". _LOG_PREFIX = re.compile(r"^\([^)]*\) [A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?") _FRAME = re.compile(r'^\s+File "([^"]+)", line \d+') -_EXCEPTION = re.compile(r"^(AssertionError|RuntimeError|NotImplementedError|ValueError|TypeError)(?:: (.*))?$") +_EXCEPTION = re.compile(r"^(?:AssertionError|RuntimeError|NotImplementedError|ValueError)(?:: .*)?$") +_TRACEBACK_BREAK = re.compile(r"^(?:Traceback \(most recent call last\):|The above exception|During handling of the above)") # Model implementations: vllm/model_executor/models/.py, and the # per-model packages newer day-0 images ship under vllm/models//. _MODEL_CODE = re.compile(r"/vllm/(?:model_executor/)?models/") -# Raised from model code but about the host, not the model: a different host -# can succeed, so these stay with the platform restart. -_HOST_FAULT = re.compile(r"CUDA error|cudaError|NCCL|CUBLAS_STATUS|illegal memory access|uncorrectable ECC", re.I) +# The model's code states what it needs: "Flint requires FlashInfer attention", +# "... requires the FlashInfer TRTLLM BF16 MoE backend", "only supported on ...". +# Where an error is raised says nothing about permanence (a driver fault inside +# load_weights also ends in model code); a stated requirement does. +_REQUIREMENT = re.compile( + r"\brequires?\b|\bonly supports?\b|\bonly supported\b|\bnot supported (?:on|with|for|by)\b", re.I +) +# Host or transport trouble anywhere in the failed launch, including a CUDA error +# wrapped by a model-code exception: another host or attempt can succeed. +_HOST_FAULT = re.compile( + r"CUDA (?:driver )?error|cudaError|NCCL error|nccl(?:System|Unhandled|Internal)\w*Error|NVML|CUBLAS_STATUS|" + r"CUDNN_STATUS|illegal memory access|uncorrectable ECC|\bXid\b|Connection (?:reset|refused|aborted)|" + r"timed out|Max retries exceeded", + re.I, +) def model_code_error(output: str) -> Optional[str]: - """The exception line of the first traceback whose innermost frame is model code. + """The stated requirement a model's own vLLM code failed on, if that is all that went wrong. - A model's own vLLM implementation can refuse a setup outright: flint's - asserts the FlashInfer attention backend and its BF16 MoE needs a - Blackwell-only kernel. Those fail identically on every start. + Matches an exception whose innermost frame is model code and whose message + states a requirement the setup does not meet: flint asserts the FlashInfer + attention backend, and its BF16 MoE needs a Blackwell-only kernel. Those + fail identically on every start. Returns None when the failed launch shows + any host or transport fault, so those keep getting a restart. """ - last_file = None + if _HOST_FAULT.search(output): + return None + last_file, message = None, None for raw in output.splitlines(): line = _LOG_PREFIX.sub("", raw.rstrip()) + if message is not None: + # Continuation lines of a multi-line message, up to the next frame or traceback. + if line and not line.startswith(" ") and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line): + message.append(line) + continue + text = " ".join(message) + if _REQUIREMENT.search(text): + return text[:300] + message = None frame = _FRAME.match(line) if frame: last_file = frame.group(1) continue - exc = _EXCEPTION.match(line) - if exc and last_file: - if _MODEL_CODE.search(last_file) and not _HOST_FAULT.search(line): - return line[:300] + if _EXCEPTION.match(line) and last_file: + if _MODEL_CODE.search(last_file): + message = [line] last_file = None + if message and _REQUIREMENT.search(" ".join(message)): + return " ".join(message)[:300] return None @@ -274,10 +301,11 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False if raised: return ( f"{named} failed inside vLLM's implementation of the model during " - f"startup: {raised.rstrip('.')}. A restart will not change this. A model's vLLM " - f"code can require a specific attention backend (set ATTENTION_BACKEND), " - f"GPU generation or dtype: check the model card and vLLM's recipe for " - f"the model, then change the endpoint's settings or GPU type." + f"startup: {raised.rstrip('.')}. The model's code requires a setup this " + f"endpoint does not have, so a restart will not change it. Check the model " + f"card and vLLM's recipe for the model for the attention backend " + f"(ATTENTION_BACKEND), GPU generation or dtype it needs, then change the " + f"endpoint's settings or GPU type." ) return None diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 6d786e60..63cff220 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -268,12 +268,12 @@ def test_non_memory_failures_do_not_match(self): ]) + API_SERVER_TAIL -def _traceback(innermost: str, exception: str) -> str: +def _traceback(innermost: str, exception: str, function: str = "__init__") -> str: return ( "Traceback (most recent call last):\n" ' File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 157, in __init__\n' " kv_cache_config = self._initialize_kv_caches(vllm_config)\n" - f' File "/usr/local/lib/python3.12/dist-packages/{innermost}", line 42, in forward\n' + f' File "/usr/local/lib/python3.12/dist-packages/{innermost}", line 42, in {function}\n' " raise\n" f"{exception}\n" ) @@ -295,24 +295,55 @@ def test_quote_the_root_cause_not_the_api_server_wrapper(self, output, cause): assert "Engine core initialization failed" not in message assert "ATTENTION_BACKEND" in message - def test_classic_model_executor_layout(self): - output = _traceback("vllm/model_executor/models/foo.py", "NotImplementedError: FooAttention needs sm90") + @pytest.mark.parametrize( + "exception", + [ + "NotImplementedError: FooAttention only supports compute capability 9.0 and above", + "ValueError: Sliding window attention is not supported on this platform", + "RuntimeError: FooMoE requires the CUTLASS backend", + ], + ) + def test_stated_requirements_in_classic_model_code(self, exception): + assert exception in classify(_traceback("vllm/model_executor/models/foo.py", exception)) + + def test_multiline_requirement_is_quoted_whole(self): + output = _traceback("vllm/models/foo/model.py", "RuntimeError: Foo decode kernel\nrequires sm100 (got sm90)") - assert "NotImplementedError: FooAttention needs sm90" in classify(output) + assert "RuntimeError: Foo decode kernel requires sm100 (got sm90)" in classify(output) @pytest.mark.parametrize( - "innermost,exception", + "innermost,function,exception", [ - # Same exception types outside model code: could be anything. - ("vllm/v1/engine/core.py", "RuntimeError: something nobody has seen before"), - ("safetensors/torch.py", "ValueError: truncated file"), - # Raised in model code but about the host: another host can work. - ("vllm/models/flint/ops/moe.py", "RuntimeError: CUDA error: an illegal memory access was encountered"), - ("vllm/model_executor/models/llama.py", "RuntimeError: NCCL error: unhandled system error"), + # Where an error is raised says nothing about permanence: model code can be + # the innermost Python frame of a native failure during weight loading. + ("vllm/model_executor/models/llama.py", "load_weights", "RuntimeError: CUDA driver error: initialization error"), + ("vllm/model_executor/models/llama.py", "load_weights", + "RuntimeError: NVML_SUCCESS == r INTERNAL ASSERT FAILED at \"/pytorch/c10/cuda/CUDACachingAllocator.cpp\":1131"), + ("vllm/model_executor/models/llama.py", "load_weights", "RuntimeError: Failed to load weights: Connection reset by peer"), + ("vllm/models/flint/ops/moe.py", "forward", "RuntimeError: CUDA error: an illegal memory access was encountered"), + # No stated requirement: could be anything. + ("vllm/model_executor/models/foo.py", "__init__", "AssertionError"), + ("vllm/model_executor/models/foo.py", "forward", "ValueError: Error during index put operation"), + # A requirement, but not raised by model code. + ("vllm/v1/engine/core.py", "__init__", "RuntimeError: this step requires a restart"), + ("safetensors/torch.py", "load_file", "ValueError: truncated file"), ], ) - def test_everything_else_still_restarts(self, innermost, exception): - assert classify(_traceback(innermost, exception)) is None + def test_everything_else_still_restarts(self, innermost, function, exception): + assert classify(_traceback(innermost, exception, function)) is None + + def test_a_wrapped_cuda_cause_still_restarts(self): + # vLLM's _merge_multimodal_embeddings re-raises a failed index_put as ValueError. + cause = _traceback("vllm/model_executor/models/utils.py", "RuntimeError: CUDA error: too many resources requested for launch") + wrapper = _traceback("vllm/model_executor/models/utils.py", "ValueError: Error during index put operation requires retry") + output = cause + "\nThe above exception was the direct cause of the following exception:\n\n" + wrapper + + assert classify(output) is None + + def test_a_cuda_error_on_a_continuation_line_still_restarts(self): + output = _traceback("vllm/models/foo/model.py", "RuntimeError: Foo requires a warmup pass\nCUDA error: unspecified launch failure") + + assert classify(output) is None def test_oom_raised_in_model_code_keeps_the_memory_advice(self): output = _traceback( From 2df7fede97e87763bf9cd43352f0a66c64ad408b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 23:30:22 +0000 Subject: [PATCH 3/4] model-code errors: quote only the error's own record; tighter host-fault match Second review found the multi-line joining appended the next log record ("Process EngineCore:") to the quoted cause on every real flint log, and could borrow a requirement word from an unrelated INFO line. Continuation lines now count only within the same log record (same prefix). Also from the review: - host faults are matched by fault signatures (CUDA/NCCL/NVML errors, cuBLAS/cuDNN status, ECC, Xid, driver too old, collective timeout), not any NVML/timeout/connection wording, so torch's NVML warnings or a retried download elsewhere in the launch don't send flint back to restarting; transport wording only vetoes when it is in the quoted error itself - "requires grad" autograd errors don't count as requirements - tracebacks logged below ERROR, and engines killed by a signal, don't count - a model-code frame no longer attaches to a later frameless error - vLLM 0.30's bare "(APIServer pid=N) " raw-output prefix is stripped - the message names this worker's GPU or settings and says what to do on endpoints with several GPU types; README lists the case with the others Fixtures now follow the real log order (prefixed traceback, "Process EngineCore:", raw traceback, API server tail) and assert the exact quoted cause. All 5 launches of both full H200 logs, the H100 log, the 48 GB OOM log and both K2 logs classify as intended. 173 tests (29 new). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- README.md | 4 +- src/startup_errors.py | 107 ++++++++++++++++++++----------- tests/test_startup_errors.py | 119 ++++++++++++++++++++++++++--------- 3 files changed, 165 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 480f1c29..790c75aa 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ For the complete list of all available environment variables, examples, and deta ### When vLLM will not start -Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result. +Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`, a model whose own vLLM code states a requirement this worker's GPU or settings don't meet (for example the FlashInfer attention backend, or a newer GPU generation). Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result. The worker recognises these and stays up instead, answering every job with the cause and the fix: @@ -119,7 +119,7 @@ The worker recognises these and stays up instead, answering every job with the c } ``` -The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. One more case is treated as permanent: an error raised inside a model's own vLLM implementation (`vllm/model_executor/models/` or `vllm/models/`) that states a requirement the setup doesn't meet, for example a model that requires the FlashInfer attention backend or a newer GPU generation. Any other error from model code, and any failed start with a CUDA, NCCL or NVML error in its output, still restarts. +The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. For model code (`vllm/model_executor/models/`, `vllm/models/`), only an error whose message states a requirement ("requires", "only supports", "not supported on") counts; any other error there, and any failed start that shows a CUDA, NCCL, NVML, cuBLAS/cuDNN, ECC or Xid fault, still restarts. A requirement can depend on the GPU type, so on an endpoint that lists several, a worker on a GPU the model can't use answers jobs with the cause instead of leaving them to a worker that can: remove those GPU types from the endpoint. ## Option 2: Build Docker Image with Model Inside diff --git a/src/startup_errors.py b/src/startup_errors.py index e78cd10a..488c9540 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -57,31 +57,56 @@ # --- Disk ------------------------------------------------------------------- _NO_SPACE = re.compile(r"No space left on device|ENOSPC|errno 28|Disk quota exceeded|EDQUOT|errno 122", re.I) -# vLLM's log prefix on EngineCore lines, e.g. -# "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] ". -_LOG_PREFIX = re.compile(r"^\([^)]*\) [A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?") +# vLLM's log prefixes: "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] " +# on logged lines, and a bare "(APIServer pid=496) " on raw output (vLLM 0.30). +_LOG_PREFIX = re.compile(r"^\([^)]*\) (?:[A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?)?") _FRAME = re.compile(r'^\s+File "([^"]+)", line \d+') _EXCEPTION = re.compile(r"^(?:AssertionError|RuntimeError|NotImplementedError|ValueError)(?:: .*)?$") -_TRACEBACK_BREAK = re.compile(r"^(?:Traceback \(most recent call last\):|The above exception|During handling of the above)") -# Model implementations: vllm/model_executor/models/.py, and the -# per-model packages newer day-0 images ship under vllm/models//. +_TRACEBACK_BREAK = re.compile( + r"^(?:Traceback \(most recent call last\):|The above exception|During handling of the above)" +) +# Model implementations: vllm/model_executor/models/.py, and the per-model +# packages under vllm/models// (since v0.22; v0.32.0 ships deepseek_v4, +# kimi_k3, minimax_m3 and others there). _MODEL_CODE = re.compile(r"/vllm/(?:model_executor/)?models/") # The model's code states what it needs: "Flint requires FlashInfer attention", # "... requires the FlashInfer TRTLLM BF16 MoE backend", "only supported on ...". # Where an error is raised says nothing about permanence (a driver fault inside -# load_weights also ends in model code); a stated requirement does. +# load_weights also ends in model code); a stated requirement does. Autograd's +# "requires grad" errors are not configuration requirements. _REQUIREMENT = re.compile( - r"\brequires?\b|\bonly supports?\b|\bonly supported\b|\bnot supported (?:on|with|for|by)\b", re.I + r"\brequires?\b(?![ _]grad)|\bonly supports?\b|\bonly supported\b|\bnot supported (?:on|with|for|by)\b", + re.I, ) -# Host or transport trouble anywhere in the failed launch, including a CUDA error -# wrapped by a model-code exception: another host or attempt can succeed. +# Host faults anywhere in the failed launch, including a CUDA error wrapped by a +# model-code exception: another host can succeed. Fault signatures only, so +# torch's NVML/pynvml warnings on a healthy host don't count. _HOST_FAULT = re.compile( - r"CUDA (?:driver )?error|cudaError|NCCL error|nccl(?:System|Unhandled|Internal)\w*Error|NVML|CUBLAS_STATUS|" - r"CUDNN_STATUS|illegal memory access|uncorrectable ECC|\bXid\b|Connection (?:reset|refused|aborted)|" - r"timed out|Max retries exceeded", + r"CUDA (?:driver |unknown )?error|cudaError|CUDA-capable device|NVIDIA driver on your system is too old|" + r"NCCL error|nccl(?:System|Unhandled|Internal)\w*Error|Watchdog caught collective operation timeout|" + r"NVML_SUCCESS == r|NVMLError|nvml\w*Error|CUBLAS_STATUS|CUDNN_STATUS|illegal memory access|" + r"uncorrectable ECC|\bXid\b", + re.I, +) +# Transport trouble inside the quoted error itself (a retried download that +# succeeded elsewhere in the log must not count, so this only checks the message). +_TRANSPORT_FAULT = re.compile( + r"Connection (?:reset|refused|aborted)|timed out|Max retries exceeded|Temporary failure in name resolution", re.I, ) +# An engine process killed by a signal ("Failed core proc(s): {'EngineCore': -9}") +# died of something else, whatever model code logged earlier. +_ENGINE_KILLED = re.compile(r"Failed core proc\(s\): \{[^}]*: -\d+") +# A traceback logged below ERROR was caught and handled; it didn't end the launch. +_NOT_AN_ERROR_RECORD = re.compile(r"\) (?:DEBUG|INFO|WARNING) ") + + +def _split_prefix(raw: str): + line = raw.rstrip() + match = _LOG_PREFIX.match(line) + return (match.group(0), line[match.end():]) if match else ("", line) + def model_code_error(output: str) -> Optional[str]: """The stated requirement a model's own vLLM code failed on, if that is all that went wrong. @@ -90,34 +115,46 @@ def model_code_error(output: str) -> Optional[str]: states a requirement the setup does not meet: flint asserts the FlashInfer attention backend, and its BF16 MoE needs a Blackwell-only kernel. Those fail identically on every start. Returns None when the failed launch shows - any host or transport fault, so those keep getting a restart. + a host fault, or the message itself a transport fault, so those keep + getting a restart. The last match wins: it is the one that ended the launch. """ - if _HOST_FAULT.search(output): + if _HOST_FAULT.search(output) or _ENGINE_KILLED.search(output): + return None + found = None + last_file = None + message, message_prefix = None, None + + def finish(lines): + text = " ".join(lines) + if _REQUIREMENT.search(text) and not _TRANSPORT_FAULT.search(text): + return text[:300] return None - last_file, message = None, None + for raw in output.splitlines(): - line = _LOG_PREFIX.sub("", raw.rstrip()) + prefix, line = _split_prefix(raw) if message is not None: - # Continuation lines of a multi-line message, up to the next frame or traceback. - if line and not line.startswith(" ") and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line): + # A multi-line message continues only within the same log record (same + # prefix): the next record, e.g. multiprocessing's "Process EngineCore:", + # is not part of it. + if (prefix == message_prefix and line and not line.startswith(" ") + and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line)): message.append(line) continue - text = " ".join(message) - if _REQUIREMENT.search(text): - return text[:300] + found = finish(message) or found message = None frame = _FRAME.match(line) if frame: last_file = frame.group(1) continue - if _EXCEPTION.match(line) and last_file: - if _MODEL_CODE.search(last_file): - message = [line] - last_file = None - if message and _REQUIREMENT.search(" ".join(message)): - return " ".join(message)[:300] - return None - + if line.startswith(" "): + continue # source and caret lines under a frame + if (_EXCEPTION.match(line) and last_file and _MODEL_CODE.search(last_file) + and not _NOT_AN_ERROR_RECORD.search(prefix)): + message, message_prefix = [line], prefix + last_file = None # any other unindented line ends the frame list + if message is not None: + found = finish(message) or found + return found # The model-access messages are shared with model_preflight.py, which detects @@ -301,11 +338,11 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False if raised: return ( f"{named} failed inside vLLM's implementation of the model during " - f"startup: {raised.rstrip('.')}. The model's code requires a setup this " - f"endpoint does not have, so a restart will not change it. Check the model " - f"card and vLLM's recipe for the model for the attention backend " - f"(ATTENTION_BACKEND), GPU generation or dtype it needs, then change the " - f"endpoint's settings or GPU type." + f"startup: {raised.rstrip('.')}. The model's code requires something this " + f"worker's GPU or settings don't provide, so restarting this worker will not " + f"change it. If the endpoint lists several GPU types, remove the ones the " + f"model can't use; otherwise check the model card and vLLM's recipe for the " + f"attention backend (ATTENTION_BACKEND), GPU generation or dtype it needs." ) return None diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 63cff220..ff435194 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -233,17 +233,32 @@ def test_non_memory_failures_do_not_match(self): # Tails of real worker logs (RecursiveSI/flint on an H200, vLLM 0.32.0rc1 day-0 -# image), trimmed to the frames that matter. EngineCore prints the root cause; -# the API server's own traceback follows it and ends in a generic RuntimeError. +# image) in the order vLLM prints them, trimmed to the frames that matter: +# EngineCore logs the root cause with a prefix, multiprocessing prints an +# unprefixed "Process EngineCore:" and the raw traceback, and the API server's +# own traceback ends in a generic RuntimeError. _EC = "(EngineCore pid=1079) ERROR 10-21 20:16:36 [core.py:1543] " -API_SERVER_TAIL = """Traceback (most recent call last): - File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", line 1102, in wait_for_engine_startup +API_SERVER_TAIL = """(APIServer pid=503) INFO 10-21 20:16:36 [utils.py:628] [shutdown] Process manager: send sigterm to process EngineCore +Traceback (most recent call last): + File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", line 1468, in wait_for_engine_startup raise RuntimeError( RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} """ -FLINT_NEEDS_FLASHINFER = "".join(_EC + line + "\n" for line in [ - "EngineCore failed to start.", - "Traceback (most recent call last):", + + +def _flint_log(frames, exception): + traceback = ["Traceback (most recent call last):", *frames, exception] + logged = "".join(_EC + line + "\n" for line in ["EngineCore failed to start.", *traceback]) + raw = "Process EngineCore:\n" + "".join(line + "\n" for line in traceback) + return logged + raw + API_SERVER_TAIL + + +FLINT_ATTN = "AssertionError: Flint requires FlashInfer attention" +FLINT_MOE = ( + "RuntimeError: Flint BF16 decode MoE requires the FlashInfer TRTLLM BF16 MoE backend " + "without padding (got UnquantizedMoeBackend.TRITON)" +) +FLINT_NEEDS_FLASHINFER = _flint_log([ ' File "/usr/local/lib/python3.12/dist-packages/vllm/v1/engine/core.py", line 1515, in run_engine_core', " engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)", ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/model.py", line 479, in __init__', @@ -252,20 +267,15 @@ def test_non_memory_failures_do_not_match(self): ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/model.py", line 316, in __init__', ' assert isinstance(impl, FlashInferImpl), "Flint requires FlashInfer attention"', " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^", - "AssertionError: Flint requires FlashInfer attention", -]) + API_SERVER_TAIL -FLINT_MOE_NEEDS_BLACKWELL = "".join(_EC + line + "\n" for line in [ - "EngineCore failed to start.", - "Traceback (most recent call last):", +], FLINT_ATTN) +FLINT_MOE_NEEDS_BLACKWELL = _flint_log([ ' File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/warmup/kernel_warmup.py", line 294, in kernel_warmup', " mixed_batch_attention_warmup(worker)", ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/ops/moe.py", line 806, in forward', " self._check(routed_experts)", ' File "/usr/local/lib/python3.12/dist-packages/vllm/models/flint/ops/moe.py", line 792, in _check', " raise RuntimeError(", - "RuntimeError: Flint BF16 decode MoE requires the FlashInfer TRTLLM BF16 MoE backend " - "without padding (got UnquantizedMoeBackend.TRITON)", -]) + API_SERVER_TAIL +], FLINT_MOE) def _traceback(innermost: str, exception: str, function: str = "__init__") -> str: @@ -280,20 +290,35 @@ def _traceback(innermost: str, exception: str, function: str = "__init__") -> st class TestModelCodeErrors: + @pytest.mark.parametrize("output,cause", [(FLINT_NEEDS_FLASHINFER, FLINT_ATTN), (FLINT_MOE_NEEDS_BLACKWELL, FLINT_MOE)]) + def test_quote_exactly_the_root_cause(self, output, cause): + message = classify(output, model="RecursiveSI/flint") + + # Exactly the exception, not the next log line ("Process EngineCore:") or the wrapper. + assert f"during startup: {cause}. The model's code requires" in message + assert "Process EngineCore" not in message and "Engine core initialization failed" not in message + assert "ATTENTION_BACKEND" in message and "several GPU types" in message + @pytest.mark.parametrize( - "output,cause", + "harmless", [ - (FLINT_NEEDS_FLASHINFER, "AssertionError: Flint requires FlashInfer attention"), - (FLINT_MOE_NEEDS_BLACKWELL, "requires the FlashInfer TRTLLM BF16 MoE backend"), + # torch on a Runpod host with partial NVML support (seen in this image's logs) + "UserWarning: nvmlDeviceGetComputeRunningProcesses failed; omitting per-process memory info " + "from CUDA out-of-memory messages. This is expected on platforms with partial NVML support.", + "FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead.", + # a download retry that then succeeded + "WARNING huggingface_hub: HTTPSConnectionPool(host='cas-bridge.xethub.hf.co', port=443): " + "Read timed out. (read timeout=10) Retrying in 1s [Retry 1/5].", + # c10d's connect retry with tensor parallelism + "[W1021 20:16:30 socket.cpp:767] [c10d] The client socket has failed to connect to [::1]:29500 " + "(errno: 111 - Connection refused).", + "(EngineCore pid=1079) INFO 10-21 20:16:34 [parallel_state.py:1911] world_size=1 rank=0 backend=nccl", ], ) - def test_quote_the_root_cause_not_the_api_server_wrapper(self, output, cause): - message = classify(output, model="RecursiveSI/flint") + def test_harmless_lines_elsewhere_in_the_launch_do_not_block_it(self, harmless): + message = classify(harmless + "\n" + FLINT_NEEDS_FLASHINFER, model="RecursiveSI/flint") - assert message.startswith("RecursiveSI/flint failed inside vLLM's implementation") - assert cause in message - assert "Engine core initialization failed" not in message - assert "ATTENTION_BACKEND" in message + assert message and FLINT_ATTN in message @pytest.mark.parametrize( "exception", @@ -306,11 +331,17 @@ def test_quote_the_root_cause_not_the_api_server_wrapper(self, output, cause): def test_stated_requirements_in_classic_model_code(self, exception): assert exception in classify(_traceback("vllm/model_executor/models/foo.py", exception)) - def test_multiline_requirement_is_quoted_whole(self): + def test_multiline_requirement_within_one_record(self): output = _traceback("vllm/models/foo/model.py", "RuntimeError: Foo decode kernel\nrequires sm100 (got sm90)") assert "RuntimeError: Foo decode kernel requires sm100 (got sm90)" in classify(output) + def test_raw_output_with_vllm_030_short_prefix(self): + lines = _traceback("vllm/models/foo/model.py", "RuntimeError: FooMoE requires the CUTLASS backend").splitlines() + output = "".join(f"(APIServer pid=496) {line}\n" for line in lines) + + assert "RuntimeError: FooMoE requires the CUTLASS backend" in classify(output) + @pytest.mark.parametrize( "innermost,function,exception", [ @@ -320,20 +351,26 @@ def test_multiline_requirement_is_quoted_whole(self): ("vllm/model_executor/models/llama.py", "load_weights", "RuntimeError: NVML_SUCCESS == r INTERNAL ASSERT FAILED at \"/pytorch/c10/cuda/CUDACachingAllocator.cpp\":1131"), ("vllm/model_executor/models/llama.py", "load_weights", "RuntimeError: Failed to load weights: Connection reset by peer"), + ("vllm/model_executor/models/llama.py", "load_weights", + "RuntimeError: Download requires the shard, but the request timed out"), ("vllm/models/flint/ops/moe.py", "forward", "RuntimeError: CUDA error: an illegal memory access was encountered"), # No stated requirement: could be anything. ("vllm/model_executor/models/foo.py", "__init__", "AssertionError"), ("vllm/model_executor/models/foo.py", "forward", "ValueError: Error during index put operation"), - # A requirement, but not raised by model code. - ("vllm/v1/engine/core.py", "__init__", "RuntimeError: this step requires a restart"), ("safetensors/torch.py", "load_file", "ValueError: truncated file"), + # Autograd misuse is a bug, not a configuration requirement. + ("vllm/model_executor/models/foo.py", "forward", + "RuntimeError: a leaf Variable that requires grad is being used in an in-place operation."), + # A stated requirement, but not raised by model code. + ("vllm/v1/engine/core.py", "__init__", "RuntimeError: this step requires a restart"), ], ) def test_everything_else_still_restarts(self, innermost, function, exception): assert classify(_traceback(innermost, exception, function)) is None def test_a_wrapped_cuda_cause_still_restarts(self): - # vLLM's _merge_multimodal_embeddings re-raises a failed index_put as ValueError. + # Synthetic wrapper text: vLLM's _merge_multimodal_embeddings re-raises a failed + # index_put as ValueError; "requires" is added so only the CUDA cause can veto it. cause = _traceback("vllm/model_executor/models/utils.py", "RuntimeError: CUDA error: too many resources requested for launch") wrapper = _traceback("vllm/model_executor/models/utils.py", "ValueError: Error during index put operation requires retry") output = cause + "\nThe above exception was the direct cause of the following exception:\n\n" + wrapper @@ -345,6 +382,32 @@ def test_a_cuda_error_on_a_continuation_line_still_restarts(self): assert classify(output) is None + def test_the_next_log_record_is_not_part_of_the_message(self): + # A bare model-code assertion, then an INFO record that happens to say "requires". + logged = "".join(_EC + line + "\n" for line in _traceback("vllm/models/foo/model.py", "AssertionError").splitlines()) + output = logged + "(EngineCore pid=1079) INFO 10-21 20:16:37 [deepep.py:88] DeepEP v2 requires NCCL >= 2.30.4\n" + + assert classify(output) is None + + def test_a_handled_warning_traceback_does_not_count(self): + warned = _traceback("vllm/models/foo/model.py", "RuntimeError: FooMoE requires the CUTLASS backend") + output = "".join(f"(EngineCore pid=1079) WARNING 10-21 20:16:36 [moe.py:40] {line}\n" for line in warned.splitlines()) + + assert classify(output) is None + + def test_an_engine_killed_by_a_signal_is_not_blamed_on_model_code(self): + output = FLINT_NEEDS_FLASHINFER.replace("Failed core proc(s): {}", "Failed core proc(s): {'EngineCore': -9}") + + assert classify(output) is None + + def test_a_stale_model_frame_does_not_attach_to_a_later_error(self): + output = ( + _traceback("vllm/models/foo/model.py", "safetensors_rust.SafetensorError: Error while deserializing header") + + "RuntimeError: FooMoE requires the CUTLASS backend\n" + ) + + assert classify(output) is None + def test_oom_raised_in_model_code_keeps_the_memory_advice(self): output = _traceback( "vllm/model_executor/models/llama.py", From a17e9aa93d87ee223420a2dd2291a96eb238a9d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 23:39:01 +0000 Subject: [PATCH 4/4] model-code errors: no joining in raw output; the match must end the launch From the re-review: - multi-line messages join only inside a timestamped log record; raw tracebacks (no record boundary) never take the next line, e.g. torch's destroy_process_group() warning at exit - a different exception after the match (other than vLLM's "Engine core / WorkerProc initialization failed" wrappers or a repeat of the cause) cancels it: the match wasn't what ended the launch - transport wording (now also 429/5xx status text) is checked over the matched exception's whole chain, not just its message - vLLM's "NVLink detection failed. This is normal ..." record, logged with an NVMLError traceback on PCIe hosts, isn't read as a host fault - the bare raw-output prefix is anchored to "(Name pid=N) ", so a message line starting with a parenthesis isn't stripped - message: the ATTENTION_BACKEND advice no longer reads as conditional All 5 launches of both full flint H200 logs, the H100 log, the 48 GB OOM log and the K2 logs classify as before. 179 tests (35 new). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- src/startup_errors.py | 68 ++++++++++++++++++++++++++++-------- tests/test_startup_errors.py | 50 +++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 16 deletions(-) diff --git a/src/startup_errors.py b/src/startup_errors.py index 488c9540..97ef1106 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -59,9 +59,15 @@ # vLLM's log prefixes: "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] " # on logged lines, and a bare "(APIServer pid=496) " on raw output (vLLM 0.30). -_LOG_PREFIX = re.compile(r"^\([^)]*\) (?:[A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?)?") +_LOG_PREFIX = re.compile(r"^\([\w.-]+ pid=\d+\) (?:[A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?)?") +# A logged record's prefix has a level and a timestamp; only lines that share one +# belong to the same record. Raw output has no such boundary. +_RECORD = re.compile(r"\) [A-Z]+ [\d-]+ [\d:]+ \[") _FRAME = re.compile(r'^\s+File "([^"]+)", line \d+') _EXCEPTION = re.compile(r"^(?:AssertionError|RuntimeError|NotImplementedError|ValueError)(?:: .*)?$") +_ANY_EXCEPTION = re.compile(r"^[A-Za-z_][\w.]*(?:Error|Exception|Interrupt|Exit)\b(?::|$)") +# Wrappers vLLM raises after an engine process fails; they don't replace the cause. +_WRAPPER = re.compile(r"Engine core initialization failed|WorkerProc initialization failed") _TRACEBACK_BREAK = re.compile( r"^(?:Traceback \(most recent call last\):|The above exception|During handling of the above)" ) @@ -91,7 +97,8 @@ # Transport trouble inside the quoted error itself (a retried download that # succeeded elsewhere in the log must not count, so this only checks the message). _TRANSPORT_FAULT = re.compile( - r"Connection (?:reset|refused|aborted)|timed out|Max retries exceeded|Temporary failure in name resolution", + r"Connection (?:reset|refused|aborted)|timed out|Max retries exceeded|Temporary failure in name resolution|" + r"Too Many Requests|Service Unavailable|Bad Gateway|Gateway Time-?out|Internal Server Error", re.I, ) @@ -102,6 +109,22 @@ _NOT_AN_ERROR_RECORD = re.compile(r"\) (?:DEBUG|INFO|WARNING) ") +def _without_nvlink_probe(output: str) -> str: + """Drop vLLM's "NVLink detection failed. This is normal ..." record (an NVMLError traceback + logged on PCIe hosts with tensor parallelism), so it isn't read as a host fault.""" + kept, skip_prefix = [], None + for raw in output.splitlines(): + prefix, line = _split_prefix(raw) + if "NVLink detection failed" in line and _RECORD.search(prefix): + skip_prefix = prefix + continue + if skip_prefix is not None and prefix == skip_prefix: + continue + skip_prefix = None + kept.append(raw) + return "\n".join(kept) + + def _split_prefix(raw: str): line = raw.rstrip() match = _LOG_PREFIX.match(line) @@ -115,42 +138,57 @@ def model_code_error(output: str) -> Optional[str]: states a requirement the setup does not meet: flint asserts the FlashInfer attention backend, and its BF16 MoE needs a Blackwell-only kernel. Those fail identically on every start. Returns None when the failed launch shows - a host fault, or the message itself a transport fault, so those keep - getting a restart. The last match wins: it is the one that ended the launch. + a host fault, when the matched exception's chain shows a transport fault, + or when a different exception (other than vLLM's wrappers) comes after the + match, since then the match isn't what ended the launch. """ - if _HOST_FAULT.search(output) or _ENGINE_KILLED.search(output): + if _HOST_FAULT.search(_without_nvlink_probe(output)) or _ENGINE_KILLED.search(output): return None found = None last_file = None message, message_prefix = None, None + chain, after_separator = [], False def finish(lines): text = " ".join(lines) - if _REQUIREMENT.search(text) and not _TRANSPORT_FAULT.search(text): + if _REQUIREMENT.search(text) and not _TRANSPORT_FAULT.search(" ".join(chain + [text])): return text[:300] return None for raw in output.splitlines(): prefix, line = _split_prefix(raw) if message is not None: - # A multi-line message continues only within the same log record (same - # prefix): the next record, e.g. multiprocessing's "Process EngineCore:", - # is not part of it. - if (prefix == message_prefix and line and not line.startswith(" ") - and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line)): + # A multi-line message continues only inside one logged record (same + # timestamped prefix). The next record, multiprocessing's "Process + # EngineCore:", or a warning printed after a raw traceback are not part of it. + if (_RECORD.search(message_prefix) and prefix == message_prefix and line + and not line.startswith(" ") and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line)): message.append(line) continue found = finish(message) or found + chain.append(" ".join(message)) message = None + if _TRACEBACK_BREAK.match(line): + if line.startswith("Traceback") and not after_separator: + chain = [] # a new, unchained traceback + after_separator = not line.startswith("Traceback") + last_file = None + continue frame = _FRAME.match(line) if frame: last_file = frame.group(1) continue - if line.startswith(" "): + if line.startswith(" ") or not line: continue # source and caret lines under a frame if (_EXCEPTION.match(line) and last_file and _MODEL_CODE.search(last_file) and not _NOT_AN_ERROR_RECORD.search(prefix)): message, message_prefix = [line], prefix + elif _ANY_EXCEPTION.match(line): + chain.append(line) + # A different exception after the match, other than vLLM's wrapper or a + # repeat of the same cause (logged copy, then raw copy), ended the launch. + if found and not _WRAPPER.search(line) and not found.startswith(line[:80]): + found = None last_file = None # any other unindented line ends the frame list if message is not None: found = finish(message) or found @@ -340,9 +378,9 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False f"{named} failed inside vLLM's implementation of the model during " f"startup: {raised.rstrip('.')}. The model's code requires something this " f"worker's GPU or settings don't provide, so restarting this worker will not " - f"change it. If the endpoint lists several GPU types, remove the ones the " - f"model can't use; otherwise check the model card and vLLM's recipe for the " - f"attention backend (ATTENTION_BACKEND), GPU generation or dtype it needs." + f"change it. Check the model card and vLLM's recipe for the attention backend " + f"(ATTENTION_BACKEND), GPU generation or dtype it needs. If the endpoint lists " + f"several GPU types, also remove the ones the model can't use." ) return None diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index ff435194..2c512d28 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -332,10 +332,40 @@ def test_stated_requirements_in_classic_model_code(self, exception): assert exception in classify(_traceback("vllm/model_executor/models/foo.py", exception)) def test_multiline_requirement_within_one_record(self): - output = _traceback("vllm/models/foo/model.py", "RuntimeError: Foo decode kernel\nrequires sm100 (got sm90)") + lines = _traceback("vllm/models/foo/model.py", "RuntimeError: Foo decode kernel\nrequires sm100 (got sm90)").splitlines() + output = "".join(_EC + line + "\n" for line in lines) assert "RuntimeError: Foo decode kernel requires sm100 (got sm90)" in classify(output) + def test_raw_tracebacks_are_never_joined_with_the_next_line(self): + # torch prints this at process exit; raw output has no record boundary. + teardown = ("[rank0]:[W1021 20:16:37.1 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() " + "was not called before program exit, which can leak resources.\n") + message = classify(FLINT_NEEDS_FLASHINFER.replace(API_SERVER_TAIL, teardown + API_SERVER_TAIL), model="RecursiveSI/flint") + assert f"during startup: {FLINT_ATTN}. The model's code" in message + + bare = _traceback("vllm/models/foo/model.py", "AssertionError") + "Note: this kernel requires a warm cache\n" + assert classify(bare) is None + + def test_a_parenthesised_message_line_is_not_a_log_prefix(self): + exception = "RuntimeError: Foo MoE requires the TRTLLM backend\n(got TRITON) on this GPU" + lines = _traceback("vllm/models/foo/model.py", exception).splitlines() + output = "".join(_EC + line + "\n" for line in lines) + + assert "requires the TRTLLM backend (got TRITON) on this GPU" in classify(output) + + def test_vllms_nvlink_probe_record_is_not_a_host_fault(self): + nvlink = "".join( + "(EngineCore pid=1079) ERROR 10-21 20:16:30 [cuda.py:834] " + line + "\n" + for line in ["NVLink detection failed. This is normal if your machine has no NVLink equipped.", + "Traceback (most recent call last):", + ' File "/usr/local/lib/python3.12/dist-packages/pynvml.py", line 2380, in _nvmlCheckReturn', + " raise NVMLError(ret)", + "pynvml.NVMLError_NotSupported: Not Supported"]) + message = classify(nvlink + FLINT_NEEDS_FLASHINFER, model="RecursiveSI/flint") + + assert message and FLINT_ATTN in message + def test_raw_output_with_vllm_030_short_prefix(self): lines = _traceback("vllm/models/foo/model.py", "RuntimeError: FooMoE requires the CUTLASS backend").splitlines() output = "".join(f"(APIServer pid=496) {line}\n" for line in lines) @@ -400,6 +430,24 @@ def test_an_engine_killed_by_a_signal_is_not_blamed_on_model_code(self): assert classify(output) is None + def test_a_later_unrelated_failure_is_what_ended_the_launch(self): + output = FLINT_NEEDS_FLASHINFER.replace( + API_SERVER_TAIL, + _traceback("safetensors/torch.py", "safetensors_rust.SafetensorError: Error while deserializing header: " + "MetadataIncompleteBuffer", "load_file") + API_SERVER_TAIL, + ) + + assert classify(output) is None + + @pytest.mark.parametrize("cause", ["ConnectionResetError: [Errno 104] Connection reset by peer", + "requests.exceptions.HTTPError: 503 Server Error: Service Unavailable"]) + def test_a_transport_fault_in_the_chain_still_restarts(self, cause): + output = (_traceback("huggingface_hub/file_download.py", cause, "http_get") + + "\nThe above exception was the direct cause of the following exception:\n\n" + + _traceback("vllm/models/foo/model.py", "ValueError: Foo vision requires preprocessor_config.json")) + + assert classify(output) is None + def test_a_stale_model_frame_does_not_attach_to_a_later_error(self): output = ( _traceback("vllm/models/foo/model.py", "safetensors_rust.SafetensorError: Error while deserializing header")