fix: answer jobs when a model's own vLLM code rejects the setup

README.md

@@ -105,7 +105,7 @@ For the complete list of all available environment variables, examples, and deta
 
 ### When vLLM will not start
 
-Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result.
+Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`, a model whose own vLLM code states a requirement this worker's GPU or settings don't meet (for example the FlashInfer attention backend, or a newer GPU generation). Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result.
 
 The worker recognises these and stays up instead, answering every job with the cause and the fix:
 
@@ -119,7 +119,7 @@ The worker recognises these and stays up instead, answering every job with the c
 }
 ```
 
-The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt.
+The same message is logged by the worker, and the full vLLM traceback stays in the worker logs above it. Anything unrecognised still exits non-zero, because a failed download or a bad host is worth another attempt. For model code (`vllm/model_executor/models/`, `vllm/models/`), only an error whose message states a requirement ("requires", "only supports", "not supported on") counts; any other error there, and any failed start that shows a CUDA, NCCL, NVML, cuBLAS/cuDNN, ECC or Xid fault, still restarts. A requirement can depend on the GPU type, so on an endpoint that lists several, a worker on a GPU the model can't use answers jobs with the cause instead of leaving them to a worker that can: remove those GPU types from the endpoint.
 
 ## Option 2: Build Docker Image with Model Inside
 

src/startup_errors.py

@@ -57,6 +57,143 @@
 # --- Disk -------------------------------------------------------------------
 _NO_SPACE = re.compile(r"No space left on device|ENOSPC|errno 28|Disk quota exceeded|EDQUOT|errno 122", re.I)
 
+# vLLM's log prefixes: "(EngineCore pid=1014) ERROR 10-21 20:41:12 [core.py:1543] "
+# on logged lines, and a bare "(APIServer pid=496) " on raw output (vLLM 0.30).
+_LOG_PREFIX = re.compile(r"^\([\w.-]+ pid=\d+\) (?:[A-Z]+ [\d-]+ [\d:]+ \[[^\]]*\] ?)?")
+# A logged record's prefix has a level and a timestamp; only lines that share one
+# belong to the same record. Raw output has no such boundary.
+_RECORD = re.compile(r"\) [A-Z]+ [\d-]+ [\d:]+ \[")
+_FRAME = re.compile(r'^\s+File "([^"]+)", line \d+')
+_EXCEPTION = re.compile(r"^(?:AssertionError|RuntimeError|NotImplementedError|ValueError)(?:: .*)?$")
+_ANY_EXCEPTION = re.compile(r"^[A-Za-z_][\w.]*(?:Error|Exception|Interrupt|Exit)\b(?::|$)")
+# Wrappers vLLM raises after an engine process fails; they don't replace the cause.
+_WRAPPER = re.compile(r"Engine core initialization failed|WorkerProc initialization failed")
+_TRACEBACK_BREAK = re.compile(
+    r"^(?:Traceback \(most recent call last\):|The above exception|During handling of the above)"
+)
+# Model implementations: vllm/model_executor/models/<name>.py, and the per-model
+# packages under vllm/models/<name>/ (since v0.22; v0.32.0 ships deepseek_v4,
+# kimi_k3, minimax_m3 and others there).
+_MODEL_CODE = re.compile(r"/vllm/(?:model_executor/)?models/")
+# The model's code states what it needs: "Flint requires FlashInfer attention",
+# "... requires the FlashInfer TRTLLM BF16 MoE backend", "only supported on ...".
+# Where an error is raised says nothing about permanence (a driver fault inside
+# load_weights also ends in model code); a stated requirement does. Autograd's
+# "requires grad" errors are not configuration requirements.
+_REQUIREMENT = re.compile(
+    r"\brequires?\b(?![ _]grad)|\bonly supports?\b|\bonly supported\b|\bnot supported (?:on|with|for|by)\b",
+    re.I,
+)
+# Host faults anywhere in the failed launch, including a CUDA error wrapped by a
+# model-code exception: another host can succeed. Fault signatures only, so
+# torch's NVML/pynvml warnings on a healthy host don't count.
+_HOST_FAULT = re.compile(
+    r"CUDA (?:driver |unknown )?error|cudaError|CUDA-capable device|NVIDIA driver on your system is too old|"
+    r"NCCL error|nccl(?:System|Unhandled|Internal)\w*Error|Watchdog caught collective operation timeout|"
+    r"NVML_SUCCESS == r|NVMLError|nvml\w*Error|CUBLAS_STATUS|CUDNN_STATUS|illegal memory access|"
+    r"uncorrectable ECC|\bXid\b",
+    re.I,
+)
+# Transport trouble inside the quoted error itself (a retried download that
+# succeeded elsewhere in the log must not count, so this only checks the message).
+_TRANSPORT_FAULT = re.compile(
+    r"Connection (?:reset|refused|aborted)|timed out|Max retries exceeded|Temporary failure in name resolution|"
+    r"Too Many Requests|Service Unavailable|Bad Gateway|Gateway Time-?out|Internal Server Error",
+    re.I,
+)
+
+# An engine process killed by a signal ("Failed core proc(s): {'EngineCore': -9}")
+# died of something else, whatever model code logged earlier.
+_ENGINE_KILLED = re.compile(r"Failed core proc\(s\): \{[^}]*: -\d+")
+# A traceback logged below ERROR was caught and handled; it didn't end the launch.
+_NOT_AN_ERROR_RECORD = re.compile(r"\) (?:DEBUG|INFO|WARNING) ")
+
+
+def _without_nvlink_probe(output: str) -> str:
+    """Drop vLLM's "NVLink detection failed. This is normal ..." record (an NVMLError traceback
+    logged on PCIe hosts with tensor parallelism), so it isn't read as a host fault."""
+    kept, skip_prefix = [], None
+    for raw in output.splitlines():
+        prefix, line = _split_prefix(raw)
+        if "NVLink detection failed" in line and _RECORD.search(prefix):
+            skip_prefix = prefix
+            continue
+        if skip_prefix is not None and prefix == skip_prefix:
+            continue
+        skip_prefix = None
+        kept.append(raw)
+    return "\n".join(kept)
+
+
+def _split_prefix(raw: str):
+    line = raw.rstrip()
+    match = _LOG_PREFIX.match(line)
+    return (match.group(0), line[match.end():]) if match else ("", line)
+
+
+def model_code_error(output: str) -> Optional[str]:
+    """The stated requirement a model's own vLLM code failed on, if that is all that went wrong.
+
+    Matches an exception whose innermost frame is model code and whose message
+    states a requirement the setup does not meet: flint asserts the FlashInfer
+    attention backend, and its BF16 MoE needs a Blackwell-only kernel. Those
+    fail identically on every start. Returns None when the failed launch shows
+    a host fault, when the matched exception's chain shows a transport fault,
+    or when a different exception (other than vLLM's wrappers) comes after the
+    match, since then the match isn't what ended the launch.
+    """
+    if _HOST_FAULT.search(_without_nvlink_probe(output)) or _ENGINE_KILLED.search(output):
+        return None
+    found = None
+    last_file = None
+    message, message_prefix = None, None
+    chain, after_separator = [], False
+
+    def finish(lines):
+        text = " ".join(lines)
+        if _REQUIREMENT.search(text) and not _TRANSPORT_FAULT.search(" ".join(chain + [text])):
+            return text[:300]
+        return None
+
+    for raw in output.splitlines():
+        prefix, line = _split_prefix(raw)
+        if message is not None:
+            # A multi-line message continues only inside one logged record (same
+            # timestamped prefix). The next record, multiprocessing's "Process
+            # EngineCore:", or a warning printed after a raw traceback are not part of it.
+            if (_RECORD.search(message_prefix) and prefix == message_prefix and line
+                    and not line.startswith(" ") and not _FRAME.match(line) and not _TRACEBACK_BREAK.match(line)):
+                message.append(line)
+                continue
+            found = finish(message) or found
+            chain.append(" ".join(message))
+            message = None
+        if _TRACEBACK_BREAK.match(line):
+            if line.startswith("Traceback") and not after_separator:
+                chain = []  # a new, unchained traceback
+            after_separator = not line.startswith("Traceback")
+            last_file = None
+            continue
+        frame = _FRAME.match(line)
+        if frame:
+            last_file = frame.group(1)
+            continue
+        if line.startswith(" ") or not line:
+            continue  # source and caret lines under a frame
+        if (_EXCEPTION.match(line) and last_file and _MODEL_CODE.search(last_file)
+                and not _NOT_AN_ERROR_RECORD.search(prefix)):
+            message, message_prefix = [line], prefix
+        elif _ANY_EXCEPTION.match(line):
+            chain.append(line)
+            # A different exception after the match, other than vLLM's wrapper or a
+            # repeat of the same cause (logged copy, then raw copy), ended the launch.
+            if found and not _WRAPPER.search(line) and not found.startswith(line[:80]):
+                found = None
+        last_file = None  # any other unindented line ends the frame list
+    if message is not None:
+        found = finish(message) or found
+    return found
+
 
 # The model-access messages are shared with model_preflight.py, which detects
 # the same failures before the cold start; whether the user hits the fast path
@@ -235,4 +372,15 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False
             f"weights are cached there instead."
         )
 
+    raised = model_code_error(output)
+    if raised:
+        return (
+            f"{named} failed inside vLLM's implementation of the model during "
+            f"startup: {raised.rstrip('.')}. The model's code requires something this "
+            f"worker's GPU or settings don't provide, so restarting this worker will not "
+            f"change it. Check the model card and vLLM's recipe for the attention backend "
+            f"(ATTENTION_BACKEND), GPU generation or dtype it needs. If the endpoint lists "
+            f"several GPU types, also remove the ones the model can't use."
+        )
+
     return None