From c1b457fdd2284bd057837488bb4f989e91620388 Mon Sep 17 00:00:00 2001 From: Ashwin Sreedhar Date: Wed, 21 Oct 2026 19:19:09 +0000 Subject: [PATCH 1/2] fix: answer jobs with a TRUST_REMOTE_CODE hint instead of crash-looping When loading a model requires code from its repository and TRUST_REMOTE_CODE isn't set, vLLM fails config validation with a ValidationError asking for trust_remote_code=True. The worker didn't recognise it, so it exited and was restarted into the same error, and the job got no answer. Seen with IFM/K2-Horizon-7B on v2.27.2 and v2.28.0: the worker failed the same way on every start, and the submitted job had no result after 15 minutes. Classify it like the other fatal startup failures and tell the user to set TRUST_REMOTE_CODE=true after reviewing the code. It is checked before the unsupported-architecture match because with remote code allowed, vLLM can serve an architecture it has no native class for through its Transformers backend (K2-Horizon loaded that way on vLLM 0.28.0). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- README.md | 2 +- src/startup_errors.py | 19 +++++++++++++++++++ tests/test_startup_errors.py | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f634fd5..3fa4f3c8 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ For the complete list of all available environment variables, examples, and deta ### When vLLM will not start -Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`. Exiting on those crash-loops the worker: every attempt pays for the download and the model load, and the console shows only silent restarts with no error anywhere. +Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that ships its own modeling code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: every attempt pays for the download and the model load, and the console shows only silent restarts with no error anywhere. The worker recognises these and stays up instead, answering every job with the cause and the fix: diff --git a/src/startup_errors.py b/src/startup_errors.py index eb2d0ca5..669abac9 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -46,6 +46,12 @@ _REVISION_NOT_FOUND = re.compile(r"RevisionNotFoundError|Revision Not Found", re.I) _GATED = re.compile(r"GatedRepoError|401 Client Error", re.I) _NOT_FOUND = re.compile(r"RepositoryNotFoundError|404 Client Error", re.I) +# transformers/dynamic_module_utils.py: config.json has an auto_map pointing at +# the repo's own modeling code, and vLLM won't run it without trust_remote_code. +# Wrapped in a pydantic ValidationError by vLLM's ModelConfig, so match the text. +_NEEDS_REMOTE_CODE = re.compile( + r"contains custom code which must be executed|pass the argument `trust_remote_code=True`", re.I +) _UNSUPPORTED_ARCH = re.compile(r"Model architectures \[.*?\] (?:are not supported|failed to be inspected)", re.I) # --- Disk ------------------------------------------------------------------- @@ -199,6 +205,19 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False if _NOT_FOUND.search(output): return not_found_message(named) + if _NEEDS_REMOTE_CODE.search(output): + # Checked before the architecture match: with remote code allowed, vLLM + # can run an architecture it has no native class for through its + # Transformers backend, so "unsupported" would be the wrong advice. + return ( + f"{named} ships its own modeling code on Hugging Face (an auto_map in " + f"its config.json), and vLLM only runs that code with " + f"TRUST_REMOTE_CODE=true. Review the repository's code, then set " + f"TRUST_REMOTE_CODE=true if you trust it. This also lets vLLM serve " + f"architectures it has no native implementation for yet, through its " + f"Transformers backend." + ) + if _UNSUPPORTED_ARCH.search(output): return ( f"{named} uses an architecture this vLLM version cannot serve. Check " diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 799a995e..35ba86fd 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -44,6 +44,13 @@ "for url https://huggingface.co/org/nope/resolve/main/config.json." ) UNSUPPORTED = "ValueError: Model architectures ['FooForCausalLM'] are not supported for now." +# vLLM v0.30 worker log for IFM/K2-Horizon-7B without TRUST_REMOTE_CODE (the worker crash-looped on it). +NEEDS_REMOTE_CODE = ( + "pydantic_core._pydantic_core.ValidationError: 1 validation error for ModelConfig\n" + " Value error, The repository IFM/K2-Horizon-7B contains custom code which must be executed to " + "correctly load the model. You can inspect the repository content at https://hf.co/IFM/K2-Horizon-7B .\n" + "Please pass the argument `trust_remote_code=True` to allow custom code to be run. [type=value_error]" +) NO_SPACE = "OSError: [Errno 28] No space left on device" @@ -102,6 +109,17 @@ def test_missing_repo_points_at_model_name(self): def test_unsupported_architecture(self): assert "architecture" in classify(UNSUPPORTED) + def test_custom_code_points_at_trust_remote_code(self): + message = classify(NEEDS_REMOTE_CODE, model="IFM/K2-Horizon-7B") + + assert message.startswith("IFM/K2-Horizon-7B ships its own modeling code") + assert "TRUST_REMOTE_CODE=true" in message + + def test_custom_code_wins_over_unsupported_architecture(self): + # With remote code allowed, vLLM's Transformers backend can serve an + # architecture it has no native class for, so don't say "unsupported". + assert "TRUST_REMOTE_CODE=true" in classify(UNSUPPORTED + "\n" + NEEDS_REMOTE_CODE) + def test_out_of_disk_points_at_container_disk(): assert "container disk" in classify(NO_SPACE) From 1f5ff2b65ec9072f1fd32acbf7f26de74a411279 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 21:41:59 +0000 Subject: [PATCH 2/2] README: qualify the remote-code case and the crash-loop cost Only a model that requires repository-provided code fails without TRUST_REMOTE_CODE, and its failure happens at config validation, before any weights load, with the error in the logs. Say what holds for every case in the list: the startup repeats and jobs get no result. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fa4f3c8..a3db1ddf 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ For the complete list of all available environment variables, examples, and deta ### When vLLM will not start -Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that ships its own modeling code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: every attempt pays for the download and the model load, and the console shows only silent restarts with no error anywhere. +Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result. The worker recognises these and stays up instead, answering every job with the cause and the fix: