diff --git a/README.md b/README.md index 5f634fd5..a3db1ddf 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ For the complete list of all available environment variables, examples, and deta ### When vLLM will not start -Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`. Exiting on those crash-loops the worker: every attempt pays for the download and the model load, and the console shows only silent restarts with no error anywhere. +Some startup failures a restart cannot fix: the model does not fit the GPU, `MAX_MODEL_LEN` is larger than the KV cache the GPU can hold, a flag or value vLLM rejects, a gated model without `HF_TOKEN`, a model that requires repository-provided code without `TRUST_REMOTE_CODE=true`. Exiting on those crash-loops the worker: each attempt repeats the startup (for most of these, the download and the model load), and submitted jobs receive no result. The worker recognises these and stays up instead, answering every job with the cause and the fix: diff --git a/src/startup_errors.py b/src/startup_errors.py index eb2d0ca5..669abac9 100644 --- a/src/startup_errors.py +++ b/src/startup_errors.py @@ -46,6 +46,12 @@ _REVISION_NOT_FOUND = re.compile(r"RevisionNotFoundError|Revision Not Found", re.I) _GATED = re.compile(r"GatedRepoError|401 Client Error", re.I) _NOT_FOUND = re.compile(r"RepositoryNotFoundError|404 Client Error", re.I) +# transformers/dynamic_module_utils.py: config.json has an auto_map pointing at +# the repo's own modeling code, and vLLM won't run it without trust_remote_code. +# Wrapped in a pydantic ValidationError by vLLM's ModelConfig, so match the text. +_NEEDS_REMOTE_CODE = re.compile( + r"contains custom code which must be executed|pass the argument `trust_remote_code=True`", re.I +) _UNSUPPORTED_ARCH = re.compile(r"Model architectures \[.*?\] (?:are not supported|failed to be inspected)", re.I) # --- Disk ------------------------------------------------------------------- @@ -199,6 +205,19 @@ def classify(output: str, model: Optional[str] = None, oom_retried: bool = False if _NOT_FOUND.search(output): return not_found_message(named) + if _NEEDS_REMOTE_CODE.search(output): + # Checked before the architecture match: with remote code allowed, vLLM + # can run an architecture it has no native class for through its + # Transformers backend, so "unsupported" would be the wrong advice. + return ( + f"{named} ships its own modeling code on Hugging Face (an auto_map in " + f"its config.json), and vLLM only runs that code with " + f"TRUST_REMOTE_CODE=true. Review the repository's code, then set " + f"TRUST_REMOTE_CODE=true if you trust it. This also lets vLLM serve " + f"architectures it has no native implementation for yet, through its " + f"Transformers backend." + ) + if _UNSUPPORTED_ARCH.search(output): return ( f"{named} uses an architecture this vLLM version cannot serve. Check " diff --git a/tests/test_startup_errors.py b/tests/test_startup_errors.py index 799a995e..35ba86fd 100644 --- a/tests/test_startup_errors.py +++ b/tests/test_startup_errors.py @@ -44,6 +44,13 @@ "for url https://huggingface.co/org/nope/resolve/main/config.json." ) UNSUPPORTED = "ValueError: Model architectures ['FooForCausalLM'] are not supported for now." +# vLLM v0.30 worker log for IFM/K2-Horizon-7B without TRUST_REMOTE_CODE (the worker crash-looped on it). +NEEDS_REMOTE_CODE = ( + "pydantic_core._pydantic_core.ValidationError: 1 validation error for ModelConfig\n" + " Value error, The repository IFM/K2-Horizon-7B contains custom code which must be executed to " + "correctly load the model. You can inspect the repository content at https://hf.co/IFM/K2-Horizon-7B .\n" + "Please pass the argument `trust_remote_code=True` to allow custom code to be run. [type=value_error]" +) NO_SPACE = "OSError: [Errno 28] No space left on device" @@ -102,6 +109,17 @@ def test_missing_repo_points_at_model_name(self): def test_unsupported_architecture(self): assert "architecture" in classify(UNSUPPORTED) + def test_custom_code_points_at_trust_remote_code(self): + message = classify(NEEDS_REMOTE_CODE, model="IFM/K2-Horizon-7B") + + assert message.startswith("IFM/K2-Horizon-7B ships its own modeling code") + assert "TRUST_REMOTE_CODE=true" in message + + def test_custom_code_wins_over_unsupported_architecture(self): + # With remote code allowed, vLLM's Transformers backend can serve an + # architecture it has no native class for, so don't say "unsupported". + assert "TRUST_REMOTE_CODE=true" in classify(UNSUPPORTED + "\n" + NEEDS_REMOTE_CODE) + def test_out_of_disk_points_at_container_disk(): assert "container disk" in classify(NO_SPACE)