From b779895689133c9d403dbd4265cb2c7c806c6991 Mon Sep 17 00:00:00 2001 From: Ashwin Sreedhar Date: Wed, 21 Oct 2026 18:57:46 +0000 Subject: [PATCH] fix: let the vLLM version guard accept non-release base tags The guard added in #335 compares vllm.__version__ to VLLM_VERSION with the leading "v" stripped. That only holds for release tags (v0.32.0). Nightly, latest and vLLM's model-named day-0 images (step-5-preview, muse-spark-open, gemma4, ...) carry dev versions, so every build from them fails, including the nightly path the README documents. Record the base image's vLLM before any pip install and verify afterwards that it is unchanged; this still catches the vllm-bnb-plugin downgrade for any base tag. For release tags (including -cu130 style suffixes) also check the installed version is that release. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016kxoD6sb3tVrig5ddmB8f1 --- Dockerfile | 10 ++++- builder/check_vllm_version.py | 64 ++++++++++++++++++++++++++++++++ tests/test_check_vllm_version.py | 49 ++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 builder/check_vllm_version.py create mode 100644 tests/test_check_vllm_version.py diff --git a/Dockerfile b/Dockerfile index 3074f744..cf076157 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,10 @@ FROM vllm/vllm-openai:${VLLM_VERSION} # Re-declare so the stage can reference it in RUN steps below. ARG VLLM_VERSION +# Record the base image's vLLM before any pip install can touch it (checked below). +COPY builder/check_vllm_version.py /check_vllm_version.py +RUN python3 /check_vllm_version.py record + # RunPod serverless SDK + HTTP proxy deps (vLLM itself comes from the base image). COPY builder/requirements.txt /requirements.txt RUN python3 -m ensurepip --upgrade 2>/dev/null || true \ @@ -21,8 +25,10 @@ RUN python3 -m ensurepip --upgrade 2>/dev/null || true \ RUN python3 -m pip install --no-cache-dir --no-deps "vllm-bnb-plugin>=0.0.3" # Guard rail: the base image's vLLM must survive our pip installs — fail the -# build loudly if the installed version no longer matches the requested tag. -RUN test "$(python3 -c 'import vllm; print(vllm.__version__)')" = "${VLLM_VERSION#v}" +# build loudly if it changed, and for release tags (vX.Y.Z) if it isn't that +# release. Nightly, latest and model-named day-0 tags (e.g. step-5-preview) +# have no release number, so only the "unchanged" check applies to them. +RUN python3 /check_vllm_version.py verify "${VLLM_VERSION}" # Setup for Option 2: building the image with the model baked in. ARG MODEL_NAME="" diff --git a/builder/check_vllm_version.py b/builder/check_vllm_version.py new file mode 100644 index 00000000..6d384d64 --- /dev/null +++ b/builder/check_vllm_version.py @@ -0,0 +1,64 @@ +"""Build-time guard: the base image's vLLM must survive the worker's pip installs. + +Run twice from the Dockerfile: + + python3 check_vllm_version.py record # before any pip install + python3 check_vllm_version.py verify "$VLLM_VERSION" # after the last one + +`verify` fails the build if the installed vLLM differs from the one recorded, +which is how a dependency once silently replaced vLLM 0.28.0 with 0.19.1 +(vllm-bnb-plugin, #335). When VLLM_VERSION is a release tag (v0.32.0, +v0.32.0-cu130) it also checks the installed version is that release. + +Any other base tag (nightly, latest, or a model-named day-0 image such as +step-5-preview) has no release number to compare against, so only the +"unchanged during the build" check applies. Comparing such tags to +vllm.__version__ directly would reject them outright. +""" + +import re +import sys +from pathlib import Path + +RECORD = Path("/usr/local/share/worker-vllm/base_vllm_version") +RELEASE_TAG = re.compile(r"^v?(\d+\.\d+\.\d+)(?:[-+].*)?$") + + +def installed_version() -> str: + import vllm + + return vllm.__version__ + + +def check(base: str, installed: str, requested_tag: str) -> str | None: + """Return an error message, or None if the installed vLLM is acceptable.""" + if installed != base: + return (f"vLLM changed during the build: base image had {base}, now {installed}. " + "A pip install replaced it; pin or --no-deps the offending package.") + m = RELEASE_TAG.match(requested_tag) + if m and installed != m.group(1): + return (f"VLLM_VERSION={requested_tag} names release {m.group(1)}, " + f"but the base image contains vLLM {installed}.") + return None + + +def main(argv: list[str]) -> int: + if len(argv) == 2 and argv[1] == "record": + RECORD.parent.mkdir(parents=True, exist_ok=True) + RECORD.write_text(installed_version()) + print(f"base image vLLM: {RECORD.read_text()}") + return 0 + if len(argv) == 3 and argv[1] == "verify": + installed = installed_version() + err = check(RECORD.read_text().strip(), installed, argv[2]) + if err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + print(f"vLLM {installed} OK for base tag {argv[2]}") + return 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_check_vllm_version.py b/tests/test_check_vllm_version.py new file mode 100644 index 00000000..1575d833 --- /dev/null +++ b/tests/test_check_vllm_version.py @@ -0,0 +1,49 @@ +"""Build-time vLLM version guard (builder/check_vllm_version.py).""" + +import importlib.util +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +spec = importlib.util.spec_from_file_location("check_vllm_version", ROOT / "builder" / "check_vllm_version.py") +cvv = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cvv) + + +@pytest.mark.parametrize("tag", ["v0.32.0", "0.32.0", "v0.32.0-cu130", "v0.32.0-x86_64-cu129"]) +def test_release_tag_matching_install_passes(tag): + assert cvv.check("0.32.0", "0.32.0", tag) is None + + +def test_release_tag_with_other_install_fails(): + assert "names release 0.32.0" in cvv.check("0.31.0", "0.31.0", "v0.32.0") + + +@pytest.mark.parametrize("tag", ["nightly", "latest", "step-5-preview", "muse-spark-open", + "deepseekv41-flash-0909", "cu129-nightly-77154ca6"]) +def test_non_release_tags_pass_when_unchanged(tag): + # Model-named day-0 and nightly images carry dev versions; the old check + # (vllm.__version__ == tag) rejected every one of them. + assert cvv.check("0.32.1.dev45+g1b2a666", "0.32.1.dev45+g1b2a666", tag) is None + + +@pytest.mark.parametrize("tag", ["v0.28.0", "step-5-preview"]) +def test_downgrade_during_build_fails_for_any_tag(tag): + # The vllm-bnb-plugin incident: base 0.28.0 silently replaced by 0.19.1. + err = cvv.check("0.28.0", "0.19.1", tag) + assert err and "changed during the build" in err + + +def _instruction_index(lines, pattern): + return next(i for i, line in enumerate(lines) if re.search(pattern, line)) + + +def test_dockerfile_records_before_and_verifies_after_every_pip_install(): + lines = (ROOT / "Dockerfile").read_text().splitlines() + record = _instruction_index(lines, r"check_vllm_version\.py record") + verify = _instruction_index(lines, r"check_vllm_version\.py verify") + pip = [i for i, line in enumerate(lines) if "pip install" in line and not line.lstrip().startswith("#")] + assert pip, "expected pip installs in the Dockerfile" + assert record < min(pip) and verify > max(pip)