From bc2de13f5f5451332151169d99af80db22b737fb Mon Sep 17 00:00:00 2001 From: FenjuFu <92919259+FenjuFu@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:21:39 +0800 Subject: [PATCH 1/2] fix(cases): update RTX 3090 evaluation runtime for vLLM security fixes Signed-off-by: FenjuFu <92919259+FenjuFu@users.noreply.github.com> --- cases/domux-4057-rtx3090-eval/README.md | 4 +++ cases/domux-4057-rtx3090-eval/RUNBOOK.md | 30 +++++++++++-------- .../prepare_and_serve.sh | 11 +++---- cases/domux-4057-rtx3090-eval/render_case.py | 13 ++++---- .../domux-4057-rtx3090-eval/render_preview.py | 2 +- .../requirements-case.txt | 6 +++- 6 files changed, 41 insertions(+), 25 deletions(-) diff --git a/cases/domux-4057-rtx3090-eval/README.md b/cases/domux-4057-rtx3090-eval/README.md index fe82be7..9d7f68d 100644 --- a/cases/domux-4057-rtx3090-eval/README.md +++ b/cases/domux-4057-rtx3090-eval/README.md @@ -13,6 +13,10 @@ channels: # Domux 4,057-sample open evaluation on RTX 3090 +The results below record the original vLLM 0.22.0 experiment. New installations +use the security-updated runtime in [RUNBOOK.md](RUNBOOK.md); those versions need +a new GPU evaluation before their results can replace this historical evidence. + ## Task / 真实任务 I reproduced the complete public Domux evaluation on a consumer RTX 3090, then diff --git a/cases/domux-4057-rtx3090-eval/RUNBOOK.md b/cases/domux-4057-rtx3090-eval/RUNBOOK.md index 36c478c..f1fd57d 100644 --- a/cases/domux-4057-rtx3090-eval/RUNBOOK.md +++ b/cases/domux-4057-rtx3090-eval/RUNBOOK.md @@ -4,29 +4,31 @@ This runbook keeps Hugging Face credentials and private server details outside the repository. Run every command from a clone of your GitHub Fork on the Linux GPU server. +New runs use vLLM 0.31.0 to include the published security fixes through +[GHSA-h3rc-6mm3-gc2m](https://github.com/vllm-project/vllm/security/advisories/GHSA-h3rc-6mm3-gc2m). +The committed README metrics and artifacts describe the original vLLM 0.22.0 +run; they are not validation of the upgraded runtime. Before publishing updated +results, run all smoke, correctness, and latency gates below on the RTX 3090. + ## 1. One-time account and environment setup Accept the Gemma terms on the Domux Hugging Face model page, then authenticate interactively on the server: ```bash -python3 -m venv .venv +python3.11 -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip pip install -r cases/domux-4057-rtx3090-eval/requirements-case.txt -pip install "transformers==5.5.1" -pip install \ - "nvidia-cuda-nvcc==13.0.88" \ - "nvidia-cuda-crt==13.0.88" \ - "nvidia-nvvm==13.0.88" +python -m pip check hf auth login ``` -The second install is intentional: vLLM 0.22.0 can otherwise resolve to -Transformers 4.57.x through its optional structured-output dependency, while -Domux uses the newer `gemma4` configuration. This case does not use structured -decoding; it requires Transformers 5.5.1 and verifies the version before model -startup. +Use a fresh Python 3.11 environment instead of upgrading the historical +environment in place. The requirements resolve vLLM, Transformers 5.10.4, and +the CUDA compiler together. Transformers 5.10.4 satisfies vLLM 0.31.0's minimum +version and supports Domux's `gemma4` configuration; the old second install of +Transformers 5.5.1 is no longer needed. The startup script verifies the pins. The CUDA compiler pins keep FlashInfer JIT on CUDA 13.0. The unpinned CUDA compiler packages can otherwise resolve to a newer 13.x toolchain than the 580 @@ -40,7 +42,8 @@ Never pass a token on the command line and never enable shell tracing with nvidia-smi ``` -The official `vllm==0.22.0` wheel uses CUDA 13, so its NVIDIA driver must be +The official [vLLM 0.31.0 PyPI wheel](https://github.com/vllm-project/vllm/releases/tag/v0.31.0) +uses CUDA 13, so its NVIDIA driver must be 580 or newer. Driver installation and reboot are host-administration tasks and must be completed before this runbook when the server has an older driver. @@ -59,7 +62,8 @@ export CUDA_VISIBLE_DEVICES=0 The script downloads the pinned Hugging Face revision `6c71a32f4d624cadfd9fce9d10240d8068e53456`, records only sanitized metadata -(including the effective download endpoint), and starts vLLM 0.22.0 on +(including the effective download endpoint and installed package versions), +and starts vLLM 0.31.0 on `127.0.0.1:8000`. The private snapshot path stays out of public artifacts. Wait for the server to report that it is ready before continuing. diff --git a/cases/domux-4057-rtx3090-eval/prepare_and_serve.sh b/cases/domux-4057-rtx3090-eval/prepare_and_serve.sh index ee610d9..0ddc757 100644 --- a/cases/domux-4057-rtx3090-eval/prepare_and_serve.sh +++ b/cases/domux-4057-rtx3090-eval/prepare_and_serve.sh @@ -14,7 +14,7 @@ for command_name in python hf nvidia-smi; do fi done -if ! python -c "import importlib.metadata as m, transformers, vllm; assert vllm.__version__ == '0.22.0', vllm.__version__; assert transformers.__version__ == '5.5.1', transformers.__version__; assert m.version('nvidia-cuda-nvcc') == '13.0.88', m.version('nvidia-cuda-nvcc')"; then +if ! python -c "import importlib.metadata as m, transformers, vllm; assert vllm.__version__ == '0.31.0', vllm.__version__; assert transformers.__version__ == '5.10.4', transformers.__version__; assert m.version('nvidia-cuda-nvcc') == '13.0.88', m.version('nvidia-cuda-nvcc')"; then echo "ERROR: activate the pinned vLLM, Transformers, and CUDA 13.0 environment from RUNBOOK.md" >&2 exit 1 fi @@ -37,7 +37,7 @@ export PATH="${cuda_home}/bin:${PATH}" driver_version="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n 1 | tr -d '[:space:]')" driver_major="${driver_version%%.*}" if [[ ! "${driver_major}" =~ ^[0-9]+$ ]] || (( driver_major < 580 )); then - echo "ERROR: the official vllm==0.22.0 wheel uses CUDA 13 and requires NVIDIA driver 580 or newer; found ${driver_version}" >&2 + echo "ERROR: the official vllm==0.31.0 wheel uses CUDA 13 and requires NVIDIA driver 580 or newer; found ${driver_version}" >&2 exit 1 fi @@ -75,6 +75,7 @@ import json import os import sys from datetime import datetime, timezone +from importlib.metadata import version from pathlib import Path payload = { @@ -85,9 +86,9 @@ payload = { "snapshot_bytes": int(os.environ["DOMUX_SNAPSHOT_BYTES"]), "download_source": os.environ["DOMUX_DOWNLOAD_ENDPOINT"], "artifact_type": "full BF16 snapshot", - "runtime": "vllm-0.22.0", - "transformers": "5.5.1", - "cuda_compiler": "13.0.88", + "runtime": f"vllm-{version('vllm')}", + "transformers": version("transformers"), + "cuda_compiler": version("nvidia-cuda-nvcc"), "precision": "bfloat16", "visible_gpu": os.environ.get("CUDA_VISIBLE_DEVICES", "0"), } diff --git a/cases/domux-4057-rtx3090-eval/render_case.py b/cases/domux-4057-rtx3090-eval/render_case.py index 544ddc9..bee22f1 100644 --- a/cases/domux-4057-rtx3090-eval/render_case.py +++ b/cases/domux-4057-rtx3090-eval/render_case.py @@ -153,6 +153,10 @@ def build_body( ) return f"""# Domux 4,057-sample open evaluation on RTX 3090 +The results below record the {download['runtime']} experiment. New installations +use the security-updated runtime in [RUNBOOK.md](RUNBOOK.md); rerun the GPU +evaluation before replacing this evidence with results from another runtime. + ## Task / 真实任务 I reproduced the complete public Domux evaluation on a consumer RTX 3090, then @@ -174,7 +178,7 @@ def build_body( ## Setup / 环境 -- Runtime: vLLM 0.22.0, Transformers {download['transformers']}, OpenAI-compatible `/v1/chat/completions` +- Runtime: {download['runtime']}, Transformers {download['transformers']}, OpenAI-compatible `/v1/chat/completions` - CUDA compiler used for FlashInfer JIT: {download['cuda_compiler']}; the driver and CUDA runtime are recorded below - Hardware: one NVIDIA GeForce RTX 3090 24 GB; the second installed RTX 3090 was not used - Precision: BF16 @@ -245,9 +249,8 @@ def build_body( - Pin the 40-character Hugging Face revision before downloading and record it again in the case frontmatter. - The official evaluator defaults to concurrency 20; its latency is not directly comparable with sequential latency. -- On this host, vLLM 0.22.0 installed a CUDA 13 runtime, which required upgrading the NVIDIA driver to the 580 series. -- The dependency resolver selected Transformers 4.x because xgrammar 0.2.4 declares `transformers<5`, but that version did not recognize the checkpoint's `gemma4` architecture. Transformers 5.5.1 was installed explicitly. Structured decoding was not used in this experiment. -- FlashInfer JIT initially found the host CUDA 11.5 compiler. The run scripts point `CUDA_HOME` and `CUDACXX` to the environment's pinned CUDA 13.0 compiler instead. +- Follow [RUNBOOK.md](RUNBOOK.md) for the current runtime, Transformers, and CUDA requirements. The versions recorded above belong to this measured run. +- The run scripts point `CUDA_HOME` and `CUDACXX` to the environment's pinned CUDA 13.0 compiler for FlashInfer JIT. - Preserve raw per-sample output locally for audit, but publish only the aggregate metrics and representative examples. - Run `python scripts/validate_cases.py` before opening the PR. """ @@ -295,7 +298,7 @@ def main() -> int: date: {args.date} category: evaluation testedRevision: {download['revision']} -runtime: vllm-0.22.0 +runtime: {download['runtime']} hardware: NVIDIA-GeForce-RTX-3090-24GB-Linux-single-GPU downloadSource: huggingface channels: diff --git a/cases/domux-4057-rtx3090-eval/render_preview.py b/cases/domux-4057-rtx3090-eval/render_preview.py index 2ddcb05..5197e86 100644 --- a/cases/domux-4057-rtx3090-eval/render_preview.py +++ b/cases/domux-4057-rtx3090-eval/render_preview.py @@ -86,7 +86,7 @@ def main() -> int: strong, ), ("", "#f8fafc", small), - ("RTX 3090 24 GB | BF16 | vLLM 0.22.0 | temperature 0", "#94a3b8", small), + (f"RTX 3090 24 GB | BF16 | {download['runtime']} | temperature 0", "#94a3b8", small), ("Rendered from committed sanitized JSON artifacts; this is not a raw terminal capture.", "#64748b", small), ] ) diff --git a/cases/domux-4057-rtx3090-eval/requirements-case.txt b/cases/domux-4057-rtx3090-eval/requirements-case.txt index ec77d13..45e7dfd 100644 --- a/cases/domux-4057-rtx3090-eval/requirements-case.txt +++ b/cases/domux-4057-rtx3090-eval/requirements-case.txt @@ -1,4 +1,8 @@ huggingface_hub Pillow>=10 requests>=2.31,<3 -vllm==0.22.0 +vllm==0.31.0 +transformers==5.10.4 +nvidia-cuda-nvcc==13.0.88 +nvidia-cuda-crt==13.0.88 +nvidia-nvvm==13.0.88 From 686e066a04485ce968ee5516ebbfe109c4482d60 Mon Sep 17 00:00:00 2001 From: FenjuFu <92919259+FenjuFu@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:33:24 +0800 Subject: [PATCH 2/2] fix(cases): reject mixed runtime and benchmark evidence Signed-off-by: FenjuFu <92919259+FenjuFu@users.noreply.github.com> --- .github/workflows/cases-lint.yml | 2 + cases/domux-4057-rtx3090-eval/RUNBOOK.md | 6 + cases/domux-4057-rtx3090-eval/render_case.py | 45 +++++++- .../domux-4057-rtx3090-eval/render_preview.py | 17 ++- .../test_case_tools.py | 105 ++++++++++++++++-- 5 files changed, 163 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cases-lint.yml b/.github/workflows/cases-lint.yml index 40fe104..2d0f590 100644 --- a/.github/workflows/cases-lint.yml +++ b/.github/workflows/cases-lint.yml @@ -35,3 +35,5 @@ jobs: python -m py_compile scripts/validate_cases.py python scripts/validate_cases.py --self-test python scripts/validate_cases.py + - name: Validate RTX 3090 case tools (offline; no GPU required) + run: python -m unittest discover -s cases/domux-4057-rtx3090-eval -p 'test_*.py' -v diff --git a/cases/domux-4057-rtx3090-eval/RUNBOOK.md b/cases/domux-4057-rtx3090-eval/RUNBOOK.md index f1fd57d..24c407c 100644 --- a/cases/domux-4057-rtx3090-eval/RUNBOOK.md +++ b/cases/domux-4057-rtx3090-eval/RUNBOOK.md @@ -100,6 +100,12 @@ Raw JSONL and raw console logs remain ignored under `artifacts/raw/`. Aggregated JSON, Markdown, environment information, and sanitized console excerpts are intended for review and commit. +Both renderers verify that the recorded runtime, Transformers, and CUDA compiler +match `environment.txt`, and that smoke, evaluation, and latency timestamps follow +the download metadata in pipeline order. If server startup or the pipeline stops +early, finish a new pipeline run before rendering; do not mix the new runtime +metadata with previously committed results or edit timestamps to bypass the gate. + ## 4. Review for privacy Before publishing, search the case directory for secrets, private addresses, diff --git a/cases/domux-4057-rtx3090-eval/render_case.py b/cases/domux-4057-rtx3090-eval/render_case.py index bee22f1..0c8d562 100644 --- a/cases/domux-4057-rtx3090-eval/render_case.py +++ b/cases/domux-4057-rtx3090-eval/render_case.py @@ -6,7 +6,7 @@ import argparse import json import re -from datetime import date +from datetime import date, datetime from pathlib import Path from typing import Any @@ -108,7 +108,8 @@ def failure_examples(analysis: dict[str, Any], count: int = 5) -> str: def validate_artifacts( - download: dict[str, Any], smoke: dict[str, Any], run: dict[str, Any], summary: dict[str, Any], latency: dict[str, Any] + download: dict[str, Any], smoke: dict[str, Any], run: dict[str, Any], summary: dict[str, Any], latency: dict[str, Any], + environment: str, ) -> None: revision = str(download.get("revision", "")) if not REVISION_RE.fullmatch(revision): @@ -124,6 +125,44 @@ def validate_artifacts( if not latency.get("overall"): raise ValueError("latency benchmark summary is empty") + # Starting a server writes new download metadata before the evaluation runs. + # Never attach that runtime to results left over from an earlier run. + environment_fields = dict(line.split("=", 1) for line in environment.splitlines() if "=" in line) + for metadata_key, environment_key, prefix in ( + ("runtime", "vllm", "vllm-"), + ("transformers", "transformers", ""), + ("cuda_compiler", "nvidia-cuda-nvcc", ""), + ): + recorded = environment_fields.get(environment_key) + if not recorded or download.get(metadata_key) != prefix + recorded: + raise ValueError(f"download metadata {metadata_key} does not match environment.txt; rerun the pipeline") + + def timestamp(value: Any, label: str) -> datetime: + try: + parsed = datetime.fromisoformat(str(value)) + except ValueError as exc: + raise ValueError(f"missing or invalid {label}; rerun the pipeline") from exc + if parsed.tzinfo is None: + raise ValueError(f"{label} must include a timezone") + return parsed + + prepared = timestamp(download.get("created_at_utc"), "download timestamp") + collected = timestamp(environment_fields.get("collected_at_utc"), "environment timestamp") + if collected < prepared: + raise ValueError("environment.txt predates download metadata; rerun the pipeline") + previous = prepared + for label, value in ( + ("smoke result", smoke.get("created_at_utc")), + ("evaluation start", run.get("started_at_utc")), + ("evaluation finish", run.get("finished_at_utc")), + ("latency start", latency.get("started_at_utc")), + ("latency finish", latency.get("finished_at_utc")), + ): + current = timestamp(value, f"{label} timestamp") + if current < previous: + raise ValueError(f"{label} predates the current run or an earlier pipeline stage; rerun the pipeline") + previous = current + def build_body( *, @@ -280,7 +319,7 @@ def main() -> int: line.rstrip() for line in redact_text(environment_path.read_text(encoding="utf-8", errors="replace")).splitlines() ) - validate_artifacts(download, smoke, run, summary, latency) + validate_artifacts(download, smoke, run, summary, latency, environment) body = build_body( discussion_url=args.discussion_url or "", download=download, diff --git a/cases/domux-4057-rtx3090-eval/render_preview.py b/cases/domux-4057-rtx3090-eval/render_preview.py index 5197e86..55c48f7 100644 --- a/cases/domux-4057-rtx3090-eval/render_preview.py +++ b/cases/domux-4057-rtx3090-eval/render_preview.py @@ -5,8 +5,12 @@ import json from pathlib import Path +from typing import TYPE_CHECKING -from PIL import Image, ImageDraw, ImageFont +from render_case import validate_artifacts + +if TYPE_CHECKING: + from PIL import ImageFont CASE_DIR = Path(__file__).resolve().parent @@ -18,6 +22,8 @@ def load_json(name: str) -> dict: def font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont.ImageFont: + from PIL import ImageFont + names = ( "C:/Windows/Fonts/consolab.ttf" if bold else "C:/Windows/Fonts/consola.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSansMono-Bold.ttf" @@ -33,8 +39,15 @@ def font(size: int, *, bold: bool = False) -> ImageFont.FreeTypeFont | ImageFont def main() -> int: summary = load_json("eval_summary.json") run = load_json("eval_run_metadata.json") - latency = load_json("latency_summary.json")["overall"] + latency_artifact = load_json("latency_summary.json") download = load_json("download_metadata.json") + smoke = load_json("smoke_results.json") + environment = (ARTIFACTS_DIR / "environment.txt").read_text(encoding="utf-8") + validate_artifacts(download, smoke, run, summary, latency_artifact, environment) + + from PIL import Image, ImageDraw + + latency = latency_artifact["overall"] image = Image.new("RGB", (1600, 1200), "#0b1020") draw = ImageDraw.Draw(image) diff --git a/cases/domux-4057-rtx3090-eval/test_case_tools.py b/cases/domux-4057-rtx3090-eval/test_case_tools.py index b21aa94..0a5016a 100644 --- a/cases/domux-4057-rtx3090-eval/test_case_tools.py +++ b/cases/domux-4057-rtx3090-eval/test_case_tools.py @@ -2,10 +2,13 @@ from __future__ import annotations +import json import sys import unittest from collections import Counter +from datetime import datetime, timedelta from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -18,6 +21,8 @@ redact_text, stratified_samples, ) +import render_case # noqa: E402 +import render_preview # noqa: E402 from render_case import validate_artifacts # noqa: E402 @@ -115,16 +120,102 @@ def test_rejects_incomplete_real_run(self) -> None: summary = {"overall": {"total": 4057}} latency = {"errors": [], "overall": {"count": 300}} with self.assertRaisesRegex(ValueError, "4,057"): - validate_artifacts(download, smoke, run, summary, latency) + validate_artifacts(download, smoke, run, summary, latency, "") - def test_accepts_complete_artifact_gate(self) -> None: + +class RenderProvenanceTest(unittest.TestCase): + def setUp(self) -> None: + self.artifacts = { + name: json.loads((render_case.ARTIFACTS_DIR / name).read_text(encoding="utf-8")) + for name in ( + "download_metadata.json", "smoke_results.json", "eval_run_metadata.json", + "eval_summary.json", "failure_analysis.json", "latency_summary.json", + ) + } + self.environment = (render_case.ARTIFACTS_DIR / "environment.txt").read_text(encoding="utf-8") + + def validate(self) -> None: validate_artifacts( - {"revision": "a" * 40}, - {"all_passed": True}, - {"completed_samples": 4057, "api_error_count": 0}, - {"overall": {"total": 4057}}, - {"errors": [], "overall": {"count": 300}}, + *(self.artifacts[name] for name in ( + "download_metadata.json", "smoke_results.json", "eval_run_metadata.json", + "eval_summary.json", "latency_summary.json", + )), + self.environment, + ) + + def upgrade_metadata(self) -> None: + self.artifacts["download_metadata.json"].update( + runtime="vllm-0.31.0", transformers="5.10.4", created_at_utc="2026-10-08T03:09:29+00:00", + ) + + def upgrade_environment(self) -> None: + self.environment = self.environment.replace("vllm=0.22.0", "vllm=0.31.0").replace( + "transformers=5.5.1", "transformers=5.10.4", + ).replace("2026-08-25", "2026-10-08") + + def assert_renderers_reject(self, message: str) -> None: + for renderer in (render_case, render_preview): + with self.subTest(renderer=renderer.__name__), \ + patch.object(renderer, "load_json", side_effect=self.artifacts.__getitem__), \ + patch.object(Path, "read_text", return_value=self.environment), \ + patch.object(sys, "argv", ["render_case.py"]), \ + patch.object(render_case, "write_text_lf") as write: + with self.assertRaisesRegex(ValueError, message): + renderer.main() + write.assert_not_called() + + def test_accepts_committed_historical_artifacts(self) -> None: + self.validate() + + def test_accepts_consistent_new_run(self) -> None: + self.upgrade_metadata() + self.upgrade_environment() + for name in ("smoke_results.json", "eval_run_metadata.json", "latency_summary.json"): + for key in ("created_at_utc", "started_at_utc", "finished_at_utc"): + if key in self.artifacts[name]: + original = datetime.fromisoformat(self.artifacts[name][key]) + self.artifacts[name][key] = (original + timedelta(days=44)).isoformat() + self.validate() + + def test_new_download_metadata_cannot_relabel_historical_results(self) -> None: + self.upgrade_metadata() + self.assert_renderers_reject("runtime does not match environment") + + def test_new_environment_cannot_relabel_historical_results(self) -> None: + self.upgrade_metadata() + self.upgrade_environment() + self.assert_renderers_reject("smoke result predates") + + def test_partial_pipeline_cannot_reuse_old_evaluation(self) -> None: + self.upgrade_metadata() + self.upgrade_environment() + self.artifacts["smoke_results.json"]["created_at_utc"] = "2026-10-08T03:18:34+00:00" + self.assert_renderers_reject("evaluation start predates") + + def test_partial_pipeline_cannot_reuse_old_latency(self) -> None: + self.upgrade_metadata() + self.upgrade_environment() + self.artifacts["smoke_results.json"]["created_at_utc"] = "2026-10-08T03:18:34+00:00" + self.artifacts["eval_run_metadata.json"].update( + started_at_utc="2026-10-08T03:18:35+00:00", finished_at_utc="2026-10-08T03:19:31+00:00", ) + self.assert_renderers_reject("latency start predates") + + def test_rejects_missing_or_timezone_free_timestamps(self) -> None: + for value in (None, "2026-08-25T03:18:34"): + with self.subTest(timestamp=value): + self.artifacts["smoke_results.json"]["created_at_utc"] = value + with self.assertRaises(ValueError): + self.validate() + + def test_rejects_mismatched_transformers_or_cuda_versions(self) -> None: + for field in ("transformers", "cuda_compiler"): + with self.subTest(field=field): + original = self.artifacts["download_metadata.json"][field] + self.artifacts["download_metadata.json"][field] = "0.0.0" + with self.assertRaisesRegex(ValueError, f"{field} does not match environment"): + self.validate() + self.artifacts["download_metadata.json"][field] = original if __name__ == "__main__":