From b9628b8b78e3f95cdfd89a67816a229c5b31c5d2 Mon Sep 17 00:00:00 2001 From: michael <78113554+chowhsu@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:38:23 +0000 Subject: [PATCH] npu 910b4: ModelArts bring-up fixes (visible NPU id, CANN 8.x version file, Ray NPU visibility, pin waiver) Found on a ModelArts notebook (1x910B4, CANN 8.5.1, verl fork acad9875): - npu-smi board query used ID 0; the container only sees physical ID 4. - CANN 8.x keeps its version in -linux/ascend_toolkit_install.info. - Ray hides NPUs from CPU-only actors; verl then refuses vLLM-Ascend in YetoTaskRunner. Set RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES=1 on Ascend nodes only. - YETO_VERL_ALLOW_UNPINNED=1 (bring-up only) records pin mismatches as problems_waived instead of exit 2. Co-Authored-By: Claude Opus 5.5 --- tests/test_npu_910b4_prep.py | 63 +++++++++++++++++++++++++++ yeto/hw/catalog.py | 27 +++++++++++- yeto/rl/adapters/verl/island_entry.py | 10 +++++ yeto/rl/adapters/verl/verl_main.py | 19 ++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) diff --git a/tests/test_npu_910b4_prep.py b/tests/test_npu_910b4_prep.py index 0b8b9cd4..622f578c 100644 --- a/tests/test_npu_910b4_prep.py +++ b/tests/test_npu_910b4_prep.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os import sys from types import SimpleNamespace @@ -226,3 +227,65 @@ def test_threshold_key_separates_npu_from_gpu(): assert _threshold_card({"device_family": "ascend", "npu_smi": NPU_SMI_INFO}) == "910B4" assert _threshold_card({"device_family": "nvidia", "nvidia_smi": "NVIDIA H100 80GB"}) == "H100" + + +# --- S20 ModelArts bring-up: the container sees one card with a non-zero ID -- +NPU_SMI_LIST_ID4 = "\tTotal Count : 1\n\n\tNPU ID : 4\n\tChip Count : 1\n" + + +def test_first_npu_id_reads_npu_smi_list(monkeypatch): + class _Res: + stdout = NPU_SMI_LIST_ID4 + + monkeypatch.setattr(catalog.subprocess, "run", lambda *a, **k: _Res()) + assert catalog._first_npu_id() == "4" + + +def test_first_npu_id_falls_back_to_visible_devices(monkeypatch): + def _missing(*_a, **_k): + raise OSError("npu-smi: not found") + + monkeypatch.setattr(catalog.subprocess, "run", _missing) + monkeypatch.setenv("ASCEND_VISIBLE_DEVICES", "6,7") + assert catalog._first_npu_id() == "6" + monkeypatch.delenv("ASCEND_VISIBLE_DEVICES") + assert catalog._first_npu_id() == "0" + + +def test_board_query_uses_the_visible_npu_id(monkeypatch): + calls = [] + + class _Res: + def __init__(self, out): + self.stdout = out + + def _run(cmd, **_k): + calls.append(cmd) + if cmd[:3] == ["npu-smi", "info", "-l"]: + return _Res(NPU_SMI_LIST_ID4) + return _Res("\tSoftware Version : 25.5.1\n") + + monkeypatch.setattr(catalog.subprocess, "run", _run) + out = catalog._npu_runtime_versions() + assert ["npu-smi", "info", "-t", "board", "-i", "4"] in calls + assert out["npu_driver_version"] == "25.5.1" + + +def test_unpinned_waiver_env_name_is_stable(): + from yeto.rl.adapters.verl import island_entry + + assert island_entry.ALLOW_UNPINNED_ENV == "YETO_VERL_ALLOW_UNPINNED" + + +def test_verl_main_keeps_npus_visible_only_on_ascend(monkeypatch): + pytest.importorskip("ray") + from yeto.hw import catalog as cat + from yeto.rl.adapters.verl import verl_main + + monkeypatch.delenv(verl_main.NPU_NOSET_ENV, raising=False) + monkeypatch.setattr(cat, "device_family", lambda *_a: "nvidia") + verl_main._keep_npu_visible_in_cpu_actors() + assert verl_main.NPU_NOSET_ENV not in os.environ + monkeypatch.setattr(cat, "device_family", lambda *_a: "ascend") + verl_main._keep_npu_visible_in_cpu_actors() + assert os.environ[verl_main.NPU_NOSET_ENV] == "1" diff --git a/yeto/hw/catalog.py b/yeto/hw/catalog.py index a18f0ef7..023999da 100644 --- a/yeto/hw/catalog.py +++ b/yeto/hw/catalog.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import platform import re import subprocess from typing import Mapping @@ -120,12 +121,31 @@ def check_compat_group(local: str | None, peer: str | None) -> None: raise CompatGroupMismatch(reason) +def _first_npu_id() -> str: + """NPU ID of the first card ``npu-smi`` can see. + + A container may expose only a physical card such as ID 4 (ModelArts + notebooks do); ``npu-smi info -t board -i 0`` then fails. ``npu-smi info -l`` + lists the visible IDs. Falls back to ``ASCEND_VISIBLE_DEVICES``, then "0". + """ + try: + res = subprocess.run(["npu-smi", "info", "-l"], capture_output=True, text=True, timeout=10) + for line in res.stdout.splitlines(): + key, _, value = line.partition(":") + if key.strip() == "NPU ID" and value.strip().isdigit(): + return value.strip() + except (OSError, subprocess.SubprocessError): + pass + visible = os.environ.get("ASCEND_VISIBLE_DEVICES", "").split(",")[0].strip() + return visible if visible.isdigit() else "0" + + def _npu_runtime_versions() -> dict[str, str | None]: """Ascend driver, CANN and torch_npu versions (best effort; rl-verl-backend 3.6).""" out: dict[str, str | None] = {"npu_driver_version": None, "cann_version": None, "torch_npu_version": None} try: - res = subprocess.run(["npu-smi", "info", "-t", "board", "-i", "0"], + res = subprocess.run(["npu-smi", "info", "-t", "board", "-i", _first_npu_id()], capture_output=True, text=True, timeout=10) for line in res.stdout.splitlines(): if "Software Version" in line or "Driver Version" in line: @@ -134,7 +154,10 @@ def _npu_runtime_versions() -> dict[str, str | None]: except (OSError, subprocess.SubprocessError): pass for path in ("/usr/local/Ascend/ascend-toolkit/latest/version.cfg", - "/usr/local/Ascend/ascend-toolkit/latest/version.info"): + "/usr/local/Ascend/ascend-toolkit/latest/version.info", + # CANN 8.x layout (seen on ModelArts snt9b images, CANN 8.5.1). + f"/usr/local/Ascend/ascend-toolkit/latest/{platform.machine()}-linux/" + "ascend_toolkit_install.info"): try: with open(path) as fh: text = fh.read() diff --git a/yeto/rl/adapters/verl/island_entry.py b/yeto/rl/adapters/verl/island_entry.py index e7b3fe1d..5ac2f8ba 100644 --- a/yeto/rl/adapters/verl/island_entry.py +++ b/yeto/rl/adapters/verl/island_entry.py @@ -31,6 +31,9 @@ from .reward_fn import function_for OUTPUT = Path(os.path.expanduser("~/yeto-output")) +# Bring-up only: "1" turns version/commit pin mismatches into a recorded +# warning (manifest "problems_waived") instead of exit 2. Never set in production. +ALLOW_UNPINNED_ENV = "YETO_VERL_ALLOW_UNPINNED" TEST_FILE = Path(os.path.expanduser("~/.yeto-verl-test.json")) # TEST ONLY switches @@ -255,6 +258,13 @@ def main(argv=None) -> int: f" || {sys.executable} -m pip freeze > {OUTPUT}/verl-pip-freeze-{lid}.txt 2>/dev/null", shell=True) print(f"[yeto-verl] island {lid} runtime {json.dumps(manifest)}", flush=True) + if manifest["problems"] and os.environ.get(ALLOW_UNPINNED_ENV) == "1": + # Bring-up only (new card, vendor image): run anyway, keep the evidence. + manifest["problems_waived"] = manifest.pop("problems") + manifest["problems"] = [] + (OUTPUT / f"verl-runtime-{lid}.json").write_text(json.dumps(manifest, indent=1)) + print(f"[yeto-verl] runtime check WAIVED by {ALLOW_UNPINNED_ENV}=1: " + f"{manifest['problems_waived']}", file=sys.stderr, flush=True) if manifest["problems"]: print(f"[yeto-verl] runtime check failed: {manifest['problems']}", file=sys.stderr, flush=True) return 2 diff --git a/yeto/rl/adapters/verl/verl_main.py b/yeto/rl/adapters/verl/verl_main.py index d012795c..ff0989c1 100644 --- a/yeto/rl/adapters/verl/verl_main.py +++ b/yeto/rl/adapters/verl/verl_main.py @@ -72,12 +72,31 @@ def _runtime_env() -> dict: return env +NPU_NOSET_ENV = "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES" + + +def _keep_npu_visible_in_cpu_actors() -> None: + """On an Ascend node, stop Ray from hiding the NPUs from CPU-only actors. + + Ray sets ``ASCEND_RT_VISIBLE_DEVICES=""`` in actors that request no NPU, + such as ``YetoTaskRunner``. verl then reads ``is_npu_available=False`` there, + and ``verl.third_party.vllm`` refuses the vLLM-Ascend version + (S20 ModelArts 910B4 run, runY_try2). Only set on an NPU node; the + caller's own value wins. + """ + from yeto.hw.catalog import device_family + + if device_family() == "ascend": + os.environ.setdefault(NPU_NOSET_ENV, "1") + + def _plan_and_ray() -> dict: from yeto.island_credential_guard import check_island_credentials check_island_credentials() # secret-handling-hardening D4 (driver, both paths) plan = json.loads(open(os.environ[PLAN_ENV]).read()) if not ray.is_initialized(): + _keep_npu_visible_in_cpu_actors() address = os.environ.get("RAY_ADDRESS") or None ray.init(address=address, runtime_env=_runtime_env(), namespace="yeto-verl") return plan