npu 910b4: ModelArts bring-up fixes (visible NPU id, CANN 8.x, Ray NPU visibility, pin waiver)

tests/test_npu_910b4_prep.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import os
 import sys
 from types import SimpleNamespace
 
@@ -226,3 +227,65 @@ def test_threshold_key_separates_npu_from_gpu():
 
     assert _threshold_card({"device_family": "ascend", "npu_smi": NPU_SMI_INFO}) == "910B4"
     assert _threshold_card({"device_family": "nvidia", "nvidia_smi": "NVIDIA H100 80GB"}) == "H100"
+
+
+# --- S20 ModelArts bring-up: the container sees one card with a non-zero ID --
+NPU_SMI_LIST_ID4 = "\tTotal Count                    : 1\n\n\tNPU ID                         : 4\n\tChip Count                     : 1\n"
+
+
+def test_first_npu_id_reads_npu_smi_list(monkeypatch):
+    class _Res:
+        stdout = NPU_SMI_LIST_ID4
+
+    monkeypatch.setattr(catalog.subprocess, "run", lambda *a, **k: _Res())
+    assert catalog._first_npu_id() == "4"
+
+
+def test_first_npu_id_falls_back_to_visible_devices(monkeypatch):
+    def _missing(*_a, **_k):
+        raise OSError("npu-smi: not found")
+
+    monkeypatch.setattr(catalog.subprocess, "run", _missing)
+    monkeypatch.setenv("ASCEND_VISIBLE_DEVICES", "6,7")
+    assert catalog._first_npu_id() == "6"
+    monkeypatch.delenv("ASCEND_VISIBLE_DEVICES")
+    assert catalog._first_npu_id() == "0"
+
+
+def test_board_query_uses_the_visible_npu_id(monkeypatch):
+    calls = []
+
+    class _Res:
+        def __init__(self, out):
+            self.stdout = out
+
+    def _run(cmd, **_k):
+        calls.append(cmd)
+        if cmd[:3] == ["npu-smi", "info", "-l"]:
+            return _Res(NPU_SMI_LIST_ID4)
+        return _Res("\tSoftware Version               : 25.5.1\n")
+
+    monkeypatch.setattr(catalog.subprocess, "run", _run)
+    out = catalog._npu_runtime_versions()
+    assert ["npu-smi", "info", "-t", "board", "-i", "4"] in calls
+    assert out["npu_driver_version"] == "25.5.1"
+
+
+def test_unpinned_waiver_env_name_is_stable():
+    from yeto.rl.adapters.verl import island_entry
+
+    assert island_entry.ALLOW_UNPINNED_ENV == "YETO_VERL_ALLOW_UNPINNED"
+
+
+def test_verl_main_keeps_npus_visible_only_on_ascend(monkeypatch):
+    pytest.importorskip("ray")
+    from yeto.hw import catalog as cat
+    from yeto.rl.adapters.verl import verl_main
+
+    monkeypatch.delenv(verl_main.NPU_NOSET_ENV, raising=False)
+    monkeypatch.setattr(cat, "device_family", lambda *_a: "nvidia")
+    verl_main._keep_npu_visible_in_cpu_actors()
+    assert verl_main.NPU_NOSET_ENV not in os.environ
+    monkeypatch.setattr(cat, "device_family", lambda *_a: "ascend")
+    verl_main._keep_npu_visible_in_cpu_actors()
+    assert os.environ[verl_main.NPU_NOSET_ENV] == "1"

yeto/hw/catalog.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 import os
+import platform
 import re
 import subprocess
 from typing import Mapping
@@ -120,12 +121,31 @@ def check_compat_group(local: str | None, peer: str | None) -> None:
         raise CompatGroupMismatch(reason)
 
 
+def _first_npu_id() -> str:
+    """NPU ID of the first card ``npu-smi`` can see.
+
+    A container may expose only a physical card such as ID 4 (ModelArts
+    notebooks do); ``npu-smi info -t board -i 0`` then fails. ``npu-smi info -l``
+    lists the visible IDs. Falls back to ``ASCEND_VISIBLE_DEVICES``, then "0".
+    """
+    try:
+        res = subprocess.run(["npu-smi", "info", "-l"], capture_output=True, text=True, timeout=10)
+        for line in res.stdout.splitlines():
+            key, _, value = line.partition(":")
+            if key.strip() == "NPU ID" and value.strip().isdigit():
+                return value.strip()
+    except (OSError, subprocess.SubprocessError):
+        pass
+    visible = os.environ.get("ASCEND_VISIBLE_DEVICES", "").split(",")[0].strip()
+    return visible if visible.isdigit() else "0"
+
+
 def _npu_runtime_versions() -> dict[str, str | None]:
     """Ascend driver, CANN and torch_npu versions (best effort; rl-verl-backend 3.6)."""
     out: dict[str, str | None] = {"npu_driver_version": None, "cann_version": None,
                                   "torch_npu_version": None}
     try:
-        res = subprocess.run(["npu-smi", "info", "-t", "board", "-i", "0"],
+        res = subprocess.run(["npu-smi", "info", "-t", "board", "-i", _first_npu_id()],
                              capture_output=True, text=True, timeout=10)
         for line in res.stdout.splitlines():
             if "Software Version" in line or "Driver Version" in line:
@@ -134,7 +154,10 @@ def _npu_runtime_versions() -> dict[str, str | None]:
     except (OSError, subprocess.SubprocessError):
         pass
     for path in ("/usr/local/Ascend/ascend-toolkit/latest/version.cfg",
-                 "/usr/local/Ascend/ascend-toolkit/latest/version.info"):
+                 "/usr/local/Ascend/ascend-toolkit/latest/version.info",
+                 # CANN 8.x layout (seen on ModelArts snt9b images, CANN 8.5.1).
+                 f"/usr/local/Ascend/ascend-toolkit/latest/{platform.machine()}-linux/"
+                 "ascend_toolkit_install.info"):
         try:
             with open(path) as fh:
                 text = fh.read()

yeto/rl/adapters/verl/island_entry.py

@@ -31,6 +31,9 @@
 from .reward_fn import function_for
 
 OUTPUT = Path(os.path.expanduser("~/yeto-output"))
+# Bring-up only: "1" turns version/commit pin mismatches into a recorded
+# warning (manifest "problems_waived") instead of exit 2. Never set in production.
+ALLOW_UNPINNED_ENV = "YETO_VERL_ALLOW_UNPINNED"
 TEST_FILE = Path(os.path.expanduser("~/.yeto-verl-test.json"))  # TEST ONLY switches
 
 
@@ -255,6 +258,13 @@ def main(argv=None) -> int:
                    f" || {sys.executable} -m pip freeze > {OUTPUT}/verl-pip-freeze-{lid}.txt 2>/dev/null",
                    shell=True)
     print(f"[yeto-verl] island {lid} runtime {json.dumps(manifest)}", flush=True)
+    if manifest["problems"] and os.environ.get(ALLOW_UNPINNED_ENV) == "1":
+        # Bring-up only (new card, vendor image): run anyway, keep the evidence.
+        manifest["problems_waived"] = manifest.pop("problems")
+        manifest["problems"] = []
+        (OUTPUT / f"verl-runtime-{lid}.json").write_text(json.dumps(manifest, indent=1))
+        print(f"[yeto-verl] runtime check WAIVED by {ALLOW_UNPINNED_ENV}=1: "
+              f"{manifest['problems_waived']}", file=sys.stderr, flush=True)
     if manifest["problems"]:
         print(f"[yeto-verl] runtime check failed: {manifest['problems']}", file=sys.stderr, flush=True)
         return 2

yeto/rl/adapters/verl/verl_main.py

@@ -72,12 +72,31 @@ def _runtime_env() -> dict:
     return env
 
 
+NPU_NOSET_ENV = "RAY_EXPERIMENTAL_NOSET_ASCEND_RT_VISIBLE_DEVICES"
+
+
+def _keep_npu_visible_in_cpu_actors() -> None:
+    """On an Ascend node, stop Ray from hiding the NPUs from CPU-only actors.
+
+    Ray sets ``ASCEND_RT_VISIBLE_DEVICES=""`` in actors that request no NPU,
+    such as ``YetoTaskRunner``. verl then reads ``is_npu_available=False`` there,
+    and ``verl.third_party.vllm`` refuses the vLLM-Ascend version
+    (S20 ModelArts 910B4 run, runY_try2). Only set on an NPU node; the
+    caller's own value wins.
+    """
+    from yeto.hw.catalog import device_family
+
+    if device_family() == "ascend":
+        os.environ.setdefault(NPU_NOSET_ENV, "1")
+
+
 def _plan_and_ray() -> dict:
     from yeto.island_credential_guard import check_island_credentials
 
     check_island_credentials()  # secret-handling-hardening D4 (driver, both paths)
     plan = json.loads(open(os.environ[PLAN_ENV]).read())
     if not ray.is_initialized():
+        _keep_npu_visible_in_cpu_actors()
         address = os.environ.get("RAY_ADDRESS") or None
         ray.init(address=address, runtime_env=_runtime_env(), namespace="yeto-verl")
     return plan