fix(apps): warn when gpu pins fall back to the static pool table

a failed catalog lookup was logged at debug and cached for the process, so pins silently lost exclusions the static table lacks. warn instead, and retry the catalog after a minute.

Author: justinwlin

runpod/apps/gpu.py

@@ -5,11 +5,11 @@
 configured. CpuInstanceType values are backend cpu flavor ids.
 """
 
-import functools
 import logging
 import os
+import time
 from enum import Enum
-from typing import Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 log = logging.getLogger(__name__)
 
@@ -216,18 +216,29 @@ def resolve_gpu_string(value: str) -> List[str]:
 CATALOG_URL = "https://api.runpod.io/v2/catalog/gpus"
 
 
-@functools.lru_cache(maxsize=1)
+# live catalog result, or when it last failed
+_catalog: Dict[str, Any] = {}
+_CATALOG_RETRY_SECONDS = 60.0
+
+
 def live_pool_members() -> Optional[Dict[str, Set[str]]]:
     """pool id -> device ids, from the platform's gpu catalog.
 
     the static POOLS_TO_TYPES table lags the platform (pools gain mig
     slices and new skus), so exact-device pinning reads the live
-    membership. returns None when the catalog is unreachable.
+    membership. returns None when the catalog is unreachable. a result
+    is kept for the process; a failure is retried after a minute.
     """
     import requests
 
     from .utils.network import api_key
 
+    if _catalog.get("members") is not None:
+        return _catalog["members"]
+    failed_at = _catalog.get("failed_at")
+    if failed_at is not None and time.monotonic() - failed_at < _CATALOG_RETRY_SECONDS:
+        return None
+
     url = os.environ.get("RUNPOD_GPU_CATALOG_URL", CATALOG_URL)
     try:
         resp = requests.get(
@@ -240,12 +251,23 @@ def live_pool_members() -> Optional[Dict[str, Set[str]]]:
         for gpu in gpus:
             if gpu.get("pool") and gpu.get("id"):
                 members.setdefault(gpu["pool"], set()).add(gpu["id"])
-        return members or None
+        if not members:
+            raise ValueError("catalog lists no pools")
+        _catalog["members"] = members
+        return members
     except Exception as exc:  # any failure falls back to the static table
-        log.debug("gpu catalog unavailable (%s); using the static pool table", exc)
+        _catalog["failed_at"] = time.monotonic()
+        log.warning(
+            "gpu catalog unavailable (%s); device pins use the static pool "
+            "table, which can miss newer devices in a pool",
+            exc,
+        )
         return None
 
 
+live_pool_members.cache_clear = _catalog.clear  # type: ignore[attr-defined]
+
+
 def _pool_members(pool: str) -> Set[str]:
     live = live_pool_members()
     if live is not None and pool in live:

tests/test_apps/test_app.py

@@ -427,6 +427,8 @@ def test_gpu_ids_falls_back_to_static_pools(self, monkeypatch):
     def test_live_pool_members_parses_catalog_and_fails_soft(self, monkeypatch):
         from unittest.mock import MagicMock
 
+        from runpod.apps import gpu as gpu_mod
+
         monkeypatch.setenv("RUNPOD_API_KEY", "k")
         resp = MagicMock()
         resp.json.return_value = {
@@ -441,8 +443,15 @@ def test_live_pool_members_parses_catalog_and_fails_soft(self, monkeypatch):
             assert fetch() == {"AMPERE_24": {"NVIDIA L4"}}
             assert get.call_args.kwargs["headers"]["Authorization"] == "Bearer k"
         fetch.cache_clear()
-        with patch("requests.get", side_effect=OSError("offline")):
+        with patch("requests.get", side_effect=OSError("offline")) as get:
             assert fetch() is None
+            # a failure is not retried on every lookup
+            assert fetch() is None
+            assert get.call_count == 1
+        # it is retried once the window passes
+        gpu_mod._catalog["failed_at"] -= gpu_mod._CATALOG_RETRY_SECONDS
+        with patch("requests.get", return_value=resp):
+            assert fetch() == {"AMPERE_24": {"NVIDIA L4"}}
         fetch.cache_clear()
 
     def test_gpu_ids_maps_devices_to_pools(self):