fix: cache custom /v1/models lookups by bare model id and memoize misses

cheetahclaws/providers.py

@@ -501,8 +501,14 @@ def _fetch_custom_model_limit(base_url: str, model: str, api_key: str) -> int |
     128000 default.
     """
     cache = _custom_ctx_cache.setdefault(base_url, {})
-    if model in cache:
-        return cache[model]
+    # Gateway routes keep the provider prefix ("custom/tools-pool") while the
+    # endpoint lists bare ids ("tools-pool"), so look the cache up by the bare
+    # id and remember a miss (0) — otherwise a model the endpoint does not
+    # describe is asked about before every call (184 GETs for 47 completions
+    # in one long session on 3.5.87, #183).
+    key = bare_model(model)
+    if key in cache:
+        return cache[key] or None
     try:
         url = base_url.rstrip("/") + "/models"
         req = urllib.request.Request(
@@ -515,7 +521,12 @@ def _fetch_custom_model_limit(base_url: str, model: str, api_key: str) -> int |
             limit = entry.get("max_model_len") or entry.get("context_window")
             if limit:
                 cache[mid] = int(limit)
-        result = cache.get(model)
+        result = cache.get(key)
+        if result is None:
+            # Remember the miss so /v1/models is not re-fetched before every
+            # call for a model the endpoint does not describe. 0 stands for
+            # "unknown" and is returned as None by the cache-hit branch above.
+            cache[key] = 0
         # Backfill provider-level default with the most conservative value seen.
         # Compaction reads PROVIDERS[provider]['context_limit'] without knowing
         # base_url, so this makes the threshold see the real limit.

tests/test_custom_provider_cache.py

@@ -0,0 +1,92 @@
+"""Tests for _fetch_custom_model_limit caching (#183).
+
+Regression: the cache was keyed on the full model string ("custom/tools-pool")
+while the /v1/models response stores bare ids ("tools-pool"), so the lookup
+missed on every call and neither caller remembered a miss — GET /v1/models
+fired before each chat completion (184 GETs for 47 completions in one long
+session on 3.5.87).
+"""
+from __future__ import annotations
+
+import json
+
+import cheetahclaws.providers as _providers
+from cheetahclaws.providers import _fetch_custom_model_limit
+
+
+def _fake_models(data_models, calls):
+    """urlopen stub returning a fixed /v1/models payload, counting requests."""
+
+    def fake_urlopen(req, timeout=None):
+        calls.append(req.full_url)
+        payload = {"data": [{"id": m, "max_model_len": l} for m, l in data_models]}
+
+        class _Resp:
+            def read(self):
+                return json.dumps(payload).encode()
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *exc):
+                return False
+
+        return _Resp()
+
+    return fake_urlopen
+
+
+def test_fetch_custom_model_limit_caches_by_bare_model(monkeypatch):
+    _providers._custom_ctx_cache.clear()
+    calls: list[str] = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_models(
+        [("tools-pool", 131072)], calls,
+    ))
+
+    lim1 = _fetch_custom_model_limit("http://x", "custom/tools-pool", "k")
+    lim2 = _fetch_custom_model_limit("http://x", "custom/tools-pool", "k")
+    assert lim1 == lim2 == 131072
+    assert len(calls) == 1, "second lookup must hit the cache, not /v1/models"
+
+
+def test_fetch_custom_model_limit_bare_and_prefixed_interop(monkeypatch):
+    """A bare-id lookup and a prefixed lookup of the same model share one request."""
+    _providers._custom_ctx_cache.clear()
+    calls: list[str] = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_models(
+        [("tools-pool", 65536)], calls,
+    ))
+
+    bare = _fetch_custom_model_limit("http://y", "tools-pool", "k")
+    prefixed = _fetch_custom_model_limit("http://y", "custom/tools-pool", "k")
+    assert bare == prefixed == 65536
+    assert len(calls) == 1
+
+
+def test_fetch_custom_model_limit_memoizes_miss(monkeypatch):
+    """A model the endpoint does not list is asked about once, not per call."""
+    _providers._custom_ctx_cache.clear()
+    calls: list[str] = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_models([], calls))
+
+    for _ in range(3):
+        assert _fetch_custom_model_limit("http://z", "custom/unknown", "k") is None
+    assert len(calls) == 1, "a miss must be remembered so /v1/models is not re-fetched"
+
+
+def test_fetch_custom_model_limit_network_failure_not_cached(monkeypatch):
+    """Transient failures are not memoized — a later success must still work."""
+    _providers._custom_ctx_cache.clear()
+
+    def boom(req, timeout=None):
+        raise OSError("connection refused")
+
+    monkeypatch.setattr("urllib.request.urlopen", boom)
+    assert _fetch_custom_model_limit("http://w", "custom/m", "k") is None
+
+    calls: list[str] = []
+    monkeypatch.setattr("urllib.request.urlopen", _fake_models(
+        [("m", 4096)], calls,
+    ))
+    assert _fetch_custom_model_limit("http://w", "custom/m", "k") == 4096
+    assert len(calls) == 1