diff --git a/cheetahclaws/providers.py b/cheetahclaws/providers.py index c14efae..03a65da 100644 --- a/cheetahclaws/providers.py +++ b/cheetahclaws/providers.py @@ -501,8 +501,14 @@ def _fetch_custom_model_limit(base_url: str, model: str, api_key: str) -> int | 128000 default. """ cache = _custom_ctx_cache.setdefault(base_url, {}) - if model in cache: - return cache[model] + # Gateway routes keep the provider prefix ("custom/tools-pool") while the + # endpoint lists bare ids ("tools-pool"), so look the cache up by the bare + # id and remember a miss (0) — otherwise a model the endpoint does not + # describe is asked about before every call (184 GETs for 47 completions + # in one long session on 3.5.87, #183). + key = bare_model(model) + if key in cache: + return cache[key] or None try: url = base_url.rstrip("/") + "/models" req = urllib.request.Request( @@ -515,7 +521,12 @@ def _fetch_custom_model_limit(base_url: str, model: str, api_key: str) -> int | limit = entry.get("max_model_len") or entry.get("context_window") if limit: cache[mid] = int(limit) - result = cache.get(model) + result = cache.get(key) + if result is None: + # Remember the miss so /v1/models is not re-fetched before every + # call for a model the endpoint does not describe. 0 stands for + # "unknown" and is returned as None by the cache-hit branch above. + cache[key] = 0 # Backfill provider-level default with the most conservative value seen. # Compaction reads PROVIDERS[provider]['context_limit'] without knowing # base_url, so this makes the threshold see the real limit. diff --git a/tests/test_custom_provider_cache.py b/tests/test_custom_provider_cache.py new file mode 100644 index 0000000..ce444a8 --- /dev/null +++ b/tests/test_custom_provider_cache.py @@ -0,0 +1,92 @@ +"""Tests for _fetch_custom_model_limit caching (#183). + +Regression: the cache was keyed on the full model string ("custom/tools-pool") +while the /v1/models response stores bare ids ("tools-pool"), so the lookup +missed on every call and neither caller remembered a miss — GET /v1/models +fired before each chat completion (184 GETs for 47 completions in one long +session on 3.5.87). +""" +from __future__ import annotations + +import json + +import cheetahclaws.providers as _providers +from cheetahclaws.providers import _fetch_custom_model_limit + + +def _fake_models(data_models, calls): + """urlopen stub returning a fixed /v1/models payload, counting requests.""" + + def fake_urlopen(req, timeout=None): + calls.append(req.full_url) + payload = {"data": [{"id": m, "max_model_len": l} for m, l in data_models]} + + class _Resp: + def read(self): + return json.dumps(payload).encode() + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + return _Resp() + + return fake_urlopen + + +def test_fetch_custom_model_limit_caches_by_bare_model(monkeypatch): + _providers._custom_ctx_cache.clear() + calls: list[str] = [] + monkeypatch.setattr("urllib.request.urlopen", _fake_models( + [("tools-pool", 131072)], calls, + )) + + lim1 = _fetch_custom_model_limit("http://x", "custom/tools-pool", "k") + lim2 = _fetch_custom_model_limit("http://x", "custom/tools-pool", "k") + assert lim1 == lim2 == 131072 + assert len(calls) == 1, "second lookup must hit the cache, not /v1/models" + + +def test_fetch_custom_model_limit_bare_and_prefixed_interop(monkeypatch): + """A bare-id lookup and a prefixed lookup of the same model share one request.""" + _providers._custom_ctx_cache.clear() + calls: list[str] = [] + monkeypatch.setattr("urllib.request.urlopen", _fake_models( + [("tools-pool", 65536)], calls, + )) + + bare = _fetch_custom_model_limit("http://y", "tools-pool", "k") + prefixed = _fetch_custom_model_limit("http://y", "custom/tools-pool", "k") + assert bare == prefixed == 65536 + assert len(calls) == 1 + + +def test_fetch_custom_model_limit_memoizes_miss(monkeypatch): + """A model the endpoint does not list is asked about once, not per call.""" + _providers._custom_ctx_cache.clear() + calls: list[str] = [] + monkeypatch.setattr("urllib.request.urlopen", _fake_models([], calls)) + + for _ in range(3): + assert _fetch_custom_model_limit("http://z", "custom/unknown", "k") is None + assert len(calls) == 1, "a miss must be remembered so /v1/models is not re-fetched" + + +def test_fetch_custom_model_limit_network_failure_not_cached(monkeypatch): + """Transient failures are not memoized — a later success must still work.""" + _providers._custom_ctx_cache.clear() + + def boom(req, timeout=None): + raise OSError("connection refused") + + monkeypatch.setattr("urllib.request.urlopen", boom) + assert _fetch_custom_model_limit("http://w", "custom/m", "k") is None + + calls: list[str] = [] + monkeypatch.setattr("urllib.request.urlopen", _fake_models( + [("m", 4096)], calls, + )) + assert _fetch_custom_model_limit("http://w", "custom/m", "k") == 4096 + assert len(calls) == 1