diff --git a/docs/operating.md b/docs/operating.md index be712a6..7460c60 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -144,10 +144,24 @@ export HOTOTEL_INSIGHTS_API_KEY= `modal run modal/hototel_llm.py` waits for the server and sends one test request with a tool attached, which is the quickest way to confirm tool -calling works after a model change. The first request after idle waits a -minute or two for the cold start; the very first ever boot also downloads +calling works after a model change. The very first ever boot also downloads the weights (~60GB) into the `hototel-hf-cache` volume. +**Cold starts.** The container stops ten minutes after the last request, and +Modal's endpoint answers `503 no upstreams available` while a new one boots +(about three minutes with the weights cached, and much longer when the volume +reads slowly -- the class allows thirty and loads the shards with vLLM's +parallel prefetch); that first refused request is what starts it. The server treats that 503 as "starting", not as a failure: +a successful sign-in starts the model booting in the background (people are +around; it still stops ten minutes after the last request), opening the +Insights page probes `/health` and wakes it too, `/api/insights/ +status` reports `modelState` (`ready`/`starting`), a chat during the boot is a +503 with `starting: true` that the page waits out (polling status every ten +seconds, up to twenty minutes) before sending the question again, and the +takeaways thread simply waits for `ready` before running. The model is +deliberately not kept on around the clock: it runs only after someone signs +in, for as long as they keep using it. + The alternative is the **Anthropic API**: leave `HOTOTEL_INSIGHTS_BASE_URL` unset and set `ANTHROPIC_API_KEY`; the model defaults to `claude-opus-5-5`. Either way `HOTOTEL_INSIGHTS_MODEL` overrides the model name (on Modal the diff --git a/modal/hototel_llm.py b/modal/hototel_llm.py index 57a7f23..31946fc 100644 --- a/modal/hototel_llm.py +++ b/modal/hototel_llm.py @@ -70,7 +70,12 @@ image=vllm_image, gpu=f"H100:{N_GPU}", scaledown_window=SCALEDOWN, - startup_timeout=15 * MINUTES, # the first boot downloads ~60GB of weights + # Weights come off the volume at boot. Measured: ~3 minutes on a good + # day, but the volume's 9P mount has read the 57GB at ~60MB/s, which is + # 17 minutes -- a 15-minute limit killed that boot and the page saw only + # 503s. The limit is generous so a slow read finishes rather than restarts; + # the prefetch flag below is what makes it fast. + startup_timeout=30 * MINUTES, volumes={ "/root/.cache/huggingface": hf_cache_vol, "/root/.cache/vllm": vllm_cache_vol, @@ -102,6 +107,11 @@ def start(self): # dozen query results -- fits comfortably; more only costs KV cache "--max-model-len", "32768", "--gpu-memory-utilization", "0.92", + # read the safetensors shards with parallel threads into the page + # cache before mapping them: the volume is a network filesystem + # (9P) whose single-stream reads are latency-bound, and vLLM only + # prefetches on its own for NFS/Lustre + "--safetensors-load-strategy", "prefetch", # tool calling: Qwen3 writes Hermes-style blocks "--enable-auto-tool-choice", "--tool-call-parser", "hermes", diff --git a/server/insights.py b/server/insights.py index b64be19..d173760 100644 --- a/server/insights.py +++ b/server/insights.py @@ -29,6 +29,8 @@ import sys import threading import time +import urllib.error +import urllib.request from datetime import datetime, timezone import core @@ -245,6 +247,20 @@ def validate_history(raw): return out +class ModelStarting(Exception): + """The model endpoint has no running container yet. Modal's endpoint + answers 503 while the GPU container cold-starts (a few minutes for this + model), and that first refused request is what starts it. Callers wait + or tell the page to come back, rather than report a failure.""" + + +# How long a cold start may take before it is treated as an outage. Weights +# are cached in a volume, so a boot is ~3 minutes when the volume reads well +# and has been seen to take 15+ when it does not; the Modal class allows 30. +MODEL_START_TIMEOUT = 20 * 60 +MODEL_HEALTH_TTL = 5 + + class InsightsEngine: """One per process. Holds the hotdata client, a per-database tool set and agent, and the takeaways cache. All public methods are thread-safe.""" @@ -266,6 +282,7 @@ def __init__(self, model=None, chat_effort="medium", analysis_effort="high", self._slots = threading.BoundedSemaphore(max_concurrent) self._takeaways = {} # database_id -> {"at", "data"|"error", ...} self._generating = set() # database_ids with a run in flight + self._health = (None, 0.0) # (ready: bool|None, checked_at) # -- availability ------------------------------------------------------- @@ -307,7 +324,64 @@ def from_env(cls): def status(self): return {"enabled": True, "model": self.model_name, "provider": self.provider, - "windowDays": TAKEAWAYS_WINDOW_DAYS} + "windowDays": TAKEAWAYS_WINDOW_DAYS, "modelState": self.model_state()} + + # -- the endpoint's cold start ------------------------------------------- + + def model_state(self): + """'ready' or 'starting'; always 'ready' for a hosted API. Cached a + few seconds: the page and its polls ask often. The health probe + itself is what wakes a scaled-to-zero endpoint, so asking is also + the way to make it ready -- the page's first status call does it.""" + if not self.base_url: + return "ready" + now = time.time() + with self._lock: + ready, at = self._health + if ready is not None and now - at < MODEL_HEALTH_TTL: + return "ready" if ready else "starting" + ready = self._probe() + with self._lock: + self._health = (ready, time.time()) + return "ready" if ready else "starting" + + def _probe(self): + """Is something answering behind the endpoint? vLLM's /health sits + beside /v1 and needs no key. Only Modal's 503 while nothing runs, or + no connection at all, mean "starting": another OpenAI-compatible + host without a /health route answers 404 and is up.""" + url = self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url + try: + with urllib.request.urlopen(url + "/health", timeout=10) as r: + return r.status < 500 + except urllib.error.HTTPError as e: + return e.code != 503 + except (urllib.error.URLError, TimeoutError, OSError): + return False + + def wake(self): + """Start the model booting without waiting on it. Called when someone + signs in: the container only runs while people are around, so the + sign-in is the earliest moment to pay the cold start, and it is then + a few minutes along by the time they open Insights. It still scales + to zero after the idle window; nothing here keeps it up.""" + if self.base_url: + threading.Thread(target=self.model_state, daemon=True, + name="insights-wake").start() + + def wait_ready(self, timeout=MODEL_START_TIMEOUT): + deadline = time.time() + timeout + while time.time() < deadline: + if self.model_state() == "ready": + return True + time.sleep(10) + return False + + @staticmethod + def _is_cold_start(exc): + # openai.APIStatusError (what ChatOpenAI raises) carries status_code; + # Modal's proxy says 503 "no upstreams available" while nothing runs + return getattr(exc, "status_code", None) == 503 def _model(self, effort): """The chat model for one effort level, on whichever provider is @@ -369,9 +443,16 @@ def _run(self, database_id, org_name, history, effort): """One agent run. Returns (answer_text, steps).""" agent = self._agent(database_id, effort) messages = [("system", self._system(org_name))] + list(history) - with self._slots: - result = agent.invoke({"messages": messages}, - config={"recursion_limit": MAX_TOOL_STEPS}) + try: + with self._slots: + result = agent.invoke({"messages": messages}, + config={"recursion_limit": MAX_TOOL_STEPS}) + except Exception as e: + if self._is_cold_start(e): + with self._lock: + self._health = (False, time.time()) + raise ModelStarting(str(e)[:200]) from e + raise return _unpack(result.get("messages") or []) # -- chat --------------------------------------------------------------- @@ -379,6 +460,8 @@ def _run(self, database_id, org_name, history, effort): def chat(self, database_id, org_name, history): """`history` is the validated (role, content) list. Returns the JSON the page renders: the answer plus the queries that produced it.""" + # no readiness gate here: _run turns the endpoint's 503 into + # ModelStarting, and a host without /health must still be asked answer, steps = self._run(database_id, org_name, history, self.chat_effort) return {"answer": answer, "steps": steps, "model": self.model_name} @@ -400,6 +483,10 @@ def takeaways(self, database_id, org_name, fresh=False): daemon=True, name=f"takeaways-{database_id[:8]}").start() out = {"status": "pending", "refreshing": running, "windowDays": TAKEAWAYS_WINDOW_DAYS, "model": self.model_name} + if running: + # cached, so a poll every ten seconds costs nothing; tells the + # page whether the wait is the model booting or the analysis + out["modelState"] = self.model_state() if entry: out.update(entry) out["status"] = "error" if "error" in entry else "ready" @@ -408,6 +495,10 @@ def takeaways(self, database_id, org_name, fresh=False): def _generate(self, database_id, org_name): started = time.time() try: + # a background thread can afford to wait out a cold start; the + # page shows "starting the model" meanwhile + if not self.wait_ready(): + raise ModelStarting(f"model not ready after {MODEL_START_TIMEOUT}s") answer, steps = self._run(database_id, org_name, [("user", TAKEAWAYS_PROMPT)], self.analysis_effort) data = parse_takeaways(answer) diff --git a/server/server.py b/server/server.py index 4e3fccc..1eb3c20 100644 --- a/server/server.py +++ b/server/server.py @@ -2317,6 +2317,8 @@ def _handle_login(self): time.sleep(0.3) # soften brute force self._redirect("/login?err=1&next=" + quote(nxt)) return + if self.insights: + self.insights.wake() # a person is here: get the model booting self._redirect(nxt, extra=[("Set-Cookie", self._session_cookie(token))]) def _handle_register(self): @@ -2669,6 +2671,9 @@ def do_POST(self): return self._json(self.insights.chat( viewer["database_id"], viewer.get("org_name"), history)) + except insights.ModelStarting: + # not a failure: the page waits and sends the question again + self._json({"error": "the model is starting up", "starting": True}, 503) except ValueError as e: self._json({"error": str(e)}, 400) except Exception as e: @@ -3074,6 +3079,8 @@ def do_GET(self): # packages or a model key explains itself instead of 503ing who = {"email": viewer["email"], "org": viewer.get("org_name")} if self.insights: + # status() probes the model, and that probe is what wakes + # a scaled-to-zero endpoint: opening the page starts it self._json({**self.insights.status(), "viewer": who}) else: self._json({"enabled": False, "viewer": who, diff --git a/server/static/insights.js b/server/static/insights.js index 4aae951..01dc8e4 100644 --- a/server/static/insights.js +++ b/server/static/insights.js @@ -28,10 +28,30 @@ async function api(path, body) { }); if (r.status === 401) { location.href = '/login?next=%2Finsights'; throw new Error('signed out'); } const data = await r.json().catch(() => ({})); - if (!r.ok) throw new Error(data.error || `request failed (${r.status})`); + if (!r.ok) { + const e = new Error(data.error || `request failed (${r.status})`); + e.starting = !!data.starting; // the model is cold-starting, not broken + throw e; + } return data; } +// The model scales to zero when idle and takes a few minutes to come back. +// Opening the page wakes it; this waits for the server to say it is ready. +const STARTING_NOTE = 'Starting the model. This usually takes a few minutes after it has been idle'; +const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); + +async function waitForModel(onTick) { + const deadline = Date.now() + 20 * 60 * 1000; // matches the server's MODEL_START_TIMEOUT + while (Date.now() < deadline) { + const st = await api('/api/insights/status').catch(() => null); + if (st && st.modelState !== 'starting') return true; + onTick(); + await sleep(10000); + } + return false; +} + function timeShort(iso) { if (!iso) return ''; const d = new Date(iso); @@ -167,7 +187,9 @@ function renderTakeaways(d) { for (const col of document.querySelectorAll('.takecol')) lists[col.dataset.kind] = col.querySelector('.takelist'); if (d.status === 'pending') { - meta.textContent = 'analysing the last 30 days, usually one to three minutes'; + meta.textContent = d.modelState === 'starting' + ? 'starting the model first, usually a few minutes' + : 'analysing the last 30 days, usually one to three minutes'; for (const l of Object.values(lists)) { l.replaceChildren(el('div', { class: 'take skel skel-take' }), el('div', { class: 'take skel skel-take' })); } @@ -184,7 +206,8 @@ function renderTakeaways(d) { return; } meta.textContent = `generated ${timeShort(d.generatedAt)}` + - (d.seconds ? ` in ${d.seconds}s` : '') + (d.refreshing ? ' · regenerating' : ''); + (d.seconds ? ` in ${d.seconds}s` : '') + + (d.refreshing ? (d.modelState === 'starting' ? ' · starting the model to regenerate' : ' · regenerating') : ''); $('#takeSummary').textContent = d.data.summary || ''; for (const [kind, list] of Object.entries(lists)) { const items = (d.data.highlights || []).filter((h) => h.kind === kind); @@ -266,7 +289,21 @@ async function ask(text) { placeholder.scrollIntoView({ block: 'nearest' }); setBusy(true); try { - const d = await api('/api/insights/chat', { messages: chat.messages }); + let d; + try { + d = await api('/api/insights/chat', { messages: chat.messages }); + } catch (e) { + if (!e.starting) throw e; + // cold model: say so, wait for it, then send the same question once + const body = placeholder.querySelector('.body'); + body.textContent = STARTING_NOTE; + const ok = await waitForModel(() => { + body.textContent = `${STARTING_NOTE}. Your question is sent automatically when it is ready`; + }); + if (!ok) throw new Error('the model did not start; try again in a few minutes'); + body.textContent = 'Querying'; + d = await api('/api/insights/chat', { messages: chat.messages }); + } placeholder.remove(); addAssistant(d.answer || '(no answer)', d.steps, false).scrollIntoView({ block: 'nearest' }); chat.messages.push({ role: 'assistant', content: d.answer || '' }); diff --git a/server/tests/test_insights.py b/server/tests/test_insights.py index 5f4295f..30d2455 100644 --- a/server/tests/test_insights.py +++ b/server/tests/test_insights.py @@ -109,10 +109,20 @@ class FakeEngine: def __init__(self): self.chats = [] + starting = False + + woke = 0 + def status(self): - return {"enabled": True, "model": self.model_name, "windowDays": 30} + return {"enabled": True, "model": self.model_name, "windowDays": 30, + "modelState": "starting" if self.starting else "ready"} + + def wake(self): + self.woke += 1 def chat(self, database_id, org_name, history): + if self.starting: + raise insights.ModelStarting() self.chats.append((database_id, org_name, history)) return {"answer": f"**{org_name}**: {len(history)} turns", "steps": [{"tool": "hotdata_execute_sql", "sql": "SELECT 1", "rows": 1, @@ -193,6 +203,94 @@ def test_chat_runs_for_a_signed_in_member(): server.Handler.insights = None +def test_a_cold_model_is_a_503_the_page_can_wait_on(): + """The Modal endpoint scales to zero and refuses with 503 while a + container boots. That must reach the page as 'starting', not as the + generic failure, so it can wait and resend instead of giving up.""" + engine = FakeEngine() + engine.starting = True + server.Handler.insights = engine + try: + status, raw = call(BASE, "/api/insights/status", headers=AS_ADA) + assert status == 200 and json.loads(raw)["modelState"] == "starting", raw[:200] + status, raw = call(BASE, "/api/insights/chat", method="POST", headers=AS_ADA, + body={"messages": [{"role": "user", "content": "who?"}]}) + d = json.loads(raw) + assert status == 503 and d.get("starting") is True, (status, d) + assert engine.chats == [], "nothing ran" + print("cold model: status says starting, chat is a 503 with starting=true") + finally: + server.Handler.insights = None + + +def test_the_engine_tells_a_cold_start_from_a_failure(): + """model_state probes /health beside /v1 and caches briefly; a 503 from + the client is a cold start, anything else is an error.""" + import http.server + import threading as th + + class Health(http.server.BaseHTTPRequestHandler): + code = 404 + def do_GET(self): + self.send_response(self.code); self.end_headers() + def log_message(self, *a): + pass + + httpd = http.server.HTTPServer(("127.0.0.1", 0), Health) + th.Thread(target=httpd.serve_forever, daemon=True).start() + port = httpd.server_address[1] + + eng = insights.InsightsEngine.__new__(insights.InsightsEngine) + eng._lock = th.Lock() + eng.base_url = "http://127.0.0.1:9/v1" # nothing listens: the probe fails fast + eng._health = (None, 0.0) + assert eng.model_state() == "starting" + # a host with no /health route answers 404: it is up, so the request goes out + eng.base_url = f"http://127.0.0.1:{port}/v1" + eng._health = (None, 0.0) + assert eng.model_state() == "ready", "a 404 from /health is an answer, not a cold start" + Health.code = 503 + eng._health = (None, 0.0) + assert eng.model_state() == "starting", "503 is Modal with nothing running" + httpd.shutdown() + eng._health = (True, __import__("time").time()) + assert eng.model_state() == "ready", "a fresh positive probe is trusted" + eng.base_url = "" + assert eng.model_state() == "ready", "a hosted API is always up" + + class Err(Exception): + status_code = 503 + assert insights.InsightsEngine._is_cold_start(Err()) + assert not insights.InsightsEngine._is_cold_start(RuntimeError("x")) + print("engine: unreachable -> starting; 404 /health -> ready; 503 -> starting; cached ready trusted") + + +def test_signing_in_wakes_the_model(): + """The model runs only while people are around, so a sign-in is what + starts its cold boot; a failed sign-in must not.""" + import urllib.parse + import urllib.request + engine = FakeEngine() + server.Handler.insights = engine + server.Handler._rate.clear() + try: + def post(pw): + body = urllib.parse.urlencode({"email": "ada@x.dev", "password": pw}).encode() + req = urllib.request.Request(BASE + "/login", data=body, method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}) + try: + urllib.request.urlopen(req, timeout=10).read() + except urllib.error.HTTPError: + pass + post("wrong-password") + assert engine.woke == 0, "a failed sign-in woke the model" + post("hunter2hunter2") + assert engine.woke == 1, engine.woke + print("a successful sign-in wakes the model once; a failed one does not") + finally: + server.Handler.insights = None + + def test_chat_refuses_everyone_but_a_cookie_session(): server.Handler.insights = FakeEngine() try: