Insights: treat the model's cold start as a wait, not a failure

docs/operating.md

@@ -144,10 +144,24 @@ export HOTOTEL_INSIGHTS_API_KEY=<the same random string>
 
 `modal run modal/hototel_llm.py` waits for the server and sends one test
 request with a tool attached, which is the quickest way to confirm tool
-calling works after a model change. The first request after idle waits a
-minute or two for the cold start; the very first ever boot also downloads
+calling works after a model change. The very first ever boot also downloads
 the weights (~60GB) into the `hototel-hf-cache` volume.
 
+**Cold starts.** The container stops ten minutes after the last request, and
+Modal's endpoint answers `503 no upstreams available` while a new one boots
+(about three minutes with the weights cached, and much longer when the volume
+reads slowly -- the class allows thirty and loads the shards with vLLM's
+parallel prefetch); that first refused request is what starts it. The server treats that 503 as "starting", not as a failure:
+a successful sign-in starts the model booting in the background (people are
+around; it still stops ten minutes after the last request), opening the
+Insights page probes `/health` and wakes it too, `/api/insights/
+status` reports `modelState` (`ready`/`starting`), a chat during the boot is a
+503 with `starting: true` that the page waits out (polling status every ten
+seconds, up to twenty minutes) before sending the question again, and the
+takeaways thread simply waits for `ready` before running. The model is
+deliberately not kept on around the clock: it runs only after someone signs
+in, for as long as they keep using it.
+
 The alternative is the **Anthropic API**: leave `HOTOTEL_INSIGHTS_BASE_URL`
 unset and set `ANTHROPIC_API_KEY`; the model defaults to `claude-opus-5-5`.
 Either way `HOTOTEL_INSIGHTS_MODEL` overrides the model name (on Modal the

modal/hototel_llm.py

@@ -70,7 +70,12 @@
     image=vllm_image,
     gpu=f"H100:{N_GPU}",
     scaledown_window=SCALEDOWN,
-    startup_timeout=15 * MINUTES,     # the first boot downloads ~60GB of weights
+    # Weights come off the volume at boot. Measured: ~3 minutes on a good
+    # day, but the volume's 9P mount has read the 57GB at ~60MB/s, which is
+    # 17 minutes -- a 15-minute limit killed that boot and the page saw only
+    # 503s. The limit is generous so a slow read finishes rather than restarts;
+    # the prefetch flag below is what makes it fast.
+    startup_timeout=30 * MINUTES,
     volumes={
         "/root/.cache/huggingface": hf_cache_vol,
         "/root/.cache/vllm": vllm_cache_vol,
@@ -102,6 +107,11 @@ def start(self):
             # dozen query results -- fits comfortably; more only costs KV cache
             "--max-model-len", "32768",
             "--gpu-memory-utilization", "0.92",
+            # read the safetensors shards with parallel threads into the page
+            # cache before mapping them: the volume is a network filesystem
+            # (9P) whose single-stream reads are latency-bound, and vLLM only
+            # prefetches on its own for NFS/Lustre
+            "--safetensors-load-strategy", "prefetch",
             # tool calling: Qwen3 writes Hermes-style <tool_call> blocks
             "--enable-auto-tool-choice",
             "--tool-call-parser", "hermes",

server/insights.py

@@ -29,6 +29,8 @@
 import sys
 import threading
 import time
+import urllib.error
+import urllib.request
 from datetime import datetime, timezone
 
 import core
@@ -245,6 +247,20 @@ def validate_history(raw):
     return out
 
 
+class ModelStarting(Exception):
+    """The model endpoint has no running container yet. Modal's endpoint
+    answers 503 while the GPU container cold-starts (a few minutes for this
+    model), and that first refused request is what starts it. Callers wait
+    or tell the page to come back, rather than report a failure."""
+
+
+# How long a cold start may take before it is treated as an outage. Weights
+# are cached in a volume, so a boot is ~3 minutes when the volume reads well
+# and has been seen to take 15+ when it does not; the Modal class allows 30.
+MODEL_START_TIMEOUT = 20 * 60
+MODEL_HEALTH_TTL = 5
+
+
 class InsightsEngine:
     """One per process. Holds the hotdata client, a per-database tool set and
     agent, and the takeaways cache. All public methods are thread-safe."""
@@ -266,6 +282,7 @@ def __init__(self, model=None, chat_effort="medium", analysis_effort="high",
         self._slots = threading.BoundedSemaphore(max_concurrent)
         self._takeaways = {}         # database_id -> {"at", "data"|"error", ...}
         self._generating = set()     # database_ids with a run in flight
+        self._health = (None, 0.0)   # (ready: bool|None, checked_at)
 
     # -- availability -------------------------------------------------------
 
@@ -307,7 +324,64 @@ def from_env(cls):
 
     def status(self):
         return {"enabled": True, "model": self.model_name, "provider": self.provider,
-                "windowDays": TAKEAWAYS_WINDOW_DAYS}
+                "windowDays": TAKEAWAYS_WINDOW_DAYS, "modelState": self.model_state()}
+
+    # -- the endpoint's cold start -------------------------------------------
+
+    def model_state(self):
+        """'ready' or 'starting'; always 'ready' for a hosted API. Cached a
+        few seconds: the page and its polls ask often. The health probe
+        itself is what wakes a scaled-to-zero endpoint, so asking is also
+        the way to make it ready -- the page's first status call does it."""
+        if not self.base_url:
+            return "ready"
+        now = time.time()
+        with self._lock:
+            ready, at = self._health
+        if ready is not None and now - at < MODEL_HEALTH_TTL:
+            return "ready" if ready else "starting"
+        ready = self._probe()
+        with self._lock:
+            self._health = (ready, time.time())
+        return "ready" if ready else "starting"
+
+    def _probe(self):
+        """Is something answering behind the endpoint? vLLM's /health sits
+        beside /v1 and needs no key. Only Modal's 503 while nothing runs, or
+        no connection at all, mean "starting": another OpenAI-compatible
+        host without a /health route answers 404 and is up."""
+        url = self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url
+        try:
+            with urllib.request.urlopen(url + "/health", timeout=10) as r:
+                return r.status < 500
+        except urllib.error.HTTPError as e:
+            return e.code != 503
+        except (urllib.error.URLError, TimeoutError, OSError):
+            return False
+
+    def wake(self):
+        """Start the model booting without waiting on it. Called when someone
+        signs in: the container only runs while people are around, so the
+        sign-in is the earliest moment to pay the cold start, and it is then
+        a few minutes along by the time they open Insights. It still scales
+        to zero after the idle window; nothing here keeps it up."""
+        if self.base_url:
+            threading.Thread(target=self.model_state, daemon=True,
+                             name="insights-wake").start()
+
+    def wait_ready(self, timeout=MODEL_START_TIMEOUT):
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            if self.model_state() == "ready":
+                return True
+            time.sleep(10)
+        return False
+
+    @staticmethod
+    def _is_cold_start(exc):
+        # openai.APIStatusError (what ChatOpenAI raises) carries status_code;
+        # Modal's proxy says 503 "no upstreams available" while nothing runs
+        return getattr(exc, "status_code", None) == 503
 
     def _model(self, effort):
         """The chat model for one effort level, on whichever provider is
@@ -369,16 +443,25 @@ def _run(self, database_id, org_name, history, effort):
         """One agent run. Returns (answer_text, steps)."""
         agent = self._agent(database_id, effort)
         messages = [("system", self._system(org_name))] + list(history)
-        with self._slots:
-            result = agent.invoke({"messages": messages},
-                                  config={"recursion_limit": MAX_TOOL_STEPS})
+        try:
+            with self._slots:
+                result = agent.invoke({"messages": messages},
+                                      config={"recursion_limit": MAX_TOOL_STEPS})
+        except Exception as e:
+            if self._is_cold_start(e):
+                with self._lock:
+                    self._health = (False, time.time())
+                raise ModelStarting(str(e)[:200]) from e
+            raise
         return _unpack(result.get("messages") or [])
 
     # -- chat ---------------------------------------------------------------
 
     def chat(self, database_id, org_name, history):
         """`history` is the validated (role, content) list. Returns the JSON
         the page renders: the answer plus the queries that produced it."""
+        # no readiness gate here: _run turns the endpoint's 503 into
+        # ModelStarting, and a host without /health must still be asked
         answer, steps = self._run(database_id, org_name, history, self.chat_effort)
         return {"answer": answer, "steps": steps, "model": self.model_name}
 
@@ -400,6 +483,10 @@ def takeaways(self, database_id, org_name, fresh=False):
                                  daemon=True, name=f"takeaways-{database_id[:8]}").start()
         out = {"status": "pending", "refreshing": running,
                "windowDays": TAKEAWAYS_WINDOW_DAYS, "model": self.model_name}
+        if running:
+            # cached, so a poll every ten seconds costs nothing; tells the
+            # page whether the wait is the model booting or the analysis
+            out["modelState"] = self.model_state()
         if entry:
             out.update(entry)
             out["status"] = "error" if "error" in entry else "ready"
@@ -408,6 +495,10 @@ def takeaways(self, database_id, org_name, fresh=False):
     def _generate(self, database_id, org_name):
         started = time.time()
         try:
+            # a background thread can afford to wait out a cold start; the
+            # page shows "starting the model" meanwhile
+            if not self.wait_ready():
+                raise ModelStarting(f"model not ready after {MODEL_START_TIMEOUT}s")
             answer, steps = self._run(database_id, org_name,
                                       [("user", TAKEAWAYS_PROMPT)], self.analysis_effort)
             data = parse_takeaways(answer)

server/server.py

@@ -2317,6 +2317,8 @@ def _handle_login(self):
             time.sleep(0.3)  # soften brute force
             self._redirect("/login?err=1&next=" + quote(nxt))
             return
+        if self.insights:
+            self.insights.wake()  # a person is here: get the model booting
         self._redirect(nxt, extra=[("Set-Cookie", self._session_cookie(token))])
 
     def _handle_register(self):
@@ -2669,6 +2671,9 @@ def do_POST(self):
                     return
                 self._json(self.insights.chat(
                     viewer["database_id"], viewer.get("org_name"), history))
+            except insights.ModelStarting:
+                # not a failure: the page waits and sends the question again
+                self._json({"error": "the model is starting up", "starting": True}, 503)
             except ValueError as e:
                 self._json({"error": str(e)}, 400)
             except Exception as e:
@@ -3074,6 +3079,8 @@ def do_GET(self):
                 # packages or a model key explains itself instead of 503ing
                 who = {"email": viewer["email"], "org": viewer.get("org_name")}
                 if self.insights:
+                    # status() probes the model, and that probe is what wakes
+                    # a scaled-to-zero endpoint: opening the page starts it
                     self._json({**self.insights.status(), "viewer": who})
                 else:
                     self._json({"enabled": False, "viewer": who,

server/static/insights.js

@@ -28,10 +28,30 @@ async function api(path, body) {
   });
   if (r.status === 401) { location.href = '/login?next=%2Finsights'; throw new Error('signed out'); }
   const data = await r.json().catch(() => ({}));
-  if (!r.ok) throw new Error(data.error || `request failed (${r.status})`);
+  if (!r.ok) {
+    const e = new Error(data.error || `request failed (${r.status})`);
+    e.starting = !!data.starting;  // the model is cold-starting, not broken
+    throw e;
+  }
   return data;
 }
 
+// The model scales to zero when idle and takes a few minutes to come back.
+// Opening the page wakes it; this waits for the server to say it is ready.
+const STARTING_NOTE = 'Starting the model. This usually takes a few minutes after it has been idle';
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+async function waitForModel(onTick) {
+  const deadline = Date.now() + 20 * 60 * 1000;  // matches the server's MODEL_START_TIMEOUT
+  while (Date.now() < deadline) {
+    const st = await api('/api/insights/status').catch(() => null);
+    if (st && st.modelState !== 'starting') return true;
+    onTick();
+    await sleep(10000);
+  }
+  return false;
+}
+
 function timeShort(iso) {
   if (!iso) return '';
   const d = new Date(iso);
@@ -167,7 +187,9 @@ function renderTakeaways(d) {
   for (const col of document.querySelectorAll('.takecol')) lists[col.dataset.kind] = col.querySelector('.takelist');
 
   if (d.status === 'pending') {
-    meta.textContent = 'analysing the last 30 days, usually one to three minutes';
+    meta.textContent = d.modelState === 'starting'
+      ? 'starting the model first, usually a few minutes'
+      : 'analysing the last 30 days, usually one to three minutes';
     for (const l of Object.values(lists)) {
       l.replaceChildren(el('div', { class: 'take skel skel-take' }), el('div', { class: 'take skel skel-take' }));
     }
@@ -184,7 +206,8 @@ function renderTakeaways(d) {
     return;
   }
   meta.textContent = `generated ${timeShort(d.generatedAt)}` +
-    (d.seconds ? ` in ${d.seconds}s` : '') + (d.refreshing ? ' · regenerating' : '');
+    (d.seconds ? ` in ${d.seconds}s` : '') +
+    (d.refreshing ? (d.modelState === 'starting' ? ' · starting the model to regenerate' : ' · regenerating') : '');
   $('#takeSummary').textContent = d.data.summary || '';
   for (const [kind, list] of Object.entries(lists)) {
     const items = (d.data.highlights || []).filter((h) => h.kind === kind);
@@ -266,7 +289,21 @@ async function ask(text) {
   placeholder.scrollIntoView({ block: 'nearest' });
   setBusy(true);
   try {
-    const d = await api('/api/insights/chat', { messages: chat.messages });
+    let d;
+    try {
+      d = await api('/api/insights/chat', { messages: chat.messages });
+    } catch (e) {
+      if (!e.starting) throw e;
+      // cold model: say so, wait for it, then send the same question once
+      const body = placeholder.querySelector('.body');
+      body.textContent = STARTING_NOTE;
+      const ok = await waitForModel(() => {
+        body.textContent = `${STARTING_NOTE}. Your question is sent automatically when it is ready`;
+      });
+      if (!ok) throw new Error('the model did not start; try again in a few minutes');
+      body.textContent = 'Querying';
+      d = await api('/api/insights/chat', { messages: chat.messages });
+    }
     placeholder.remove();
     addAssistant(d.answer || '(no answer)', d.steps, false).scrollIntoView({ block: 'nearest' });
     chat.messages.push({ role: 'assistant', content: d.answer || '' });