From ffccd50e2919a4ca9c3472fbbb7019cab6c450cc Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Fri, 16 Oct 2026 20:31:30 -0700
Subject: [PATCH 1/4] Insights: treat the model's cold start as a wait, not a
 failure

The Modal endpoint scales to zero and answers 503 "no upstreams available"
while a container boots; the first refused request is what starts it. Both
the chat and the takeaways run reported that as "the analysis did not
finish". Now opening the page probes /health to wake the model, status
reports modelState, a chat during the boot is a 503 with starting=true that
the page waits out (polling status, up to eight minutes) before resending,
and the takeaways thread waits for ready before it runs.
---
 docs/operating.md             | 15 +++++-
 server/insights.py            | 89 +++++++++++++++++++++++++++++++++--
 server/server.py              |  6 +++
 server/static/insights.js     | 45 ++++++++++++++++--
 server/tests/test_insights.py | 50 +++++++++++++++++++-
 5 files changed, 194 insertions(+), 11 deletions(-)

diff --git a/docs/operating.md b/docs/operating.md
index be712a6..bc09cca 100644
--- a/docs/operating.md
+++ b/docs/operating.md
@@ -144,10 +144,21 @@ export HOTOTEL_INSIGHTS_API_KEY=<the same random string>
 
 `modal run modal/hototel_llm.py` waits for the server and sends one test
 request with a tool attached, which is the quickest way to confirm tool
-calling works after a model change. The first request after idle waits a
-minute or two for the cold start; the very first ever boot also downloads
+calling works after a model change. The very first ever boot also downloads
 the weights (~60GB) into the `hototel-hf-cache` volume.
 
+**Cold starts.** The container stops ten minutes after the last request, and
+Modal's endpoint answers `503 no upstreams available` while a new one boots
+(about three minutes with the weights cached); that first refused request is
+what starts it. The server treats that 503 as "starting", not as a failure:
+opening the Insights page probes `/health` to wake the model, `/api/insights/
+status` reports `modelState` (`ready`/`starting`), a chat during the boot is a
+503 with `starting: true` that the page waits out (polling status every ten
+seconds, up to eight minutes) before sending the question again, and the
+takeaways thread simply waits for `ready` before running. Keeping one
+container always on (`min_containers=1` on the Modal class) would remove the
+wait at roughly the cost of an H100 around the clock.
+
 The alternative is the **Anthropic API**: leave `HOTOTEL_INSIGHTS_BASE_URL`
 unset and set `ANTHROPIC_API_KEY`; the model defaults to `claude-opus-5-5`.
 Either way `HOTOTEL_INSIGHTS_MODEL` overrides the model name (on Modal the
diff --git a/server/insights.py b/server/insights.py
index b64be19..20188b4 100644
--- a/server/insights.py
+++ b/server/insights.py
@@ -29,6 +29,8 @@
 import sys
 import threading
 import time
+import urllib.error
+import urllib.request
 from datetime import datetime, timezone
 
 import core
@@ -245,6 +247,20 @@ def validate_history(raw):
     return out
 
 
+class ModelStarting(Exception):
+    """The model endpoint has no running container yet. Modal's endpoint
+    answers 503 while the GPU container cold-starts (a few minutes for this
+    model), and that first refused request is what starts it. Callers wait
+    or tell the page to come back, rather than report a failure."""
+
+
+# How long a cold start may take before it is treated as an outage. Weights
+# are cached in a volume, so a boot is ~3 minutes; the very first boot after a
+# redeploy with a new model can be much longer, and gets to fail.
+MODEL_START_TIMEOUT = 8 * 60
+MODEL_HEALTH_TTL = 5
+
+
 class InsightsEngine:
     """One per process. Holds the hotdata client, a per-database tool set and
     agent, and the takeaways cache. All public methods are thread-safe."""
@@ -266,6 +282,7 @@ def __init__(self, model=None, chat_effort="medium", analysis_effort="high",
         self._slots = threading.BoundedSemaphore(max_concurrent)
         self._takeaways = {}         # database_id -> {"at", "data"|"error", ...}
         self._generating = set()     # database_ids with a run in flight
+        self._health = (None, 0.0)   # (ready: bool|None, checked_at)
 
     # -- availability -------------------------------------------------------
 
@@ -307,7 +324,54 @@ def from_env(cls):
 
     def status(self):
         return {"enabled": True, "model": self.model_name, "provider": self.provider,
-                "windowDays": TAKEAWAYS_WINDOW_DAYS}
+                "windowDays": TAKEAWAYS_WINDOW_DAYS, "modelState": self.model_state()}
+
+    # -- the endpoint's cold start -------------------------------------------
+
+    def model_state(self):
+        """'ready', 'starting' or 'ready' for a hosted API that is always up.
+        Cached a few seconds: the page and its polls ask often. The health
+        probe itself is what wakes a scaled-to-zero endpoint, so asking is
+        also the way to make it ready -- the page's first call does it."""
+        if not self.base_url:
+            return "ready"
+        now = time.time()
+        with self._lock:
+            ready, at = self._health
+        if ready is not None and now - at < MODEL_HEALTH_TTL:
+            return "ready" if ready else "starting"
+        ready = self._probe()
+        with self._lock:
+            self._health = (ready, time.time())
+        return "ready" if ready else "starting"
+
+    def _probe(self):
+        # vLLM's /health sits beside /v1 and needs no key
+        url = self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url
+        try:
+            with urllib.request.urlopen(url + "/health", timeout=10) as r:
+                return r.status == 200
+        except (urllib.error.URLError, TimeoutError, OSError):
+            return False
+
+    def warm(self):
+        """Kick the cold start without waiting on it."""
+        if self.base_url:
+            threading.Thread(target=self.model_state, daemon=True, name="insights-warm").start()
+
+    def wait_ready(self, timeout=MODEL_START_TIMEOUT):
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            if self.model_state() == "ready":
+                return True
+            time.sleep(10)
+        return False
+
+    @staticmethod
+    def _is_cold_start(exc):
+        # openai.APIStatusError (what ChatOpenAI raises) carries status_code;
+        # Modal's proxy says 503 "no upstreams available" while nothing runs
+        return getattr(exc, "status_code", None) == 503
 
     def _model(self, effort):
         """The chat model for one effort level, on whichever provider is
@@ -369,9 +433,16 @@ def _run(self, database_id, org_name, history, effort):
         """One agent run. Returns (answer_text, steps)."""
         agent = self._agent(database_id, effort)
         messages = [("system", self._system(org_name))] + list(history)
-        with self._slots:
-            result = agent.invoke({"messages": messages},
-                                  config={"recursion_limit": MAX_TOOL_STEPS})
+        try:
+            with self._slots:
+                result = agent.invoke({"messages": messages},
+                                      config={"recursion_limit": MAX_TOOL_STEPS})
+        except Exception as e:
+            if self._is_cold_start(e):
+                with self._lock:
+                    self._health = (False, time.time())
+                raise ModelStarting() from e
+            raise
         return _unpack(result.get("messages") or [])
 
     # -- chat ---------------------------------------------------------------
@@ -379,6 +450,8 @@ def _run(self, database_id, org_name, history, effort):
     def chat(self, database_id, org_name, history):
         """`history` is the validated (role, content) list. Returns the JSON
         the page renders: the answer plus the queries that produced it."""
+        if self.model_state() != "ready":
+            raise ModelStarting()
         answer, steps = self._run(database_id, org_name, history, self.chat_effort)
         return {"answer": answer, "steps": steps, "model": self.model_name}
 
@@ -400,6 +473,10 @@ def takeaways(self, database_id, org_name, fresh=False):
                                  daemon=True, name=f"takeaways-{database_id[:8]}").start()
         out = {"status": "pending", "refreshing": running,
                "windowDays": TAKEAWAYS_WINDOW_DAYS, "model": self.model_name}
+        if running:
+            # cached, so a poll every ten seconds costs nothing; tells the
+            # page whether the wait is the model booting or the analysis
+            out["modelState"] = self.model_state()
         if entry:
             out.update(entry)
             out["status"] = "error" if "error" in entry else "ready"
@@ -408,6 +485,10 @@ def takeaways(self, database_id, org_name, fresh=False):
     def _generate(self, database_id, org_name):
         started = time.time()
         try:
+            # a background thread can afford to wait out a cold start; the
+            # page shows "starting the model" meanwhile
+            if not self.wait_ready():
+                raise ModelStarting()
             answer, steps = self._run(database_id, org_name,
                                       [("user", TAKEAWAYS_PROMPT)], self.analysis_effort)
             data = parse_takeaways(answer)
diff --git a/server/server.py b/server/server.py
index 4e3fccc..fecc3b0 100644
--- a/server/server.py
+++ b/server/server.py
@@ -2669,6 +2669,9 @@ def do_POST(self):
                     return
                 self._json(self.insights.chat(
                     viewer["database_id"], viewer.get("org_name"), history))
+            except insights.ModelStarting:
+                # not a failure: the page waits and sends the question again
+                self._json({"error": "the model is starting up", "starting": True}, 503)
             except ValueError as e:
                 self._json({"error": str(e)}, 400)
             except Exception as e:
@@ -3074,6 +3077,9 @@ def do_GET(self):
                 # packages or a model key explains itself instead of 503ing
                 who = {"email": viewer["email"], "org": viewer.get("org_name")}
                 if self.insights:
+                    # opening the page is the moment to wake a scaled-to-zero
+                    # model, so it is booting while the person reads or types
+                    self.insights.warm()
                     self._json({**self.insights.status(), "viewer": who})
                 else:
                     self._json({"enabled": False, "viewer": who,
diff --git a/server/static/insights.js b/server/static/insights.js
index 4aae951..32e3d45 100644
--- a/server/static/insights.js
+++ b/server/static/insights.js
@@ -28,10 +28,30 @@ async function api(path, body) {
   });
   if (r.status === 401) { location.href = '/login?next=%2Finsights'; throw new Error('signed out'); }
   const data = await r.json().catch(() => ({}));
-  if (!r.ok) throw new Error(data.error || `request failed (${r.status})`);
+  if (!r.ok) {
+    const e = new Error(data.error || `request failed (${r.status})`);
+    e.starting = !!data.starting;  // the model is cold-starting, not broken
+    throw e;
+  }
   return data;
 }
 
+// The model scales to zero when idle and takes a few minutes to come back.
+// Opening the page wakes it; this waits for the server to say it is ready.
+const STARTING_NOTE = 'Starting the model. This takes two to three minutes after it has been idle';
+const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+async function waitForModel(onTick) {
+  const deadline = Date.now() + 8 * 60 * 1000;
+  while (Date.now() < deadline) {
+    const st = await api('/api/insights/status').catch(() => null);
+    if (st && st.modelState !== 'starting') return true;
+    onTick();
+    await sleep(10000);
+  }
+  return false;
+}
+
 function timeShort(iso) {
   if (!iso) return '';
   const d = new Date(iso);
@@ -167,7 +187,9 @@ function renderTakeaways(d) {
   for (const col of document.querySelectorAll('.takecol')) lists[col.dataset.kind] = col.querySelector('.takelist');
 
   if (d.status === 'pending') {
-    meta.textContent = 'analysing the last 30 days, usually one to three minutes';
+    meta.textContent = d.modelState === 'starting'
+      ? 'starting the model first, two to three minutes'
+      : 'analysing the last 30 days, usually one to three minutes';
     for (const l of Object.values(lists)) {
       l.replaceChildren(el('div', { class: 'take skel skel-take' }), el('div', { class: 'take skel skel-take' }));
     }
@@ -184,7 +206,8 @@ function renderTakeaways(d) {
     return;
   }
   meta.textContent = `generated ${timeShort(d.generatedAt)}` +
-    (d.seconds ? ` in ${d.seconds}s` : '') + (d.refreshing ? ' · regenerating' : '');
+    (d.seconds ? ` in ${d.seconds}s` : '') +
+    (d.refreshing ? (d.modelState === 'starting' ? ' · starting the model to regenerate' : ' · regenerating') : '');
   $('#takeSummary').textContent = d.data.summary || '';
   for (const [kind, list] of Object.entries(lists)) {
     const items = (d.data.highlights || []).filter((h) => h.kind === kind);
@@ -266,7 +289,21 @@ async function ask(text) {
   placeholder.scrollIntoView({ block: 'nearest' });
   setBusy(true);
   try {
-    const d = await api('/api/insights/chat', { messages: chat.messages });
+    let d;
+    try {
+      d = await api('/api/insights/chat', { messages: chat.messages });
+    } catch (e) {
+      if (!e.starting) throw e;
+      // cold model: say so, wait for it, then send the same question once
+      const body = placeholder.querySelector('.body');
+      body.textContent = STARTING_NOTE;
+      const ok = await waitForModel(() => {
+        body.textContent = `${STARTING_NOTE}. Your question is sent automatically when it is ready`;
+      });
+      if (!ok) throw new Error('the model did not start; try again in a few minutes');
+      body.textContent = 'Querying';
+      d = await api('/api/insights/chat', { messages: chat.messages });
+    }
     placeholder.remove();
     addAssistant(d.answer || '(no answer)', d.steps, false).scrollIntoView({ block: 'nearest' });
     chat.messages.push({ role: 'assistant', content: d.answer || '' });
diff --git a/server/tests/test_insights.py b/server/tests/test_insights.py
index 5f4295f..2c32386 100644
--- a/server/tests/test_insights.py
+++ b/server/tests/test_insights.py
@@ -109,10 +109,18 @@ class FakeEngine:
     def __init__(self):
         self.chats = []
 
+    starting = False
+
     def status(self):
-        return {"enabled": True, "model": self.model_name, "windowDays": 30}
+        return {"enabled": True, "model": self.model_name, "windowDays": 30,
+                "modelState": "starting" if self.starting else "ready"}
+
+    def warm(self):
+        pass
 
     def chat(self, database_id, org_name, history):
+        if self.starting:
+            raise insights.ModelStarting()
         self.chats.append((database_id, org_name, history))
         return {"answer": f"**{org_name}**: {len(history)} turns",
                 "steps": [{"tool": "hotdata_execute_sql", "sql": "SELECT 1", "rows": 1,
@@ -193,6 +201,46 @@ def test_chat_runs_for_a_signed_in_member():
         server.Handler.insights = None
 
 
+def test_a_cold_model_is_a_503_the_page_can_wait_on():
+    """The Modal endpoint scales to zero and refuses with 503 while a
+    container boots. That must reach the page as 'starting', not as the
+    generic failure, so it can wait and resend instead of giving up."""
+    engine = FakeEngine()
+    engine.starting = True
+    server.Handler.insights = engine
+    try:
+        status, raw = call(BASE, "/api/insights/status", headers=AS_ADA)
+        assert status == 200 and json.loads(raw)["modelState"] == "starting", raw[:200]
+        status, raw = call(BASE, "/api/insights/chat", method="POST", headers=AS_ADA,
+                           body={"messages": [{"role": "user", "content": "who?"}]})
+        d = json.loads(raw)
+        assert status == 503 and d.get("starting") is True, (status, d)
+        assert engine.chats == [], "nothing ran"
+        print("cold model: status says starting, chat is a 503 with starting=true")
+    finally:
+        server.Handler.insights = None
+
+
+def test_the_engine_tells_a_cold_start_from_a_failure():
+    """model_state probes /health beside /v1 and caches briefly; a 503 from
+    the client is a cold start, anything else is an error."""
+    eng = insights.InsightsEngine.__new__(insights.InsightsEngine)
+    eng.base_url = "http://127.0.0.1:9/v1"   # nothing listens: the probe fails fast
+    eng._lock = __import__("threading").Lock()
+    eng._health = (None, 0.0)
+    assert eng.model_state() == "starting"
+    eng._health = (True, __import__("time").time())
+    assert eng.model_state() == "ready", "a fresh positive probe is trusted"
+    eng.base_url = ""
+    assert eng.model_state() == "ready", "a hosted API is always up"
+
+    class Err(Exception):
+        status_code = 503
+    assert insights.InsightsEngine._is_cold_start(Err())
+    assert not insights.InsightsEngine._is_cold_start(RuntimeError("x"))
+    print("engine: unreachable endpoint -> starting; cached ready -> ready; 503 is a cold start")
+
+
 def test_chat_refuses_everyone_but_a_cookie_session():
     server.Handler.insights = FakeEngine()
     try:

From b6d096cf2ddaada27f1bb9786038e50f3559f2d0 Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Fri, 16 Oct 2026 20:33:09 -0700
Subject: [PATCH 2/4] Survive a slow weight load: 30-minute startup, prefetch
 reads, 20-minute wait

A cold boot read the 57GB of weights off the volume at ~60MB/s (a minute per
shard, 16 shards) and the 15-minute startup timeout killed it, so the page
only ever saw 503s. The class now allows 30 minutes and loads the shards
with vLLM's parallel prefetch strategy, which the loader only turns on by
itself for NFS/Lustre, not the volume's 9P mount. The server and the page
wait up to 20 minutes for "ready" to match.
---
 docs/operating.md         |  7 ++++---
 modal/hototel_llm.py      | 12 +++++++++++-
 server/insights.py        |  6 +++---
 server/static/insights.js |  4 ++--
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/docs/operating.md b/docs/operating.md
index bc09cca..d46128b 100644
--- a/docs/operating.md
+++ b/docs/operating.md
@@ -149,12 +149,13 @@ the weights (~60GB) into the `hototel-hf-cache` volume.
 
 **Cold starts.** The container stops ten minutes after the last request, and
 Modal's endpoint answers `503 no upstreams available` while a new one boots
-(about three minutes with the weights cached); that first refused request is
-what starts it. The server treats that 503 as "starting", not as a failure:
+(about three minutes with the weights cached, and much longer when the volume
+reads slowly -- the class allows thirty and loads the shards with vLLM's
+parallel prefetch); that first refused request is what starts it. The server treats that 503 as "starting", not as a failure:
 opening the Insights page probes `/health` to wake the model, `/api/insights/
 status` reports `modelState` (`ready`/`starting`), a chat during the boot is a
 503 with `starting: true` that the page waits out (polling status every ten
-seconds, up to eight minutes) before sending the question again, and the
+seconds, up to twenty minutes) before sending the question again, and the
 takeaways thread simply waits for `ready` before running. Keeping one
 container always on (`min_containers=1` on the Modal class) would remove the
 wait at roughly the cost of an H100 around the clock.
diff --git a/modal/hototel_llm.py b/modal/hototel_llm.py
index 57a7f23..31946fc 100644
--- a/modal/hototel_llm.py
+++ b/modal/hototel_llm.py
@@ -70,7 +70,12 @@
     image=vllm_image,
     gpu=f"H100:{N_GPU}",
     scaledown_window=SCALEDOWN,
-    startup_timeout=15 * MINUTES,     # the first boot downloads ~60GB of weights
+    # Weights come off the volume at boot. Measured: ~3 minutes on a good
+    # day, but the volume's 9P mount has read the 57GB at ~60MB/s, which is
+    # 17 minutes -- a 15-minute limit killed that boot and the page saw only
+    # 503s. The limit is generous so a slow read finishes rather than restarts;
+    # the prefetch flag below is what makes it fast.
+    startup_timeout=30 * MINUTES,
     volumes={
         "/root/.cache/huggingface": hf_cache_vol,
         "/root/.cache/vllm": vllm_cache_vol,
@@ -102,6 +107,11 @@ def start(self):
             # dozen query results -- fits comfortably; more only costs KV cache
             "--max-model-len", "32768",
             "--gpu-memory-utilization", "0.92",
+            # read the safetensors shards with parallel threads into the page
+            # cache before mapping them: the volume is a network filesystem
+            # (9P) whose single-stream reads are latency-bound, and vLLM only
+            # prefetches on its own for NFS/Lustre
+            "--safetensors-load-strategy", "prefetch",
             # tool calling: Qwen3 writes Hermes-style <tool_call> blocks
             "--enable-auto-tool-choice",
             "--tool-call-parser", "hermes",
diff --git a/server/insights.py b/server/insights.py
index 20188b4..7c6548d 100644
--- a/server/insights.py
+++ b/server/insights.py
@@ -255,9 +255,9 @@ class ModelStarting(Exception):
 
 
 # How long a cold start may take before it is treated as an outage. Weights
-# are cached in a volume, so a boot is ~3 minutes; the very first boot after a
-# redeploy with a new model can be much longer, and gets to fail.
-MODEL_START_TIMEOUT = 8 * 60
+# are cached in a volume, so a boot is ~3 minutes when the volume reads well
+# and has been seen to take 15+ when it does not; the Modal class allows 30.
+MODEL_START_TIMEOUT = 20 * 60
 MODEL_HEALTH_TTL = 5
 
 
diff --git a/server/static/insights.js b/server/static/insights.js
index 32e3d45..58ee15a 100644
--- a/server/static/insights.js
+++ b/server/static/insights.js
@@ -38,11 +38,11 @@ async function api(path, body) {
 
 // The model scales to zero when idle and takes a few minutes to come back.
 // Opening the page wakes it; this waits for the server to say it is ready.
-const STARTING_NOTE = 'Starting the model. This takes two to three minutes after it has been idle';
+const STARTING_NOTE = 'Starting the model. This usually takes a few minutes after it has been idle';
 const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
 
 async function waitForModel(onTick) {
-  const deadline = Date.now() + 8 * 60 * 1000;
+  const deadline = Date.now() + 20 * 60 * 1000;  // matches the server's MODEL_START_TIMEOUT
   while (Date.now() < deadline) {
     const st = await api('/api/insights/status').catch(() => null);
     if (st && st.modelState !== 'starting') return true;

From 6de0ce0e8723b2a26b7c8f6b670c5197e8397a23 Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Fri, 16 Oct 2026 20:35:43 -0700
Subject: [PATCH 3/4] Insights: only a 503 or no connection means "starting";
 the chat always asks

Review fixes. The chat no longer gates on the probe: _run turns the
endpoint's 503 into ModelStarting, and that request is what wakes Modal.
The probe reports "starting" only for a 503 or a connection failure, so
an OpenAI-compatible host without /health (404) is treated as up. The
separate warm() call went: status() probes synchronously already.
ModelStarting carries a message so the log says why a run gave up.
---
 server/insights.py            | 30 +++++++++++++++---------------
 server/server.py              |  5 ++---
 server/static/insights.js     |  2 +-
 server/tests/test_insights.py | 29 ++++++++++++++++++++++++-----
 4 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/server/insights.py b/server/insights.py
index 7c6548d..59a6f5c 100644
--- a/server/insights.py
+++ b/server/insights.py
@@ -329,10 +329,10 @@ def status(self):
     # -- the endpoint's cold start -------------------------------------------
 
     def model_state(self):
-        """'ready', 'starting' or 'ready' for a hosted API that is always up.
-        Cached a few seconds: the page and its polls ask often. The health
-        probe itself is what wakes a scaled-to-zero endpoint, so asking is
-        also the way to make it ready -- the page's first call does it."""
+        """'ready' or 'starting'; always 'ready' for a hosted API. Cached a
+        few seconds: the page and its polls ask often. The health probe
+        itself is what wakes a scaled-to-zero endpoint, so asking is also
+        the way to make it ready -- the page's first status call does it."""
         if not self.base_url:
             return "ready"
         now = time.time()
@@ -346,19 +346,19 @@ def model_state(self):
         return "ready" if ready else "starting"
 
     def _probe(self):
-        # vLLM's /health sits beside /v1 and needs no key
+        """Is something answering behind the endpoint? vLLM's /health sits
+        beside /v1 and needs no key. Only Modal's 503 while nothing runs, or
+        no connection at all, mean "starting": another OpenAI-compatible
+        host without a /health route answers 404 and is up."""
         url = self.base_url[:-3] if self.base_url.endswith("/v1") else self.base_url
         try:
             with urllib.request.urlopen(url + "/health", timeout=10) as r:
-                return r.status == 200
+                return r.status < 500
+        except urllib.error.HTTPError as e:
+            return e.code != 503
         except (urllib.error.URLError, TimeoutError, OSError):
             return False
 
-    def warm(self):
-        """Kick the cold start without waiting on it."""
-        if self.base_url:
-            threading.Thread(target=self.model_state, daemon=True, name="insights-warm").start()
-
     def wait_ready(self, timeout=MODEL_START_TIMEOUT):
         deadline = time.time() + timeout
         while time.time() < deadline:
@@ -441,7 +441,7 @@ def _run(self, database_id, org_name, history, effort):
             if self._is_cold_start(e):
                 with self._lock:
                     self._health = (False, time.time())
-                raise ModelStarting() from e
+                raise ModelStarting(str(e)[:200]) from e
             raise
         return _unpack(result.get("messages") or [])
 
@@ -450,8 +450,8 @@ def _run(self, database_id, org_name, history, effort):
     def chat(self, database_id, org_name, history):
         """`history` is the validated (role, content) list. Returns the JSON
         the page renders: the answer plus the queries that produced it."""
-        if self.model_state() != "ready":
-            raise ModelStarting()
+        # no readiness gate here: _run turns the endpoint's 503 into
+        # ModelStarting, and a host without /health must still be asked
         answer, steps = self._run(database_id, org_name, history, self.chat_effort)
         return {"answer": answer, "steps": steps, "model": self.model_name}
 
@@ -488,7 +488,7 @@ def _generate(self, database_id, org_name):
             # a background thread can afford to wait out a cold start; the
             # page shows "starting the model" meanwhile
             if not self.wait_ready():
-                raise ModelStarting()
+                raise ModelStarting(f"model not ready after {MODEL_START_TIMEOUT}s")
             answer, steps = self._run(database_id, org_name,
                                       [("user", TAKEAWAYS_PROMPT)], self.analysis_effort)
             data = parse_takeaways(answer)
diff --git a/server/server.py b/server/server.py
index fecc3b0..bacb309 100644
--- a/server/server.py
+++ b/server/server.py
@@ -3077,9 +3077,8 @@ def do_GET(self):
                 # packages or a model key explains itself instead of 503ing
                 who = {"email": viewer["email"], "org": viewer.get("org_name")}
                 if self.insights:
-                    # opening the page is the moment to wake a scaled-to-zero
-                    # model, so it is booting while the person reads or types
-                    self.insights.warm()
+                    # status() probes the model, and that probe is what wakes
+                    # a scaled-to-zero endpoint: opening the page starts it
                     self._json({**self.insights.status(), "viewer": who})
                 else:
                     self._json({"enabled": False, "viewer": who,
diff --git a/server/static/insights.js b/server/static/insights.js
index 58ee15a..01dc8e4 100644
--- a/server/static/insights.js
+++ b/server/static/insights.js
@@ -188,7 +188,7 @@ function renderTakeaways(d) {
 
   if (d.status === 'pending') {
     meta.textContent = d.modelState === 'starting'
-      ? 'starting the model first, two to three minutes'
+      ? 'starting the model first, usually a few minutes'
       : 'analysing the last 30 days, usually one to three minutes';
     for (const l of Object.values(lists)) {
       l.replaceChildren(el('div', { class: 'take skel skel-take' }), el('div', { class: 'take skel skel-take' }));
diff --git a/server/tests/test_insights.py b/server/tests/test_insights.py
index 2c32386..4cd04bf 100644
--- a/server/tests/test_insights.py
+++ b/server/tests/test_insights.py
@@ -115,9 +115,6 @@ def status(self):
         return {"enabled": True, "model": self.model_name, "windowDays": 30,
                 "modelState": "starting" if self.starting else "ready"}
 
-    def warm(self):
-        pass
-
     def chat(self, database_id, org_name, history):
         if self.starting:
             raise insights.ModelStarting()
@@ -224,11 +221,33 @@ def test_a_cold_model_is_a_503_the_page_can_wait_on():
 def test_the_engine_tells_a_cold_start_from_a_failure():
     """model_state probes /health beside /v1 and caches briefly; a 503 from
     the client is a cold start, anything else is an error."""
+    import http.server
+    import threading as th
+
+    class Health(http.server.BaseHTTPRequestHandler):
+        code = 404
+        def do_GET(self):
+            self.send_response(self.code); self.end_headers()
+        def log_message(self, *a):
+            pass
+
+    httpd = http.server.HTTPServer(("127.0.0.1", 0), Health)
+    th.Thread(target=httpd.serve_forever, daemon=True).start()
+    port = httpd.server_address[1]
+
     eng = insights.InsightsEngine.__new__(insights.InsightsEngine)
+    eng._lock = th.Lock()
     eng.base_url = "http://127.0.0.1:9/v1"   # nothing listens: the probe fails fast
-    eng._lock = __import__("threading").Lock()
     eng._health = (None, 0.0)
     assert eng.model_state() == "starting"
+    # a host with no /health route answers 404: it is up, so the request goes out
+    eng.base_url = f"http://127.0.0.1:{port}/v1"
+    eng._health = (None, 0.0)
+    assert eng.model_state() == "ready", "a 404 from /health is an answer, not a cold start"
+    Health.code = 503
+    eng._health = (None, 0.0)
+    assert eng.model_state() == "starting", "503 is Modal with nothing running"
+    httpd.shutdown()
     eng._health = (True, __import__("time").time())
     assert eng.model_state() == "ready", "a fresh positive probe is trusted"
     eng.base_url = ""
@@ -238,7 +257,7 @@ class Err(Exception):
         status_code = 503
     assert insights.InsightsEngine._is_cold_start(Err())
     assert not insights.InsightsEngine._is_cold_start(RuntimeError("x"))
-    print("engine: unreachable endpoint -> starting; cached ready -> ready; 503 is a cold start")
+    print("engine: unreachable -> starting; 404 /health -> ready; 503 -> starting; cached ready trusted")
 
 
 def test_chat_refuses_everyone_but_a_cookie_session():

From f8f1ec531794c3d01da3985c4ddf98506d04d9ff Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Fri, 16 Oct 2026 20:36:59 -0700
Subject: [PATCH 4/4] Wake the model when someone signs in

The model is deliberately not kept on around the clock. A successful
sign-in starts its cold boot in the background, so it is minutes along by
the time the person opens Insights, and it still scales to zero ten
minutes after the last request. A failed sign-in wakes nothing.
---
 docs/operating.md             | 10 ++++++----
 server/insights.py            | 10 ++++++++++
 server/server.py              |  2 ++
 server/tests/test_insights.py | 31 +++++++++++++++++++++++++++++++
 4 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/docs/operating.md b/docs/operating.md
index d46128b..7460c60 100644
--- a/docs/operating.md
+++ b/docs/operating.md
@@ -152,13 +152,15 @@ Modal's endpoint answers `503 no upstreams available` while a new one boots
 (about three minutes with the weights cached, and much longer when the volume
 reads slowly -- the class allows thirty and loads the shards with vLLM's
 parallel prefetch); that first refused request is what starts it. The server treats that 503 as "starting", not as a failure:
-opening the Insights page probes `/health` to wake the model, `/api/insights/
+a successful sign-in starts the model booting in the background (people are
+around; it still stops ten minutes after the last request), opening the
+Insights page probes `/health` and wakes it too, `/api/insights/
 status` reports `modelState` (`ready`/`starting`), a chat during the boot is a
 503 with `starting: true` that the page waits out (polling status every ten
 seconds, up to twenty minutes) before sending the question again, and the
-takeaways thread simply waits for `ready` before running. Keeping one
-container always on (`min_containers=1` on the Modal class) would remove the
-wait at roughly the cost of an H100 around the clock.
+takeaways thread simply waits for `ready` before running. The model is
+deliberately not kept on around the clock: it runs only after someone signs
+in, for as long as they keep using it.
 
 The alternative is the **Anthropic API**: leave `HOTOTEL_INSIGHTS_BASE_URL`
 unset and set `ANTHROPIC_API_KEY`; the model defaults to `claude-opus-5-5`.
diff --git a/server/insights.py b/server/insights.py
index 59a6f5c..d173760 100644
--- a/server/insights.py
+++ b/server/insights.py
@@ -359,6 +359,16 @@ def _probe(self):
         except (urllib.error.URLError, TimeoutError, OSError):
             return False
 
+    def wake(self):
+        """Start the model booting without waiting on it. Called when someone
+        signs in: the container only runs while people are around, so the
+        sign-in is the earliest moment to pay the cold start, and it is then
+        a few minutes along by the time they open Insights. It still scales
+        to zero after the idle window; nothing here keeps it up."""
+        if self.base_url:
+            threading.Thread(target=self.model_state, daemon=True,
+                             name="insights-wake").start()
+
     def wait_ready(self, timeout=MODEL_START_TIMEOUT):
         deadline = time.time() + timeout
         while time.time() < deadline:
diff --git a/server/server.py b/server/server.py
index bacb309..1eb3c20 100644
--- a/server/server.py
+++ b/server/server.py
@@ -2317,6 +2317,8 @@ def _handle_login(self):
             time.sleep(0.3)  # soften brute force
             self._redirect("/login?err=1&next=" + quote(nxt))
             return
+        if self.insights:
+            self.insights.wake()  # a person is here: get the model booting
         self._redirect(nxt, extra=[("Set-Cookie", self._session_cookie(token))])
 
     def _handle_register(self):
diff --git a/server/tests/test_insights.py b/server/tests/test_insights.py
index 4cd04bf..30d2455 100644
--- a/server/tests/test_insights.py
+++ b/server/tests/test_insights.py
@@ -111,10 +111,15 @@ def __init__(self):
 
     starting = False
 
+    woke = 0
+
     def status(self):
         return {"enabled": True, "model": self.model_name, "windowDays": 30,
                 "modelState": "starting" if self.starting else "ready"}
 
+    def wake(self):
+        self.woke += 1
+
     def chat(self, database_id, org_name, history):
         if self.starting:
             raise insights.ModelStarting()
@@ -260,6 +265,32 @@ class Err(Exception):
     print("engine: unreachable -> starting; 404 /health -> ready; 503 -> starting; cached ready trusted")
 
 
+def test_signing_in_wakes_the_model():
+    """The model runs only while people are around, so a sign-in is what
+    starts its cold boot; a failed sign-in must not."""
+    import urllib.parse
+    import urllib.request
+    engine = FakeEngine()
+    server.Handler.insights = engine
+    server.Handler._rate.clear()
+    try:
+        def post(pw):
+            body = urllib.parse.urlencode({"email": "ada@x.dev", "password": pw}).encode()
+            req = urllib.request.Request(BASE + "/login", data=body, method="POST",
+                                         headers={"Content-Type": "application/x-www-form-urlencoded"})
+            try:
+                urllib.request.urlopen(req, timeout=10).read()
+            except urllib.error.HTTPError:
+                pass
+        post("wrong-password")
+        assert engine.woke == 0, "a failed sign-in woke the model"
+        post("hunter2hunter2")
+        assert engine.woke == 1, engine.woke
+        print("a successful sign-in wakes the model once; a failed one does not")
+    finally:
+        server.Handler.insights = None
+
+
 def test_chat_refuses_everyone_but_a_cookie_session():
     server.Handler.insights = FakeEngine()
     try: