From d551a88c4a2344c5154f552eac760db02e9c7da6 Mon Sep 17 00:00:00 2001
From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com>
Date: Sat, 17 Oct 2026 07:13:15 -0700
Subject: [PATCH] Insights: keep a run inside the model's context window

A takeaways run on the production org failed with the model's 400: 24,769
prompt tokens plus the 8,000 reserved for output exceeded the 32,768 limit.
Every query result stays in the transcript for the rest of the run, and a
dozen 200-row results add up.

Three changes: the model server's window is 64k (the model supports 262k;
KV cache for 64k is a few GB of what is left beside the weights), the
endpoint's max_tokens is 4,096 (the longest answer is the ~2k takeaways
JSON; the reservation is taken off the window up front), tool results are
capped at 80 rows, and the prompt asks the model to aggregate to the 10-30
rows that answer a question rather than return raw rows.
---
 modal/hototel_llm.py | 10 +++++++---
 server/insights.py   | 14 ++++++++++++--
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/modal/hototel_llm.py b/modal/hototel_llm.py
index 31946fc..5160338 100644
--- a/modal/hototel_llm.py
+++ b/modal/hototel_llm.py
@@ -103,9 +103,13 @@ def start(self):
             "--host", "0.0.0.0", "--port", str(VLLM_PORT),
             "--uvicorn-log-level=info",
             "--tensor-parallel-size", str(N_GPU),
-            # the agent's whole transcript -- system prompt, tool schemas, a
-            # dozen query results -- fits comfortably; more only costs KV cache
-            "--max-model-len", "32768",
+            # The agent's whole transcript -- system prompt, tool schemas and
+            # every query result of the run -- has to fit, plus the answer. A
+            # takeaways run overflowed 32k (25k of prompt once a dozen result
+            # sets were in it, with 8k reserved to answer), so 64k. The model
+            # supports 262k natively; KV cache for 64k is a few GB of the
+            # ~15GB left beside the weights on one H100.
+            "--max-model-len", "65536",
             "--gpu-memory-utilization", "0.92",
             # read the safetensors shards with parallel threads into the page
             # cache before mapping them: the volume is a network filesystem
diff --git a/server/insights.py b/server/insights.py
index d173760..3d51e76 100644
--- a/server/insights.py
+++ b/server/insights.py
@@ -96,6 +96,10 @@
 - Prefer one well-shaped query (CTEs, window functions, GROUP BY) to many
   small ones. Call describe_tables only when the notes above do not answer a
   schema question.
+- Aggregate in SQL and keep results small: GROUP BY, ORDER BY and LIMIT so a
+  query returns the 10 to 30 rows that answer it, never raw session or
+  request rows and never SELECT *. Everything a query returns stays in your
+  context for the rest of the conversation, and the window is finite.
 - When a question is about "recently" or gives no window, use the last 30
   days and say so.
 - Name people by their email address and projects by their project name, as
@@ -389,9 +393,12 @@ def _model(self, effort):
         OpenAI-compatible endpoint gets the same request either way."""
         if self.base_url:
             from langchain_openai import ChatOpenAI
+            # max_tokens is reserved out of the context window up front, so
+            # it is sized to the longest answer (the takeaways JSON is ~2k),
+            # not to the window: 8000 plus a long transcript overflowed 32k
             return ChatOpenAI(model=self.model_name, base_url=self.base_url,
                               api_key=os.environ["HOTOTEL_INSIGHTS_API_KEY"],
-                              max_tokens=8000, timeout=300, max_retries=2)
+                              max_tokens=4096, timeout=300, max_retries=2)
         from langchain_anthropic import ChatAnthropic
         return ChatAnthropic(model=self.model_name, max_tokens=16000,
                              effort=effort, timeout=300, max_retries=2)
@@ -424,7 +431,10 @@ def _agent(self, database_id, effort):
             handle_errors=True,          # a bad query costs a turn, not the run
             describe_column_stats=False, # one aggregate per table, rarely useful here
             describe_search_capabilities=False,
-            max_rows=200)
+            # every returned row lives in the transcript until the run ends;
+            # a takeaways run with a dozen 200-row results overflowed the
+            # model's window, and the prompt now asks for 10-30 anyway
+            max_rows=80)
         model = self._model(effort)
         # No system_prompt here: ours carries the org name and today's date,
         # so _run sends it as the first message instead, which the Anthropic