Insights: keep a run inside the model's context window

modal/hototel_llm.py

@@ -103,9 +103,13 @@ def start(self):
             "--host", "0.0.0.0", "--port", str(VLLM_PORT),
             "--uvicorn-log-level=info",
             "--tensor-parallel-size", str(N_GPU),
-            # the agent's whole transcript -- system prompt, tool schemas, a
-            # dozen query results -- fits comfortably; more only costs KV cache
-            "--max-model-len", "32768",
+            # The agent's whole transcript -- system prompt, tool schemas and
+            # every query result of the run -- has to fit, plus the answer. A
+            # takeaways run overflowed 32k (25k of prompt once a dozen result
+            # sets were in it, with 8k reserved to answer), so 64k. The model
+            # supports 262k natively; KV cache for 64k is a few GB of the
+            # ~15GB left beside the weights on one H100.
+            "--max-model-len", "65536",
             "--gpu-memory-utilization", "0.92",
             # read the safetensors shards with parallel threads into the page
             # cache before mapping them: the volume is a network filesystem

server/insights.py

@@ -96,6 +96,10 @@
 - Prefer one well-shaped query (CTEs, window functions, GROUP BY) to many
   small ones. Call describe_tables only when the notes above do not answer a
   schema question.
+- Aggregate in SQL and keep results small: GROUP BY, ORDER BY and LIMIT so a
+  query returns the 10 to 30 rows that answer it, never raw session or
+  request rows and never SELECT *. Everything a query returns stays in your
+  context for the rest of the conversation, and the window is finite.
 - When a question is about "recently" or gives no window, use the last 30
   days and say so.
 - Name people by their email address and projects by their project name, as
@@ -389,9 +393,12 @@ def _model(self, effort):
         OpenAI-compatible endpoint gets the same request either way."""
         if self.base_url:
             from langchain_openai import ChatOpenAI
+            # max_tokens is reserved out of the context window up front, so
+            # it is sized to the longest answer (the takeaways JSON is ~2k),
+            # not to the window: 8000 plus a long transcript overflowed 32k
             return ChatOpenAI(model=self.model_name, base_url=self.base_url,
                               api_key=os.environ["HOTOTEL_INSIGHTS_API_KEY"],
-                              max_tokens=8000, timeout=300, max_retries=2)
+                              max_tokens=4096, timeout=300, max_retries=2)
         from langchain_anthropic import ChatAnthropic
         return ChatAnthropic(model=self.model_name, max_tokens=16000,
                              effort=effort, timeout=300, max_retries=2)
@@ -424,7 +431,10 @@ def _agent(self, database_id, effort):
             handle_errors=True,          # a bad query costs a turn, not the run
             describe_column_stats=False, # one aggregate per table, rarely useful here
             describe_search_capabilities=False,
-            max_rows=200)
+            # every returned row lives in the transcript until the run ends;
+            # a takeaways run with a dozen 200-row results overflowed the
+            # model's window, and the prompt now asks for 10-30 anyway
+            max_rows=80)
         model = self._model(effort)
         # No system_prompt here: ours carries the org name and today's date,
         # so _run sends it as the first message instead, which the Anthropic