feat(voice): treat a voice message like any other message

lib/stack/ai/client.py

@@ -23,6 +23,7 @@
 
 import base64
 import os
+import re
 from dataclasses import dataclass
 
 import openai
@@ -245,6 +246,7 @@ async def complete(self, role: str, prompt: str, *,
             resp = await self._client.chat.completions.create(
                 model=model,
                 messages=[{"role": "user", "content": content}],
+                extra_body=_NO_THINKING,
                 **kwargs,
             )
         # Order matters: APITimeoutError < APIConnectionError, and
@@ -299,6 +301,23 @@ async def aclose(self) -> None:
         await self._client.close()
 
 
+# Every prompt famstack sends wants an answer, not a monologue: classify
+# this, summarise that, punctuate these words. Reasoning models default to
+# writing their deliberation into `content`, which at best burns tokens and
+# at worst *is* the output — a Qwen3 with thinking on answered the
+# transcript-polish prompt with a page of its own analysis and never
+# reached the transcript.
+#
+# `chat_template_kwargs` is the switch that actually works. Verified
+# against oMLX: `reasoning_effort`, a top-level `enable_thinking`, a
+# `reasoning` object and the `/no_think` prompt suffix all had no effect,
+# while this returned clean output with finish_reason=stop. vLLM, SGLang
+# and Ollama take the same key; servers that don't know it ignore unknown
+# body fields. Sent on every call rather than hidden behind a setting,
+# because there is no famstack prompt that wants the other behaviour.
+_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
+
+
 def _content_parts(prompt: str, images: list) -> list[dict]:
     """OpenAI-style multimodal content: one text part + N image_url parts.
 
@@ -451,27 +470,81 @@ async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg",
         raw = (getattr(resp, "text", "") or "").strip()
         if not raw or cleanup_with is None:
             return raw
-        return await self._cleanup(raw, cleanup_with)
+        return await self.polish(raw, cleanup_with)
 
     @staticmethod
-    async def _cleanup(raw: str, llm: "LLM") -> str:
-        """Run the LLM-cleanup pass; fall back to raw on any failure.
+    def _comparable(text: str) -> str:
+        """`text` reduced to what the polish is not allowed to change.
+
+        Letters and digits only, lowercased. Punctuation is dropped
+        rather than treated as a separator, because the polish is
+        *supposed* to add it — including inside words. Splitting on
+        non-word characters instead would read a restored apostrophe
+        ("theres" -> "there's") or a restored hyphen ("well known" ->
+        "well-known") as an added word and reject a correct polish.
+        That is not hypothetical: it rejected every real polish on the
+        first run against a live model.
+
+        The failures this exists to catch all move far more than a
+        boundary — reasoning leaking into the reply, a summary, a
+        truncation, reordered words — so comparing the letter stream
+        catches them while leaving legitimate punctuation alone.
+        """
+        return re.sub(r"[^a-z0-9]+", "", text.lower())
 
-        Cleanup is best-effort: if the LLM is down or the model returns
-        empty output, the caller still gets a usable transcript. We log
-        a warning so the admin can see drift between raw and clean if
-        they ever want to investigate model quality.
+    @staticmethod
+    async def polish(raw: str, llm: "LLM") -> str:
+        """Restore punctuation and sentence breaks in a raw transcript.
+
+        whisper.cpp emits one unbroken lowercase run of words. This pass
+        makes it readable again without changing what was said — the
+        prompt forbids adding, removing, reordering or rephrasing, so the
+        result is still verbatim. That restraint is deliberate: the
+        memories room holds things people said to their children, and a
+        model "improving" those is not a transcript any more.
+
+        Public because the polish and the transcription are separately
+        useful: a backfill keeps the raw text so a better model can
+        re-polish years of recordings without paying for whisper twice.
+
+        Best-effort: if the LLM is down or returns nothing, the caller
+        still gets a usable transcript. We log a warning so the admin can
+        see drift between raw and polished if they want to investigate
+        model quality.
         """
         try:
             cleaned = await llm.complete(
                 _CLEANUP_ROLE, _CLEANUP_PROMPT.format(raw=raw),
+                # A transformation, not a generation: the same words
+                # should always get the same punctuation. Sampling only
+                # invites the rephrasing the word check below rejects,
+                # which turns a paid model call into a raw transcript.
+                temperature=0.0,
             )
         except LLMError as e:
             logger.warning("[transcriber] cleanup failed, returning raw: {}", e)
             return raw
         cleaned = (cleaned or "").strip()
         # A model that returned nothing is no better than no model.
-        return cleaned or raw
+        if not cleaned:
+            return raw
+        # Verify rather than trust. The prompt permits punctuation,
+        # capitalization and line breaks and nothing else, so the word
+        # sequence is an exact, checkable invariant — and the failures
+        # worth catching all break it: reasoning leaking into `content`,
+        # a summary instead of a transcript, a polish truncated at the
+        # token limit, words reordered. Any of those would otherwise be
+        # written into the vault as what somebody said, with the audio
+        # as the only surviving copy of the truth.
+        raw_cmp, clean_cmp = Transcriber._comparable(raw), Transcriber._comparable(cleaned)
+        if raw_cmp != clean_cmp:
+            logger.warning(
+                "[transcriber] polish changed the words, keeping raw "
+                "(raw={} chars, polished={} chars)",
+                len(raw_cmp), len(clean_cmp),
+            )
+            return raw
+        return cleaned
 
     async def aclose(self) -> None:
         """Close the underlying HTTP client. Owned by us via from_env()."""

stacklets/ai/hooks/on_install.py

@@ -28,6 +28,13 @@
 OMLX_TAP_URL = "https://github.com/jundot/omlx"
 WHISPER_MODEL = "ggml-large-v3-turbo.bin"
 WHISPER_MODEL_URL = f"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/{WHISPER_MODEL}"
+# Floor for "this download finished". The model is ~1.5 GB; anything far
+# under that is a download the network cut short. Existence alone is not
+# enough: a half-file loads far enough to report "not all tensors loaded"
+# and then abort, so the install would keep saying "model ready" while
+# whisper died on every boot. Deliberately loose so a slightly smaller
+# re-quantisation upstream does not trip it.
+WHISPER_MODEL_MIN_BYTES = 1_000_000_000
 WHISPER_PORT = 42062
 PLIST_LABEL = "dev.famstack.whisper"
 
@@ -200,11 +207,25 @@ def _install_whisper(ctx, data_dir: Path, state_dir: Path):
 
     # Download model
     model_path = model_dir / WHISPER_MODEL
-    if not model_path.exists():
+    have = model_path.exists() and model_path.stat().st_size >= WHISPER_MODEL_MIN_BYTES
+    if not have:
+        if model_path.exists():
+            size_mb = model_path.stat().st_size // (1024 * 1024)
+            dim(f"  Existing model is only {size_mb} MB — re-downloading.")
+            model_path.unlink()
         ctx.step("Downloading Whisper model: large-v3-turbo (~1.5 GB)...")
         dim("  One-time download. OpenAI's distilled model — nearly")
         dim("  identical accuracy to large-v3, but 6x faster on Metal GPU.")
-        ctx.shell_live(f'curl -L --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"')
+        # -f so an HTTP error page is never written out as a model file.
+        ctx.shell_live(
+            f'curl -fL --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"'
+        )
+        if (not model_path.exists()
+                or model_path.stat().st_size < WHISPER_MODEL_MIN_BYTES):
+            raise RuntimeError(
+                "Whisper model download did not complete — run 'stack up ai' "
+                "again to resume. Voice transcription stays off until it does."
+            )
         done("Model downloaded")
     else:
         done("Whisper model ready (large-v3-turbo)")

stacklets/core/bot-runner/Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /app
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY main.py microbot.py room_context.py accounts.py ./
+COPY main.py microbot.py room_context.py accounts.py voice.py ./
 
 RUN adduser --disabled-password --uid 1000 botrunner
 RUN mkdir -p /botdata && chown botrunner:botrunner /botdata

stacklets/core/bot-runner/microbot.py

@@ -67,12 +67,15 @@
     LoginResponse,
     MegolmEvent,
     MessageDirection,
+    RoomMessage,
     RoomMessagesResponse,
     SyncResponse,
 )
 from nio.api import RelationshipType
 
+import voice
 from room_context import RoomContext, context_for
+from stack.ai.client import LLM, LLMError, LLMUnavailableError, Transcriber
 
 # The framework's "I picked this up and I'm working on it" signal. A
 # bot reacts with 👀 on the source message the moment it starts a
@@ -126,6 +129,24 @@ def __init__(self, homeserver: str, user_id: str, password: str, session_dir: st
         # via sync. The SyncResponse callback drains this set and fires
         # `on_room_joined` once nio has populated the room.
         self._pending_room_joins: set[str] = set()
+        # Speech-to-text for the decode in `_dispatch`. Built here rather
+        # than on demand because a missing whisper is a normal state, not
+        # an error: without it, audio simply stays audio and reaches no
+        # handler, the same as a message in a language we cannot read.
+        try:
+            self._transcriber = Transcriber.from_env(namespace=self.name)
+        except LLMUnavailableError as e:
+            logger.info(
+                "[{}] no transcription: {} — voice messages will be left "
+                "as audio", self.name, e,
+            )
+            self._transcriber = None
+        # Optional polish on raw whisper output (punctuation, sentence
+        # breaks). Absent, the transcript still lands, just rougher.
+        try:
+            self._transcript_cleanup = LLM.from_env(namespace=self.name)
+        except LLMUnavailableError:
+            self._transcript_cleanup = None
         self._client: AsyncClient | None = None
         # Lazily-created shared aiohttp session for non-nio HTTP (media
         # download, and subclasses' own API calls). Owned by the
@@ -493,13 +514,92 @@ async def _set_read_receipt(self, room_id: str, event_id: str | None) -> None:
                          self.name, room_id, e)
 
     async def _dispatch(self, room_id: str, event) -> None:
-        """Invoke every handler whose registered type matches the event."""
+        """Invoke every handler whose registered type matches the event.
+
+        Speech is decoded first, so what handlers match against is the
+        text of the message rather than its encoding.
+        """
         room = self.client.rooms.get(room_id)
         if room is None:
             return
+
+        decoding = voice.is_voice(event)
+        if decoding:
+            # Transcription is the one part of handling a message that can
+            # run for minutes, and it happens before any handler gets to
+            # signal it is working. A matched handler's wrap clears the
+            # indicator in its `finally`; we clear it ourselves on the two
+            # paths where no handler ever runs.
+            await self._set_typing(room_id, on=True)
+            event = await self._decode_voice(room_id, event)
+            if event is None:
+                await self._set_typing(room_id, on=False)
+                return
+
+        handled = False
         for event_type, handler in self._handlers:
             if isinstance(event, event_type):
+                handled = True
                 await handler(room, event)
+        if decoding and not handled:
+            await self._set_typing(room_id, on=False)
+
+    async def _decode_voice(self, room_id: str, event):
+        """Turn a voice message into the text event it is, or None.
+
+        None means the words could not be recovered — whisper is absent,
+        unreachable, or heard nothing. That is dispatched to nobody
+        rather than answered with an apology: the family can see their
+        own voice message sitting in the room, and a bot volunteering
+        "I could not hear that" in every room it is in, for audio nobody
+        was addressing to it, is noise.
+        """
+        if self._transcriber is None:
+            logger.debug(
+                "[{}] voice from {} left as audio (no whisper)",
+                self.name, event.sender,
+            )
+            return None
+
+        content = event.source.get("content", {})
+        url = content.get("url", "")
+        filename = content.get("filename") or content.get("body") or "voice.ogg"
+
+        async def run() -> dict:
+            audio = await self._download_media(url)
+            if not audio:
+                raise LLMError(f"could not download {url}")
+            raw = await self._transcriber.transcribe(audio, filename=filename)
+            # whisper emits one unbroken lowercase run of words; the polish
+            # pass puts the sentences back without changing them. Both are
+            # kept: polishing again with a better model is cheap, and
+            # transcribing again is not.
+            text = raw
+            if raw.strip() and self._transcript_cleanup is not None:
+                text = await Transcriber.polish(raw, self._transcript_cleanup)
+            return {"raw": raw, "text": text, "url": url, "filename": filename}
+
+        try:
+            transcript = (await voice.TRANSCRIPTS.run(event.event_id, run))["text"]
+        except LLMError as e:
+            logger.warning(
+                "[{}] transcription failed for {}: {}",
+                self.name, event.event_id, e,
+            )
+            return None
+        if not transcript.strip():
+            logger.info(
+                "[{}] no speech found in {}", self.name, event.event_id,
+            )
+            return None
+
+        logger.info(
+            "[{}] heard {} in {}: {}...",
+            self.name, event.sender, room_id, transcript[:80],
+        )
+        return RoomMessage.parse_event(
+            voice.transcribed_source(event.source, transcript)
+        )
 
     # ── Typing + error response ──────────────────────────────────────────
     #