diff --git a/lib/stack/ai/client.py b/lib/stack/ai/client.py index 52bd5700..1ab1ee4d 100644 --- a/lib/stack/ai/client.py +++ b/lib/stack/ai/client.py @@ -23,6 +23,7 @@ import base64 import os +import re from dataclasses import dataclass import openai @@ -245,6 +246,7 @@ async def complete(self, role: str, prompt: str, *, resp = await self._client.chat.completions.create( model=model, messages=[{"role": "user", "content": content}], + extra_body=_NO_THINKING, **kwargs, ) # Order matters: APITimeoutError < APIConnectionError, and @@ -299,6 +301,23 @@ async def aclose(self) -> None: await self._client.close() +# Every prompt famstack sends wants an answer, not a monologue: classify +# this, summarise that, punctuate these words. Reasoning models default to +# writing their deliberation into `content`, which at best burns tokens and +# at worst *is* the output — a Qwen3 with thinking on answered the +# transcript-polish prompt with a page of its own analysis and never +# reached the transcript. +# +# `chat_template_kwargs` is the switch that actually works. Verified +# against oMLX: `reasoning_effort`, a top-level `enable_thinking`, a +# `reasoning` object and the `/no_think` prompt suffix all had no effect, +# while this returned clean output with finish_reason=stop. vLLM, SGLang +# and Ollama take the same key; servers that don't know it ignore unknown +# body fields. Sent on every call rather than hidden behind a setting, +# because there is no famstack prompt that wants the other behaviour. +_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}} + + def _content_parts(prompt: str, images: list) -> list[dict]: """OpenAI-style multimodal content: one text part + N image_url parts. @@ -451,27 +470,81 @@ async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", raw = (getattr(resp, "text", "") or "").strip() if not raw or cleanup_with is None: return raw - return await self._cleanup(raw, cleanup_with) + return await self.polish(raw, cleanup_with) @staticmethod - async def _cleanup(raw: str, llm: "LLM") -> str: - """Run the LLM-cleanup pass; fall back to raw on any failure. + def _comparable(text: str) -> str: + """`text` reduced to what the polish is not allowed to change. + + Letters and digits only, lowercased. Punctuation is dropped + rather than treated as a separator, because the polish is + *supposed* to add it — including inside words. Splitting on + non-word characters instead would read a restored apostrophe + ("theres" -> "there's") or a restored hyphen ("well known" -> + "well-known") as an added word and reject a correct polish. + That is not hypothetical: it rejected every real polish on the + first run against a live model. + + The failures this exists to catch all move far more than a + boundary — reasoning leaking into the reply, a summary, a + truncation, reordered words — so comparing the letter stream + catches them while leaving legitimate punctuation alone. + """ + return re.sub(r"[^a-z0-9]+", "", text.lower()) - Cleanup is best-effort: if the LLM is down or the model returns - empty output, the caller still gets a usable transcript. We log - a warning so the admin can see drift between raw and clean if - they ever want to investigate model quality. + @staticmethod + async def polish(raw: str, llm: "LLM") -> str: + """Restore punctuation and sentence breaks in a raw transcript. + + whisper.cpp emits one unbroken lowercase run of words. This pass + makes it readable again without changing what was said — the + prompt forbids adding, removing, reordering or rephrasing, so the + result is still verbatim. That restraint is deliberate: the + memories room holds things people said to their children, and a + model "improving" those is not a transcript any more. + + Public because the polish and the transcription are separately + useful: a backfill keeps the raw text so a better model can + re-polish years of recordings without paying for whisper twice. + + Best-effort: if the LLM is down or returns nothing, the caller + still gets a usable transcript. We log a warning so the admin can + see drift between raw and polished if they want to investigate + model quality. """ try: cleaned = await llm.complete( _CLEANUP_ROLE, _CLEANUP_PROMPT.format(raw=raw), + # A transformation, not a generation: the same words + # should always get the same punctuation. Sampling only + # invites the rephrasing the word check below rejects, + # which turns a paid model call into a raw transcript. + temperature=0.0, ) except LLMError as e: logger.warning("[transcriber] cleanup failed, returning raw: {}", e) return raw cleaned = (cleaned or "").strip() # A model that returned nothing is no better than no model. - return cleaned or raw + if not cleaned: + return raw + # Verify rather than trust. The prompt permits punctuation, + # capitalization and line breaks and nothing else, so the word + # sequence is an exact, checkable invariant — and the failures + # worth catching all break it: reasoning leaking into `content`, + # a summary instead of a transcript, a polish truncated at the + # token limit, words reordered. Any of those would otherwise be + # written into the vault as what somebody said, with the audio + # as the only surviving copy of the truth. + raw_cmp, clean_cmp = Transcriber._comparable(raw), Transcriber._comparable(cleaned) + if raw_cmp != clean_cmp: + logger.warning( + "[transcriber] polish changed the words, keeping raw " + "(raw={} chars, polished={} chars)", + len(raw_cmp), len(clean_cmp), + ) + return raw + return cleaned async def aclose(self) -> None: """Close the underlying HTTP client. Owned by us via from_env().""" diff --git a/stacklets/ai/hooks/on_install.py b/stacklets/ai/hooks/on_install.py index 3481abe3..ad63a66b 100644 --- a/stacklets/ai/hooks/on_install.py +++ b/stacklets/ai/hooks/on_install.py @@ -28,6 +28,13 @@ OMLX_TAP_URL = "https://github.com/jundot/omlx" WHISPER_MODEL = "ggml-large-v3-turbo.bin" WHISPER_MODEL_URL = f"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/{WHISPER_MODEL}" +# Floor for "this download finished". The model is ~1.5 GB; anything far +# under that is a download the network cut short. Existence alone is not +# enough: a half-file loads far enough to report "not all tensors loaded" +# and then abort, so the install would keep saying "model ready" while +# whisper died on every boot. Deliberately loose so a slightly smaller +# re-quantisation upstream does not trip it. +WHISPER_MODEL_MIN_BYTES = 1_000_000_000 WHISPER_PORT = 42062 PLIST_LABEL = "dev.famstack.whisper" @@ -200,11 +207,25 @@ def _install_whisper(ctx, data_dir: Path, state_dir: Path): # Download model model_path = model_dir / WHISPER_MODEL - if not model_path.exists(): + have = model_path.exists() and model_path.stat().st_size >= WHISPER_MODEL_MIN_BYTES + if not have: + if model_path.exists(): + size_mb = model_path.stat().st_size // (1024 * 1024) + dim(f" Existing model is only {size_mb} MB — re-downloading.") + model_path.unlink() ctx.step("Downloading Whisper model: large-v3-turbo (~1.5 GB)...") dim(" One-time download. OpenAI's distilled model — nearly") dim(" identical accuracy to large-v3, but 6x faster on Metal GPU.") - ctx.shell_live(f'curl -L --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"') + # -f so an HTTP error page is never written out as a model file. + ctx.shell_live( + f'curl -fL --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"' + ) + if (not model_path.exists() + or model_path.stat().st_size < WHISPER_MODEL_MIN_BYTES): + raise RuntimeError( + "Whisper model download did not complete — run 'stack up ai' " + "again to resume. Voice transcription stays off until it does." + ) done("Model downloaded") else: done("Whisper model ready (large-v3-turbo)") diff --git a/stacklets/core/bot-runner/Dockerfile b/stacklets/core/bot-runner/Dockerfile index 7f0c8bcf..6b0cf8d7 100644 --- a/stacklets/core/bot-runner/Dockerfile +++ b/stacklets/core/bot-runner/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py microbot.py room_context.py accounts.py ./ +COPY main.py microbot.py room_context.py accounts.py voice.py ./ RUN adduser --disabled-password --uid 1000 botrunner RUN mkdir -p /botdata && chown botrunner:botrunner /botdata diff --git a/stacklets/core/bot-runner/microbot.py b/stacklets/core/bot-runner/microbot.py index 01a31a3c..ff5e2ed2 100644 --- a/stacklets/core/bot-runner/microbot.py +++ b/stacklets/core/bot-runner/microbot.py @@ -67,12 +67,15 @@ LoginResponse, MegolmEvent, MessageDirection, + RoomMessage, RoomMessagesResponse, SyncResponse, ) from nio.api import RelationshipType +import voice from room_context import RoomContext, context_for +from stack.ai.client import LLM, LLMError, LLMUnavailableError, Transcriber # The framework's "I picked this up and I'm working on it" signal. A # bot reacts with 👀 on the source message the moment it starts a @@ -126,6 +129,24 @@ def __init__(self, homeserver: str, user_id: str, password: str, session_dir: st # via sync. The SyncResponse callback drains this set and fires # `on_room_joined` once nio has populated the room. self._pending_room_joins: set[str] = set() + # Speech-to-text for the decode in `_dispatch`. Built here rather + # than on demand because a missing whisper is a normal state, not + # an error: without it, audio simply stays audio and reaches no + # handler, the same as a message in a language we cannot read. + try: + self._transcriber = Transcriber.from_env(namespace=self.name) + except LLMUnavailableError as e: + logger.info( + "[{}] no transcription: {} — voice messages will be left " + "as audio", self.name, e, + ) + self._transcriber = None + # Optional polish on raw whisper output (punctuation, sentence + # breaks). Absent, the transcript still lands, just rougher. + try: + self._transcript_cleanup = LLM.from_env(namespace=self.name) + except LLMUnavailableError: + self._transcript_cleanup = None self._client: AsyncClient | None = None # Lazily-created shared aiohttp session for non-nio HTTP (media # download, and subclasses' own API calls). Owned by the @@ -493,13 +514,92 @@ async def _set_read_receipt(self, room_id: str, event_id: str | None) -> None: self.name, room_id, e) async def _dispatch(self, room_id: str, event) -> None: - """Invoke every handler whose registered type matches the event.""" + """Invoke every handler whose registered type matches the event. + + Speech is decoded first, so what handlers match against is the + text of the message rather than its encoding. + """ room = self.client.rooms.get(room_id) if room is None: return + + decoding = voice.is_voice(event) + if decoding: + # Transcription is the one part of handling a message that can + # run for minutes, and it happens before any handler gets to + # signal it is working. A matched handler's wrap clears the + # indicator in its `finally`; we clear it ourselves on the two + # paths where no handler ever runs. + await self._set_typing(room_id, on=True) + event = await self._decode_voice(room_id, event) + if event is None: + await self._set_typing(room_id, on=False) + return + + handled = False for event_type, handler in self._handlers: if isinstance(event, event_type): + handled = True await handler(room, event) + if decoding and not handled: + await self._set_typing(room_id, on=False) + + async def _decode_voice(self, room_id: str, event): + """Turn a voice message into the text event it is, or None. + + None means the words could not be recovered — whisper is absent, + unreachable, or heard nothing. That is dispatched to nobody + rather than answered with an apology: the family can see their + own voice message sitting in the room, and a bot volunteering + "I could not hear that" in every room it is in, for audio nobody + was addressing to it, is noise. + """ + if self._transcriber is None: + logger.debug( + "[{}] voice from {} left as audio (no whisper)", + self.name, event.sender, + ) + return None + + content = event.source.get("content", {}) + url = content.get("url", "") + filename = content.get("filename") or content.get("body") or "voice.ogg" + + async def run() -> dict: + audio = await self._download_media(url) + if not audio: + raise LLMError(f"could not download {url}") + raw = await self._transcriber.transcribe(audio, filename=filename) + # whisper emits one unbroken lowercase run of words; the polish + # pass puts the sentences back without changing them. Both are + # kept: polishing again with a better model is cheap, and + # transcribing again is not. + text = raw + if raw.strip() and self._transcript_cleanup is not None: + text = await Transcriber.polish(raw, self._transcript_cleanup) + return {"raw": raw, "text": text, "url": url, "filename": filename} + + try: + transcript = (await voice.TRANSCRIPTS.run(event.event_id, run))["text"] + except LLMError as e: + logger.warning( + "[{}] transcription failed for {}: {}", + self.name, event.event_id, e, + ) + return None + if not transcript.strip(): + logger.info( + "[{}] no speech found in {}", self.name, event.event_id, + ) + return None + + logger.info( + "[{}] heard {} in {}: {}...", + self.name, event.sender, room_id, transcript[:80], + ) + return RoomMessage.parse_event( + voice.transcribed_source(event.source, transcript) + ) # ── Typing + error response ────────────────────────────────────────── # diff --git a/stacklets/core/bot-runner/voice.py b/stacklets/core/bot-runner/voice.py new file mode 100644 index 00000000..d8cf9854 --- /dev/null +++ b/stacklets/core/bot-runner/voice.py @@ -0,0 +1,217 @@ +"""Voice messages are messages. The transport decodes them, not the bots. + +Someone holding the mic button and someone typing are doing the same +thing: putting words in the room. Only the encoding differs, so only the +transport should know about it. By the time a handler runs, an `m.audio` +event has already become an ordinary text event — same sender, same event +id, same thread — and every gate a typed message passes through (room +mode, thread ownership, mention, routing) applies to it unchanged. + +This is deliberately not a capability bots reach for. That was the earlier +design, and it grew an audio branch in every consumer, two whisper clients +in one process, and three places that could transcribe the same bytes. +Decoding belongs beside decryption: below everyone, done once. + +What survives the decode is provenance. Whisper is lossy in a way a +keyboard is not, so the text event carries a `dev.famstack.transcript` +block naming the audio it came from. Routing ignores it; the bot that +acts on the words uses it to echo them back, which is the only way a +mistranscription is visible without opening the vault. The audio itself +stays on the timeline as the reproducibility anchor (ADR-010) — we never +need to keep a second copy of the bytes. + +The module owns the pure half: recognising a voice event and rewriting its +raw source dict. `MicroBot._dispatch` owns the I/O half (download, +whisper, share). +""" + +from __future__ import annotations + +import asyncio +import base64 +import copy +import json +import os +import time +from collections.abc import Awaitable, Callable +from pathlib import Path + +from loguru import logger + + +# Provenance on a decoded message: which audio event these words came +# from. Its presence is the answer to "were these words guessed at by a +# machine?" — a question the reply layer asks and the routing layer must not. +TRANSCRIPT_KEY = "dev.famstack.transcript" + +# Fields of the audio event that describe the payload rather than the +# message. They move into the provenance block; leaving them on a text +# event would let a consumer treat it as an upload and re-file the bytes. +_PAYLOAD_FIELDS = ("url", "file", "info", "filename") + + +def is_voice(event) -> bool: + """Whether this timeline event is speech we can decode. + + Any `m.audio` with a plain mxc payload counts. We do not try to tell + a voice memo from a music file first: whisper answering "no words + here" is a better detector than a guess at the sender's intent, and + the clients families use mark both the same way. + + Encrypted media (`file` rather than `url`) is deliberately not + claimed. The framework already tells encrypted rooms it cannot read + them, so saying yes here would only produce a download that fails. + """ + content = (getattr(event, "source", None) or {}).get("content") or {} + if content.get("msgtype") != "m.audio": + return False + return bool(content.get("url")) + + +def was_transcribed(content: dict) -> bool: + """Whether a message's words were transcribed rather than typed. + + Reads the framework contract, so it works for any consumer without + knowing which component decoded the audio. Consumers ask this to + decide whether to show the words back for checking, never to decide + routing — routing is the whole thing that must not care. + """ + return isinstance(content.get(TRANSCRIPT_KEY), dict) + + +def transcribed_source(source: dict, transcript: str) -> dict: + """Rewrite a voice event's raw source dict as the text event it is. + + Identity is preserved wholesale — event id, sender, timestamp, + thread relation — because this is the same message, read aloud + rather than typed. Bots reply to it, react on it and claim thread + ownership of it by that identity, so a synthesised id would detach + every one of those from the message the family can actually see. + """ + out = copy.deepcopy(source) + content = out.setdefault("content", {}) + + provenance = {"source_msgtype": content.get("msgtype", "m.audio")} + if url := content.get("url"): + provenance["url"] = url + if filename := (content.get("filename") or content.get("body")): + provenance["filename"] = filename + info = content.get("info") + if isinstance(info, dict): + if (duration := info.get("duration")) is not None: + provenance["duration"] = duration + if mimetype := info.get("mimetype"): + provenance["mimetype"] = mimetype + + for field in _PAYLOAD_FIELDS: + content.pop(field, None) + content["msgtype"] = "m.text" + content["body"] = transcript + content[TRANSCRIPT_KEY] = provenance + return out + + +class TranscriptStore: + """Transcripts on disk, keyed by Matrix event id. + + Transcription is the most expensive thing the stack does per message: + minutes of GPU for a long memo. Three callers want the same answer and + must not each pay for it. + + * Every bot in the room drains the same timeline in the same + process, so each reaches the same audio independently. + * The drain is at-least-once, so a handler that dies mid-flight + brings its event back around. + * A backfill walks a room's whole history in a *separate* process. + The memories room holds years of recordings; re-transcribing that + because the answer was only ever in RAM is not acceptable once, + let alone every time something wants to read it. + + So the store is durable and shared, and the in-flight map on top of it + collapses concurrent askers onto one run. One file per voice message, + written atomically, because the backfill and the bot runner are + different processes writing the same directory. + + Each record keeps the raw whisper output next to the polished text. + Polishing is cheap and improves with better models; whisper is not and + does not. Keeping both means years of recordings can be re-polished + without touching the audio again. + """ + + def __init__(self, path: str | Path | None = None): + self.path = Path( + path or os.environ.get("TRANSCRIPT_DIR", "/data/core/transcripts") + ) + self._inflight: dict[str, asyncio.Future] = {} + + # ── Durable half ───────────────────────────────────────────────── + + def _file(self, event_id: str) -> Path: + # Event ids carry `$`, `/` and `+`; base64url keeps one file per + # id without inventing a collision-prone slug. + name = base64.urlsafe_b64encode(event_id.encode()).decode().rstrip("=") + return self.path / f"{name}.json" + + def read(self, event_id: str) -> dict | None: + """The stored record for `event_id`, or None if we never ran it.""" + try: + return json.loads(self._file(event_id).read_text()) + except FileNotFoundError: + return None + except (OSError, ValueError) as e: + logger.warning("[voice] unreadable transcript for {}: {}", event_id, e) + return None + + def write(self, event_id: str, record: dict) -> None: + """Persist a record. A store we cannot write is not fatal — the + transcript still reaches the handler, it just costs again later.""" + target = self._file(event_id) + try: + self.path.mkdir(parents=True, exist_ok=True) + tmp = target.with_suffix(".tmp") + tmp.write_text(json.dumps(record, ensure_ascii=False, indent=2)) + os.replace(tmp, target) + except OSError as e: + logger.warning("[voice] could not store transcript {}: {}", event_id, e) + + # ── Single-flight half ─────────────────────────────────────────── + + async def run( + self, event_id: str, produce: Callable[[], Awaitable[dict]], + ) -> dict: + """The record for `event_id`, produced at most once across callers. + + A failure is never remembered. The drain is at-least-once and + whisper outages are transient, so caching an error would turn a + restartable service into a permanently silent message. + """ + if (stored := self.read(event_id)) is not None: + return stored + if (pending := self._inflight.get(event_id)) is not None: + return await pending + + loop = asyncio.get_running_loop() + future: asyncio.Future = loop.create_future() + self._inflight[event_id] = future + try: + record = await produce() + except BaseException as e: + self._inflight.pop(event_id, None) + if not future.done(): + future.set_exception(e) + # Retrieve it so an unawaited future does not warn. + future.exception() + raise + record.setdefault("at", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + # Publish before releasing the slot, so a caller arriving in + # between finds the answer rather than starting a second run. + self.write(event_id, record) + if not future.done(): + future.set_result(record) + self._inflight.pop(event_id, None) + return record + + +# Process-global: the point is to share across the bots in this runner, +# and with whatever process backfills history. +TRANSCRIPTS = TranscriptStore() diff --git a/stacklets/core/docker-compose.yml b/stacklets/core/docker-compose.yml index b7a2d9ca..959e3565 100644 --- a/stacklets/core/docker-compose.yml +++ b/stacklets/core/docker-compose.yml @@ -52,6 +52,7 @@ services: - ../../stacklets/core/bot-runner/microbot.py:/app/microbot.py:ro - ../../stacklets/core/bot-runner/room_context.py:/app/room_context.py:ro - ../../stacklets/core/bot-runner/accounts.py:/app/accounts.py:ro + - ../../stacklets/core/bot-runner/voice.py:/app/voice.py:ro # Per-stacklet bot data (sessions, crypto stores) - ${DATA_DIR}:/data # API is TCP on host.docker.internal:42001 — no socket mount needed diff --git a/stacklets/core/stacklet.toml b/stacklets/core/stacklet.toml index 182eae51..b8cb9156 100644 --- a/stacklets/core/stacklet.toml +++ b/stacklets/core/stacklet.toml @@ -45,6 +45,11 @@ STACK_ADMIN_USER_IDS = "{admin_user_ids}" OPENAI_URL = "{ai_openai_url_docker}" OPENAI_KEY = "{ai_openai_key}" WHISPER_URL = "{ai_whisper_url_docker}/audio/transcriptions" +# Transcripts are a cache over the audio, not a record: the mxc on the +# timeline stays the anchor. Kept on disk because whisper costs minutes +# of GPU per memo and a backfill over the memories room must never pay +# for the same recording twice. +TRANSCRIPT_DIR = "/data/core/transcripts" # Paperless — document search for archivist bot and tools server PAPERLESS_URL = "http://stack-docs-paperless:8000" diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py index f7c11c37..232bedc9 100644 --- a/stacklets/docs/bot/archivist.py +++ b/stacklets/docs/bot/archivist.py @@ -50,6 +50,7 @@ from capture_tags import CaptureTagCache from extractors import TextExtractor, UrlExtractor from git_mirror import GitMirror +import voice from microbot import CHECK, CROSS, EYES, MicroBot from pdf_analysis import ( DEFAULT_REFORMAT_MAX_PDF_PAGES, @@ -65,10 +66,7 @@ from stack.email_message import defang_links from stack.links import go_docs, go_topic, public from stack.ai.client import ( - LLMError, - LLMUnavailableError, ModelCapabilities, - Transcriber, ) # Make sibling stacklets importable. In the bot-runner container, @@ -161,7 +159,7 @@ def _llm_error_for_chat( # ── Constants ──────────────────────────────────────────────────────────────── -SUPPORTED_MSGTYPES = {"m.file", "m.image", "m.audio"} +SUPPORTED_MSGTYPES = {"m.file", "m.image"} SCAN_BEGIN = {"scan", "("} SCAN_END = {"done", "fertig", ")"} @@ -223,8 +221,6 @@ def _guess_mime(filename: str, msgtype: str) -> str: return _MIME_BY_EXT[ext] if msgtype == "m.image": return "image/jpeg" - if msgtype == "m.audio": - return "audio/ogg" return "application/octet-stream" @@ -301,10 +297,6 @@ def __init__(self, homeserver, user_id, password, session_dir, **settings): self.link_base_url = os.environ.get("LINK_BASE_URL", "") self.openai_url = os.environ.get("OPENAI_URL", "") self.openai_key = os.environ.get("OPENAI_KEY", "") - # Voice transcription endpoint. When unset (e.g. the AI stacklet - # isn't installed yet), the archivist still files PDFs/images; only - # the audio-capture path becomes a soft-skip with a friendly reply. - self.whisper_url = os.environ.get("WHISPER_URL", "") self.language = os.environ.get("LANGUAGE", "en") # Per-bot settings from stacklet.toml [bots.archivist.settings] self.classify_enabled = settings.get("classify", True) @@ -378,7 +370,6 @@ def __init__(self, homeserver, user_id, password, session_dir, **settings): self._pipeline: DocumentPipeline | None = None self._search: SearchService | None = None self._capture: CapturePipeline | None = None - self._transcriber: Transcriber | None = None self._vault: VaultContext | None = None self._paperless_version: str = "" @@ -474,27 +465,6 @@ async def start(self) -> None: shared_bucket=self.shared_bucket, vault=self._vault, ) - # Whisper speaks /v1/audio/transcriptions on its own port, so the - # Transcriber gets its own client. When WHISPER_URL is unset the - # archivist still works for PDFs/images; only the voice-capture - # branch in the pipeline soft-skips. - if self.whisper_url: - try: - self._transcriber = Transcriber.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning("[archivist] no transcription: {}", e) - self._transcriber = None - else: - logger.info( - "[archivist] WHISPER_URL unset — voice messages will be ignored " - "(set up AI with 'stack up ai' to enable transcription)" - ) - - # The capture pipeline borrows the classifier's framework LLM for - # transcript cleanup -- punctuating raw whisper output with the - # model that's already running. When the classifier isn't built - # (AI not configured), cleanup soft-skips and the raw transcript - # falls through. capture_llm = self._classifier.llm if self._classifier is not None else None self._capture = CapturePipeline( url_extractor=self._url_extractor, @@ -508,7 +478,6 @@ async def start(self) -> None: capture_keep_body=self.capture_keep_body, capture_tag_prompt_size=self.capture_tag_prompt_size, vision_max_pdf_pages=self.vision_max_pdf_pages, - transcriber=self._transcriber, llm=capture_llm, ) # Warm the vision-capability cache on every boot. Previously @@ -1508,17 +1477,6 @@ async def _on_file(self, room, event) -> None: if msgtype not in SUPPORTED_MSGTYPES: return - # Voice messages depend on the optional whisper service. When the - # transcriber wasn't wired (WHISPER_URL unset or whisper missing - # at boot), silently ignore audio so the bot still files PDFs and - # images normally; the startup log already warned the admin. - if msgtype == "m.audio" and self._transcriber is None: - logger.debug( - "[archivist] ignoring voice from {} (transcription disabled)", - event.sender, - ) - return - url = content.get("url", "") if not url or not url.startswith("mxc://"): return @@ -1552,20 +1510,12 @@ async def _on_file(self, room, event) -> None: display_name = _clean_filename(raw_filename, msgtype) reply_to = event.event_id - # Multi-page scan / multi-message batch mode. PDFs and images - # take the existing page-accumulator path; voice memos take the - # transcription-and-accumulate path so a single `(` ... `)` - # session can collect either kind (or both, filed as separate - # captures on close). + # Multi-page scan mode: accumulate pages until `)` closes the + # session and the batch is filed as one document. if event.sender in self._scan_sessions: - if msgtype == "m.audio": - await self._handle_voice_batch_message( - room.room_id, event, url, raw_filename, - ) - else: - await self._handle_scan_page( - room.room_id, event, url, raw_filename, caption, - ) + await self._handle_scan_page( + room.room_id, event, url, raw_filename, caption, + ) return file_data = await self._download_media(url) @@ -1592,11 +1542,7 @@ async def _on_file(self, room, event) -> None: # capture room: PDFs and images become visual bookmarks in the # sender's bucket; Matrix already stores the binary, so we link # to the mxc URL and don't re-archive the bytes. - # - # Voice memos always take the capture path, even in #documents: - # a transcript belongs in the sender's own notes, not in Paperless - # alongside scanned invoices and IDs. - if self._is_documents_room(ctx) and msgtype != "m.audio": + if self._is_documents_room(ctx): await self._process_document( room.room_id, raw_filename, display_name, file_data, reply_to, date_filed=self._event_date(event), @@ -1660,6 +1606,11 @@ async def _on_text(self, room, event: RoomMessageText) -> None: return query_lower = query.lower() reply_to = event.event_id + # Whether these words were spoken rather than typed. Used twice + # below, and for nothing else: never to route, only to decide + # whether the message was handed to us deliberately and whether + # to quote it back for checking. + transcribed = voice.was_transcribed(event.source.get("content", {})) ctx = self._room_context(room) # First-encounter welcome -- idempotent, gated by a per-room @@ -1738,8 +1689,7 @@ async def _on_text(self, room, event: RoomMessageText) -> None: elif (begin := _split_scan_command(query, SCAN_BEGIN))[0]: sender_name = event.sender.split(":")[0].replace("@", "").capitalize() self._scan_sessions[event.sender] = { - "files": [], "voice_inputs": [], - "room_id": room.room_id, "caption": begin[1], + "files": [], "room_id": room.room_id, "caption": begin[1], } await self._send(room.room_id, self.t("scan_started", sender=sender_name), reply_to) @@ -1811,7 +1761,9 @@ async def _on_text(self, room, event: RoomMessageText) -> None: sender=event.sender, ) - elif self._looks_like_paste(query) and self._count_humans_in_room(room) < 2: + elif transcribed or ( + self._looks_like_paste(query) and self._count_humans_in_room(room) < 2 + ): # One person in the room, paste-shaped message, no mention → # file as text capture. Nobody is being talked *to* here, so # a long message is material dropped for us to keep. @@ -1826,10 +1778,21 @@ async def _on_text(self, room, event: RoomMessageText) -> None: # # Links and files are unaffected in either case: pasting a URL # or dropping a PDF is a deliberate act, not a turn in a - # conversation. + # conversation. A voice memo is the same kind of act: nobody + # holds the mic button for thirty seconds as a turn in a chat, + # and the length heuristic that stands in for deliberateness + # elsewhere cannot read it. So speech carries itself past this + # gate, in any room, exactly as an upload does. + # + # This is not speech being routed differently. Everything + # above still applies to it unchanged -- a memo addressed to + # the bot is a question, in the documents room it is a search, + # in a thread it obeys ownership. Only "was this handed to us + # on purpose?" is answered another way, because for speech + # there is another way to answer it. await self._handle_text_capture( room.room_id, query, event.sender, reply_to, - capture_id=event.event_id, + capture_id=event.event_id, transcribed=transcribed, ) else: @@ -2153,82 +2116,21 @@ async def _handle_scan_page( # the timeline clean; the scan-complete reply is the closure. await self._react(room_id, reply_to, EYES) - async def _handle_voice_batch_message( - self, room_id: str, event, url: str, raw_filename: str, - ): - """Transcribe a voice memo and stash it on the open batch session. - - Mirrors _handle_scan_page: download, normalise input, append to - the session, ack the count. The transcript is computed eagerly - (with the LLM cleanup pass) so by the time `)` arrives we just - concatenate ready text -- no extra round-trip on close. - """ - reply_to = event.event_id - if self._transcriber is None: - await self._send( - room_id, self.t("scan_voice_no_transcriber"), reply_to, - ) - return - try: - audio = await self._download_media(url) - except Exception as e: - await self._send( - room_id, self.t("scan_voice_failed", error=str(e)), reply_to, - ) - return - if not audio: - await self._send( - room_id, self.t("scan_voice_failed_matrix"), reply_to, - ) - return - - cleanup_llm = self._classifier.llm if self._classifier is not None else None - try: - transcript = await self._transcriber.transcribe( - audio, filename=raw_filename or "voice.ogg", - cleanup_with=cleanup_llm, - ) - except LLMError as e: - logger.warning("[archivist] batch voice transcribe failed: {}", e) - await self._send( - room_id, self.t("scan_voice_failed", error=str(e)), reply_to, - ) - return - - if not transcript.strip(): - await self._send(room_id, self.t("scan_voice_empty"), reply_to) - return - - session = self._scan_sessions[event.sender] - session["voice_inputs"].append({ - "transcript": transcript, - "mxc": url, - "event_id": event.event_id, - }) - # 👀 acknowledges the memo landed in the batch; the scan-complete - # reply is the closure (mirrors _handle_scan_page). - await self._react(room_id, reply_to, EYES) - async def _handle_scan_complete( self, room_id: str, sender: str, reply_to: str | None = None, *, date_filed: str | None = None, ): session = self._scan_sessions.pop(sender) files = session["files"] - voice_inputs = session.get("voice_inputs") or [] caption = session.get("caption", "").strip() sender_name = sender.split(":")[0].replace("@", "").capitalize() # Nothing accumulated -- the user opened a session and closed it # without sending anything. Treat as cancelled, same as today. - if not files and not voice_inputs: + if not files: await self._send(room_id, self.t("scan_cancelled"), reply_to) return - # Files (PDFs / images) take the existing PDF combine path. A - # mixed batch files both: the PDF as a document and the voice - # memos as a separate note below, since the vault data model - # has one body per capture. if files: if len(files) == 1: filename, file_data = files[0] @@ -2267,40 +2169,6 @@ async def _handle_scan_complete( submitter_mxid=sender, ) - if voice_inputs: - await self._handle_voice_batch_complete( - room_id, sender, voice_inputs, reply_to, - ) - - async def _handle_voice_batch_complete( - self, room_id: str, sender: str, voice_inputs: list[dict], - reply_to: str | None, - ): - """File the accumulated voice memos as one combined note. - - Each input already carries a cleaned transcript (the LLM pass - ran at message time), so this just hands the list to the - capture pipeline and renders the resulting reply. We post a - progress line first because the classify call is the slowest - step and the sender deserves to know we're working. - """ - n = len(voice_inputs) - await self._send( - room_id, self.t("scan_voice_complete", count=n), reply_to, - ) - binding = await self._topic_binding( - self._room_by_id(room_id), sender, - ) - outcome = await self._capture.capture_voice_batch( - transcripts=[v["transcript"] for v in voice_inputs], - primary_mxc=voice_inputs[0].get("mxc"), - sender_mxid=sender, - capture_id=voice_inputs[0].get("event_id"), - seed_topics=binding.seed_topics if binding else None, - bucket=binding.bucket if binding else None, - ) - await self._reply_for_capture(room_id, outcome, reply_to) - # ── URL capture (knowledge rooms, DMs, per-person notes rooms) ─────── # # Capture mode is what runs when the bot sees a URL in any room that @@ -2353,6 +2221,7 @@ async def _handle_capture( async def _handle_text_capture( self, room_id: str, text: str, sender_mxid: str, reply_to: str | None = None, *, capture_id: str | None = None, + transcribed: bool = False, ) -> None: # 👀 before the work, like every other capture shape: a link gets # it from the pipeline's notifier, an upload from the handler. A @@ -2370,6 +2239,7 @@ async def _handle_text_capture( capture_id=capture_id, seed_topics=binding.seed_topics if binding else None, bucket=binding.bucket if binding else None, + transcribed=transcribed, ) await self._reply_for_capture(room_id, outcome, reply_to) @@ -2447,7 +2317,6 @@ async def _reply_for_capture( # back to the original URL-shaped message for legacy callers. failure_keys = { "url": "capture_failed", - "transcription": "capture_failed_transcription", "binary": "capture_failed_binary", } key = failure_keys.get(o.failure_reason or "", "capture_failed") diff --git a/stacklets/docs/bot/capture_pipeline.py b/stacklets/docs/bot/capture_pipeline.py index 856cb677..d3a5b0bc 100644 --- a/stacklets/docs/bot/capture_pipeline.py +++ b/stacklets/docs/bot/capture_pipeline.py @@ -41,7 +41,6 @@ LLMUnavailableError, ) from stack import resolve_model -from stack.ai.client import LLMError try: from pypdf import PdfReader # type: ignore @@ -65,11 +64,6 @@ } TEXT_EXTS = {"md", "markdown", "txt"} -# Voice memos: when whisper is wired, the transcript IS the note body. -# Element and most Matrix clients upload Opus-in-OGG; the other formats -# show up when people attach a file picker rather than the in-app recorder. -AUDIO_MIME_PREFIX = "audio/" - # Ratio that the reformatted body must be of the input for the # reformat to count as a useful rewrite. A well-behaved reformat # tightens whitespace and stitches broken lines but keeps essentially @@ -131,7 +125,6 @@ def __init__( capture_keep_body: bool, capture_tag_prompt_size: int, vision_max_pdf_pages: int = DEFAULT_VISION_MAX_PDF_PAGES, - transcriber=None, llm=None, ): self._url_extractor = url_extractor @@ -145,13 +138,6 @@ def __init__( self.capture_keep_body = capture_keep_body self.capture_tag_prompt_size = capture_tag_prompt_size self.vision_max_pdf_pages = vision_max_pdf_pages - # Optional: when None, audio uploads soft-skip with extract_failed - # so the bot tells the sender it can't transcribe right now. - self._transcriber = transcriber - # Optional: when present, the transcriber uses it to polish raw - # whisper output (add punctuation + sentence breaks). The same - # LLM the classifier already runs on -- no extra HTTP client. - # Missing -> raw transcript falls through, never a hard failure. self._llm = llm async def capture_url( @@ -205,23 +191,32 @@ async def capture_text( capture_id: str | None = None, seed_topics: list[str] | None = None, bucket: str | None = None, + transcribed: bool = False, ) -> CaptureOutcome: - """File a pasted body as a note. Nothing is fetched — the text is + """File a message body as a note. Nothing is fetched — the text is the source; TextExtractor surfaces any embedded URL as the link. - The body is always kept (the user typed those exact bytes). + The body is always kept (those are the exact words sent). ``seed_topics`` and ``bucket`` carry the topic-room guarantees through; see ``capture_url``. + + ``transcribed`` marks a body that whisper produced rather than a + keyboard. It changes nothing about how the note is filed — only + whether the reply quotes the words back, which is the sender's + one chance to catch a mishearing. """ source = await self._text_extractor.extract(text) if source is None: return CaptureOutcome(status="empty") + # The footer says where a capture came from. "(pasted text)" is a + # lie about something somebody said out loud. + origin = "(voice message)" if transcribed else "(pasted text)" return await self._publish( source=source, kind="note", sender_mxid=sender_mxid, - display_link=source.source_uri or "(pasted text)", + display_link=source.source_uri or origin, actor=sender_mxid, capture_id=capture_id, seed_topics=seed_topics, - bucket=bucket, + bucket=bucket, transcribed=transcribed, ) async def capture_email( @@ -267,48 +262,6 @@ async def capture_email( default_person=False, ) - async def capture_voice_batch( - self, *, - transcripts: list[str], - primary_mxc: str | None, - sender_mxid: str, - capture_id: str | None = None, - seed_topics: list[str] | None = None, - bucket: str | None = None, - ) -> CaptureOutcome: - """File N voice memos as a single combined note. - - Used by the `( ... )` batch flow: each voice memo has already - been transcribed (and LLM-cleaned) on arrival, so this just - concatenates them with paragraph breaks and ships the result - through the standard publish path. ``primary_mxc`` is the - first memo's media URL -- the vault note links back to the - start of the conversation; other memos stay discoverable via - chat scrollback. - - Marking the SourceContent as ``audio/ogg`` makes ``_publish`` - echo the combined transcript in the capture reply, same as - single-memo captures, so the sender sees the whole batch's - text quoted back to them. - """ - bodies = [t.strip() for t in transcripts if t and t.strip()] - if not bodies: - return CaptureOutcome(status="empty") - combined = "\n\n".join(bodies) - source = SourceContent( - text=combined, - mime="audio/ogg", - title_hint=None, - source_uri=primary_mxc, - ) - return await self._publish( - source=source, kind="note", sender_mxid=sender_mxid, - display_link=primary_mxc or "(voice batch)", - actor=sender_mxid, - capture_id=capture_id, seed_topics=seed_topics, - bucket=bucket, - ) - async def capture_binary( self, *, file_data: bytes, @@ -333,21 +286,13 @@ async def capture_binary( wiki entry links back to the original binary -- we don't re-store the bytes; Matrix already has them. """ - is_audio = bool(mime and mime.startswith(AUDIO_MIME_PREFIX)) - if is_audio: - source, images, kind = await self._source_from_audio( - file_data=file_data, mime=mime, filename=filename, - source_uri=source_uri, - ) - else: - source, images, kind = self._source_from_binary( - file_data=file_data, mime=mime, filename=filename, - source_uri=source_uri, - ) + source, images, kind = self._source_from_binary( + file_data=file_data, mime=mime, filename=filename, + source_uri=source_uri, + ) if source is None: return CaptureOutcome( - status="extract_failed", - failure_reason="transcription" if is_audio else "binary", + status="extract_failed", failure_reason="binary", ) source = self._cap_pdf_body(source) source = await self._maybe_reformat_pdf(source, kind) @@ -422,49 +367,6 @@ async def _maybe_reformat_pdf( source_uri=source.source_uri, ) - async def _source_from_audio( - self, *, file_data: bytes, mime: str, filename: str, - source_uri: str | None, - ) -> tuple[SourceContent | None, list[ImageAttachment], str]: - """Transcribe a voice memo into a note-shaped SourceContent. - - The transcript IS the note body: same kind ("note") that md/txt - uploads use, so the classifier sees the transcribed text and the - mirror writes the words verbatim. Returning ``None`` signals the - capture as extract_failed -- the orchestrator already renders a - friendly reply for that case, so a missing/down whisper service - surfaces uniformly across PDFs and audio. - """ - if self._transcriber is None: - logger.info( - "[capture] audio dropped: no transcriber configured " - "(WHISPER_URL unset?)" - ) - return (None, [], "note") - try: - transcript = await self._transcriber.transcribe( - file_data, filename=filename or "voice.ogg", - cleanup_with=self._llm, - ) - except LLMError as e: - # Same failure shape the LLM client uses for chat outages; - # the orchestrator already maps extract_failed to a user- - # visible "I couldn't process that" reply. - logger.warning("[capture] transcription failed: {}", e) - return (None, [], "note") - if not transcript.strip(): - return (None, [], "note") - return ( - SourceContent( - text=transcript, - mime=mime, - title_hint=filename or None, - source_uri=source_uri, - ), - [], - "note", - ) - def _source_from_binary( self, *, file_data: bytes, mime: str, filename: str, source_uri: str | None, @@ -651,6 +553,7 @@ async def _publish( bucket: str | None = None, email_meta: dict | None = None, default_person: bool = True, + transcribed: bool = False, ) -> CaptureOutcome: """Shared tail: classify, mirror, record tags, return the outcome. @@ -777,12 +680,11 @@ async def _publish( if user_hint: envelope["data"]["user_hint"] = user_hint - # Voice captures echo the transcript back to the sender so they - # can spot a mistranscription without opening the vault. Other - # capture shapes (PDFs, images, URLs, notes) leave it None. - source_mime = getattr(source, "mime", None) or "" - is_audio_source = source_mime.startswith(AUDIO_MIME_PREFIX) - transcript = source.text if is_audio_source else None + # Transcribed captures echo the words back to the sender so they + # can spot a mishearing without opening the vault. The caller says + # so; by the time the text reaches here it is an ordinary note, + # indistinguishable from a typed one, which is the point. + transcript = source.text if transcribed else None return CaptureOutcome( status="reclassified" if reclassified else "captured", diff --git a/stacklets/docs/bot/cli/capture.py b/stacklets/docs/bot/cli/capture.py index 04632996..0b6102ae 100644 --- a/stacklets/docs/bot/cli/capture.py +++ b/stacklets/docs/bot/cli/capture.py @@ -177,7 +177,6 @@ def render_receipt(outcome) -> str: what = { "url": "that link", - "transcription": "that voice memo", "binary": "that file", }.get(getattr(outcome, "failure_reason", None), "the content") return f"Nothing captured: could not read {what}." diff --git a/stacklets/docs/bot/messages/archivist.yml b/stacklets/docs/bot/messages/archivist.yml index fef7d140..ce326967 100644 --- a/stacklets/docs/bot/messages/archivist.yml +++ b/stacklets/docs/bot/messages/archivist.yml @@ -71,11 +71,6 @@ en: scan_complete_single: "\U0001F4CB Scan complete (1 page) — analyzing..." scan_complete_multi: "\U0001F4CB Scan complete — combining {count} pages into PDF..." scan_combine_failed: "\u274C Failed to combine pages into PDF: {error}" - scan_voice_failed: "\u274C Failed to download voice memo: {error}" - scan_voice_failed_matrix: "\u274C Failed to download voice memo from Matrix." - scan_voice_no_transcriber: "\U0001F3A4 Voice transcription is not configured. Set up AI with 'stack up ai'." - scan_voice_empty: "\U0001F3A4 Couldn't hear anything in that voice memo." - scan_voice_complete: "\U0001F3A4 Combining {count} voice memos into one note. Analyzing..." no_active_scan: "No active scan. Send ( or 'scan' to start one." # URL archiving @@ -89,7 +84,6 @@ en: # URL capture (knowledge rooms — links become summarized notes, not Paperless docs) capture_failed: "\u274C Couldn't read that link. Either the host is unreachable or there's no article body to extract." - capture_failed_transcription: "\U0001F3A4 Couldn't transcribe that voice memo. Whisper may be down or misconfigured." capture_failed_binary: "\u274C Couldn't read that file." capture_llm_failed: "\U0001F4DD Captured the page, but classification skipped: {error}" capture_no_mirror: "\u274C Captures need the `code` stacklet up so I can write to the memory vault. Run `stack up code`." @@ -280,11 +274,6 @@ de: scan_complete_single: "\U0001F4CB Scan abgeschlossen (1 Seite) — wird analysiert..." scan_complete_multi: "\U0001F4CB Scan abgeschlossen — {count} Seiten werden zu PDF zusammengefasst..." scan_combine_failed: "\u274C Seiten konnten nicht zu PDF zusammengefasst werden: {error}" - scan_voice_failed: "\u274C Sprachnachricht konnte nicht heruntergeladen werden: {error}" - scan_voice_failed_matrix: "\u274C Sprachnachricht konnte nicht aus Matrix heruntergeladen werden." - scan_voice_no_transcriber: "\U0001F3A4 Sprach-Transkription ist nicht eingerichtet. 'stack up ai' ausführen." - scan_voice_empty: "\U0001F3A4 Konnte nichts in der Sprachnachricht hören." - scan_voice_complete: "\U0001F3A4 Kombiniere {count} Sprachnachrichten zu einer Notiz. Wird analysiert..." no_active_scan: "Kein aktiver Scan. Sende ( oder 'scan' um einen zu starten." # URL archiving @@ -298,7 +287,6 @@ de: # URL-Capture (Wissensräume — Links werden zu zusammengefassten Notizen, nicht zu Paperless-Dokumenten) capture_failed: "\u274C Konnte den Link nicht lesen. Entweder ist die Seite nicht erreichbar, oder es gibt keinen Artikel-Inhalt zum Extrahieren." - capture_failed_transcription: "\U0001F3A4 Konnte die Sprachnachricht nicht transkribieren. Whisper ist möglicherweise nicht erreichbar oder falsch konfiguriert." capture_failed_binary: "\u274C Konnte diese Datei nicht lesen." capture_llm_failed: "\U0001F4DD Seite gespeichert, aber Klassifizierung übersprungen: {error}" capture_no_mirror: "\u274C Captures brauchen das `code`-Stacklet, damit ich in den Memory-Vault schreiben kann. Starte `stack up code`." diff --git a/stacklets/messages/bot/bot.toml b/stacklets/messages/bot/bot.toml index 90727671..dbbae6d7 100644 --- a/stacklets/messages/bot/bot.toml +++ b/stacklets/messages/bot/bot.toml @@ -1,3 +1,6 @@ +# Deprecated. Transcription moved into the framework; this bot exists only +# to tell rooms it was invited to that it is going away, and then leave. +# Remove this declaration and scribe.py one release after it ships. id = "scribe-bot" name = "Scribe" -description = "Transcribes voice messages" +description = "Retired — voice messages are transcribed automatically now" diff --git a/stacklets/messages/bot/scribe.py b/stacklets/messages/bot/scribe.py index 3a0d99d1..3dc022be 100644 --- a/stacklets/messages/bot/scribe.py +++ b/stacklets/messages/bot/scribe.py @@ -1,103 +1,101 @@ -"""Scribe — voice message transcription bot. +"""Scribe — retired, and saying so. -Send a voice message in any room where Scribe is present, and it replies -with the transcribed text. Uses whisper.cpp running natively on the host -for Metal GPU acceleration — a 5-minute voice memo transcribes in ~20 -seconds, vs 10+ minutes CPU-only inside Docker. +Transcribing voice messages used to be this bot's whole job. It is the +transport's job now: a voice message is decoded before any handler sees +it, so every bot in every room gets the words without asking, and nothing +needs Scribe to be present. -The transcription API call is delegated to the shared `Transcriber` -capability on the AI client; the bot is just the Matrix-side wiring — -download audio, hand bytes to the Transcriber, post the result. +This shell exists for one release only, because the framework has no way +to deprovision a bot that goes away. Deleting the declaration stops the +runner from launching it, but the Matrix account survives — still joined +to whatever rooms it was invited to, listed as a member, answering +nothing, forever. Somebody would eventually go looking for why. + +So instead of vanishing it explains itself once per room and leaves. The +people this reaches are the only ones it can have affected: Scribe +declared no room of its own, so it was never in a room unless a person +went and invited it by hand. Those are exactly the people who would +notice it go quiet. + +Delete this file and its bot.toml one release after it ships. """ +import asyncio +import os + from loguru import logger -from nio import ( - AsyncClient, - RoomMessageAudio, -) +from nio import AsyncClient from microbot import MicroBot -from stack.ai.client import LLM, LLMError, LLMUnavailableError, Transcriber + + +# Kept inline rather than in a message catalogue: this is two strings with +# a known expiry, and a catalogue would outlive the bot that uses it. +_GOODBYE = { + "en": ( + "**Voice messages are transcribed automatically now.**\n\n" + "famstack does it for every room, so you no longer need me here. " + "Keep sending voice messages exactly as you always have. They are " + "transcribed the moment they arrive, and the other bots read the " + "words rather than the recording.\n\n" + "Nothing is lost and there is nothing to set up. I am leaving this " + "room; you can remove my account whenever you like." + ), + "de": ( + "**Sprachnachrichten werden jetzt automatisch transkribiert.**\n\n" + "famstack übernimmt das für jeden Raum, ihr braucht mich hier also " + "nicht mehr. Schickt Sprachnachrichten weiter wie bisher. Sie " + "werden sofort transkribiert, und die anderen Bots lesen den Text " + "statt der Aufnahme.\n\n" + "Es geht nichts verloren und es ist nichts einzurichten. Ich " + "verlasse diesen Raum; mein Konto könnt ihr jederzeit löschen." + ), +} class ScribeBot(MicroBot): name = "scribe-bot" - def __init__(self, homeserver, user_id, password, session_dir, **config): - super().__init__(homeserver, user_id, password, session_dir, **config) - # The Transcriber owns the HTTP client; we don't have a clean - # shutdown hook in the MicroBot base, so we accept that the - # underlying client lives for the lifetime of the bot process. - # - # When WHISPER_URL isn't set (e.g. the AI stacklet hasn't been - # installed yet) the constructor would otherwise crash here. We - # degrade to a no-op bot instead: still in the room, still - # joinable, but silent on voice messages until whisper is wired. - try: - self._transcriber = Transcriber.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning( - "[scribe] transcription disabled: {} — " - "voice messages will be ignored until WHISPER_URL is set", - e, - ) - self._transcriber = None + def register_callbacks(self, client: AsyncClient) -> None: + """Register nothing, and start the retirement sweep. - # The LLM is optional: when present, it polishes raw whisper - # output into a punctuated paragraph (the Transcriber owns the - # prompt). When absent, scribe still posts the raw transcript -- - # less readable but never a blocker. - try: - self._llm = LLM.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning( - "[scribe] transcript cleanup disabled: {} — " - "voice messages will post the raw whisper output", - e, + No message handlers at all: this bot answers nothing. The sweep + runs on every launch rather than once, so a room it could not + leave (homeserver hiccup, lost network) is retried next boot + instead of keeping a silent member forever. + + Scheduled as a task because `register_callbacks` is sync and runs + inside `start()`'s event loop, after the initial sync has + populated `client.rooms`. + """ + asyncio.create_task(self.retire_everywhere()) + + async def on_room_joined(self, room_id: str) -> None: + """Someone followed an older guide and invited it. Same answer, + so an invite never leaves a silent member behind.""" + await self._retire_from(room_id) + + async def retire_everywhere(self) -> None: + """Say goodbye in every room this account is still in, and leave.""" + room_ids = list(self.client.rooms.keys()) + if room_ids: + logger.info( + "[{}] retiring from {} room(s)", self.name, len(room_ids), ) - self._llm = None + for room_id in room_ids: + await self._retire_from(room_id) - def register_callbacks(self, client: AsyncClient) -> None: - self.add_event_callback(self._on_voice, RoomMessageAudio) - - async def _on_voice(self, room, event: RoomMessageAudio) -> None: - if event.sender == self.user_id: - return - if self._transcriber is None: - # The startup warning already told the admin why; don't - # spam a per-message error reply or a typing indicator that - # leads nowhere. - return - - logger.info("[scribe] Voice from {} in {}", event.sender, room.room_id) - await self._set_typing(room.room_id, on=True) - - audio = await self._download_media(event.url) - if audio is None: - await self._set_typing(room.room_id, on=False) - logger.error("[scribe] Download failed for {}", event.url) - return - - filename = event.body if event.body else "voice.ogg" - text = "" + async def _retire_from(self, room_id: str) -> None: + """Explain, then leave. A failure to leave is logged and dropped: + the goodbye still landed, and the next launch tries again.""" + lang = os.environ.get("LANGUAGE", "en") + await self._send( + room_id, _GOODBYE.get(lang, _GOODBYE["en"]), msgtype="m.notice", + ) try: - text = await self._transcriber.transcribe( - audio, filename=filename, cleanup_with=self._llm, - ) - except LLMError as e: - # The Transcriber maps every transport / API failure to an - # LLMError; we log and fall through to the empty-text branch - # so the family sees a friendly reply instead of silence. - logger.error("[scribe] Transcription failed: {}", e) - await self._set_typing(room.room_id, on=False) - - if text: - logger.info("[scribe] Transcribed: {}...", text[:80]) - await self._send( - room.room_id, f"**Transcription:**\n\n{text}", - reply_to=event.event_id, - ) - else: - await self._send( - room.room_id, "Sorry, I couldn't transcribe that audio.", + await self.client.room_leave(room_id) + except Exception as e: + logger.warning( + "[{}] could not leave {}: {} — will retry on next launch", + self.name, room_id, e, ) diff --git a/stacklets/messages/cli/upload.py b/stacklets/messages/cli/upload.py index e695ba7d..8e5128b5 100644 --- a/stacklets/messages/cli/upload.py +++ b/stacklets/messages/cli/upload.py @@ -14,7 +14,8 @@ The user's password is read from the secrets store (the same place the account was created), so the sender shows up correctly in the timeline and the archivist's "received from " reply. Images (.png/.jpg) are sent -as m.image; everything else as m.file. +as m.image, audio (.ogg/.opus/.m4a/.mp3/.wav) as m.audio so it is treated +as a voice message and transcribed; everything else as m.file. """ HELP = "Upload a file to a chat room" @@ -27,7 +28,10 @@ from _matrix import MatrixClient, resolve_login # Extension → (mime type, Matrix msgtype). Images go as m.image so the -# archivist takes its vision path; everything else is m.file. +# archivist takes its vision path. Audio goes as m.audio, which is what a +# phone's mic button sends, so the framework transcribes it and the bots +# see the words — the only way to exercise voice from the terminal. +# Everything else is m.file. _MIME = { ".pdf": ("application/pdf", "m.file"), ".png": ("image/png", "m.image"), @@ -36,6 +40,11 @@ ".webp": ("image/webp", "m.image"), ".gif": ("image/gif", "m.image"), ".txt": ("text/plain", "m.file"), + ".ogg": ("audio/ogg", "m.audio"), + ".opus": ("audio/ogg", "m.audio"), + ".m4a": ("audio/mp4", "m.audio"), + ".mp3": ("audio/mpeg", "m.audio"), + ".wav": ("audio/wav", "m.audio"), } diff --git a/tests/framework/test_ai_client.py b/tests/framework/test_ai_client.py index 3e80f72e..42bcf247 100644 --- a/tests/framework/test_ai_client.py +++ b/tests/framework/test_ai_client.py @@ -554,9 +554,11 @@ def __init__(self, result: str = "Cleaned text.", self.result = result self.error = error self.calls: list[tuple[str, str]] = [] + self.kwargs: list[dict] = [] - async def complete(self, role: str, prompt: str) -> str: + async def complete(self, role: str, prompt: str, **kw) -> str: self.calls.append((role, prompt)) + self.kwargs.append(kw) if self.error is not None: raise self.error return self.result @@ -634,3 +636,163 @@ async def test_empty_cleanup_response_falls_back_to_raw(self, httpserver: HTTPSe tr = _make_transcriber(httpserver) assert await tr.transcribe(b"audio", cleanup_with=empty_llm) == "raw words present" await tr.aclose() + + +class TestPolishKeepsTheWords: + """The polish pass may add punctuation, capitalization and sentence + breaks. It may not change what was said. + + This is not a theoretical guard. Pointed at a reasoning model with + thinking left on, the cleanup prompt came back with a page of the + model's own deliberation and no transcript at all. Without a check, + that monologue is what gets written into the family's vault in place + of the words somebody actually spoke — and the audio it came from is + the only remaining copy. + + So the polish is verified against its own contract rather than + trusted: same words, same order, or we keep the raw transcript. + """ + + async def test_punctuation_and_capitalization_are_accepted( + self, httpserver: HTTPServer, + ): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "hey i forgot to renew the boiler"}) + llm = _StubLLM(result="Hey, I forgot to renew the boiler.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "Hey, I forgot to renew the boiler." + ) + await tr.aclose() + + async def test_a_model_thinking_out_loud_is_rejected( + self, httpserver: HTTPServer, + ): + """The failure that prompted this: reasoning leaks into content.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite for july"}) + llm = _StubLLM(result=( + "Here's a thinking process:\n\n1. **Analyze User Input:** The " + "user wants punctuation restored. Let me check the constraints." + )) + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "book the campsite for july" + ) + await tr.aclose() + + async def test_a_summary_is_rejected(self, httpserver: HTTPServer): + """Dropping words is the quiet failure: it reads fine and is wrong.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({ + "text": "remember to book the campsite by the lake for july", + }) + llm = _StubLLM(result="Book the campsite.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "remember to book the campsite by the lake for july" + ) + await tr.aclose() + + async def test_a_truncated_polish_is_rejected(self, httpserver: HTTPServer): + """Hitting the token limit mid-sentence loses the tail silently.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "one two three four five six"}) + llm = _StubLLM(result="One, two, three.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "one two three four five six" + ) + await tr.aclose() + + async def test_reordered_words_are_rejected(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "marge called about bart"}) + llm = _StubLLM(result="Bart called about Marge.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "marge called about bart" + ) + await tr.aclose() + + async def test_whitespace_and_line_breaks_are_accepted( + self, httpserver: HTTPServer, + ): + """whisper emits segment-per-line with leading spaces; rejoining + those into paragraphs is exactly what the polish is for.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite.\n by the lake."}) + llm = _StubLLM(result="Book the campsite, by the lake.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "Book the campsite, by the lake." + ) + await tr.aclose() + + +class TestPolishIsDeterministic: + """Polishing is a transformation, not a generation: the same words in + should give the same punctuation out. Sampling only invites the model + to rephrase, which the word check then rejects — so a run that costs a + model call ends up returning the raw transcript anyway.""" + + async def test_the_cleanup_call_pins_temperature(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite"}) + llm = _StubLLM(result="Book the campsite.") + tr = _make_transcriber(httpserver) + await tr.transcribe(b"a", cleanup_with=llm) + assert llm.kwargs[0]["temperature"] == 0.0 + await tr.aclose() + + +class TestPolishMayAddPunctuationInsideWords: + """Regression: the first word-level check rejected every real polish. + + Whisper writes "theres" and "well known"; restoring the apostrophe and + the hyphen is punctuation, which the prompt permits. A check that + split on non-word characters counted those as new words and threw the + polish away, so the feature silently did nothing against a live model + while every stubbed test passed. + """ + + async def test_a_restored_apostrophe_is_accepted(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "theres decorations in the loft"}) + llm = _StubLLM(result="There's decorations in the loft.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "There's decorations in the loft." + ) + await tr.aclose() + + async def test_a_restored_hyphen_is_accepted(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "its a well known campsite"}) + llm = _StubLLM(result="It's a well-known campsite.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "It's a well-known campsite." + ) + await tr.aclose() + + async def test_a_changed_word_is_still_rejected(self, httpserver: HTTPServer): + """The loosening must not swallow an actual edit.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "theres decorations in the loft"}) + llm = _StubLLM(result="There are decorations in the attic.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "theres decorations in the loft" + ) + await tr.aclose() diff --git a/tests/stacklets/test_archivist_corrections.py b/tests/stacklets/test_archivist_corrections.py index 8bd9d9b8..d8b7ab21 100644 --- a/tests/stacklets/test_archivist_corrections.py +++ b/tests/stacklets/test_archivist_corrections.py @@ -189,7 +189,8 @@ async def _capture_reprocess(room_id, vault_path, user_hint, sender_mxid, reply_to, *, initial_classification=None): bot.routed.append(("capture_reprocess", vault_path, user_hint)) - async def _text_capture(room_id, text, sender, reply_to=None, *, capture_id=None): + async def _text_capture(room_id, text, sender, reply_to=None, *, + capture_id=None, transcribed=False): bot.routed.append(("capture_text", text)) async def _send(room_id, text, *a, **kw): diff --git a/tests/stacklets/test_archivist_routing.py b/tests/stacklets/test_archivist_routing.py index af70d3fe..12055b4a 100644 --- a/tests/stacklets/test_archivist_routing.py +++ b/tests/stacklets/test_archivist_routing.py @@ -27,6 +27,7 @@ sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "core" / "bot-runner")) sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "docs" / "bot")) +import voice # noqa: E402 from archivist import ArchivistBot # noqa: E402 @@ -712,6 +713,51 @@ def test_only_quoted_block_returns_empty(self): # ── Mention as routing signal ───────────────────────────────────────────── +@pytest.fixture +def bot_with_recorder(tmp_path): + """Build a bot wired with stubs that record handler dispatches. + + Avoids any I/O. The handlers under test (_handle_search, + _handle_capture, _handle_text_capture) are replaced with + recorders so we can assert which branch fired with which query. + """ + bot = _build_bot(tmp_path) + calls: list[tuple[str, str]] = [] + + async def _record_search(room_id, query, reply_to=None, *, sender=None): + calls.append(("search", query)) + + async def _record_capture(room_id, url, sender, reply_to=None, + *, capture_id=None): + calls.append(("capture_url", url)) + + async def _record_text_capture(room_id, text, sender, reply_to=None, + *, capture_id=None, transcribed=False): + calls.append(("capture_text", text)) + + async def _record_url(room_id, url, reply_to=None, **kw): + calls.append(("paperless_url", url)) + + async def _record_send(*a, **kw): + calls.append(("send", a[1] if len(a) > 1 else "")) + + bot._handle_search = _record_search + bot._handle_capture = _record_capture + bot._handle_text_capture = _record_text_capture + bot._handle_url = _record_url + bot._send = _record_send + # Correction lookup needs the client; short-circuit it. What it + # resolves is pinned in test_archivist_corrections.py. + bot._correction_anchor = lambda *_a, **_kw: _none_coro() + # The per-room welcome path runs ahead of routing decisions in + # `_on_text` / `_on_file`. These tests focus on the routing + # dispatch, not the welcome -- stub it out so the recorded + # calls list stays clean. The welcome itself has its own test + # file: tests/stacklets/test_archivist_welcome.py. + bot._send_room_welcome_if_needed = lambda *_a, **_kw: _none_coro() + return bot, calls + + class TestMentionRoutesToSearch: """Mention promotes any free-text message to a search query, in any room. The mxid is stripped first so command matching and search see @@ -744,49 +790,6 @@ def _room_obj(*, alias="#family-chat:server", members=None): users={uid: object() for uid in (members or [BOT_ID, "@homer:server", "@marge:server"])}, ) - @pytest.fixture - def bot_with_recorder(self, tmp_path): - """Build a bot wired with stubs that record handler dispatches. - - Avoids any I/O. The handlers under test (_handle_search, - _handle_capture, _handle_text_capture) are replaced with - recorders so we can assert which branch fired with which query. - """ - bot = _build_bot(tmp_path) - calls: list[tuple[str, str]] = [] - - async def _record_search(room_id, query, reply_to=None, *, sender=None): - calls.append(("search", query)) - - async def _record_capture(room_id, url, sender, reply_to=None, - *, capture_id=None): - calls.append(("capture_url", url)) - - async def _record_text_capture(room_id, text, sender, reply_to=None, - *, capture_id=None): - calls.append(("capture_text", text)) - - async def _record_url(room_id, url, reply_to=None, **kw): - calls.append(("paperless_url", url)) - - async def _record_send(*a, **kw): - calls.append(("send", a[1] if len(a) > 1 else "")) - - bot._handle_search = _record_search - bot._handle_capture = _record_capture - bot._handle_text_capture = _record_text_capture - bot._handle_url = _record_url - bot._send = _record_send - # Correction lookup needs the client; short-circuit it. What it - # resolves is pinned in test_archivist_corrections.py. - bot._correction_anchor = lambda *_a, **_kw: _none_coro() - # The per-room welcome path runs ahead of routing decisions in - # `_on_text` / `_on_file`. These tests focus on the routing - # dispatch, not the welcome -- stub it out so the recorded - # calls list stays clean. The welcome itself has its own test - # file: tests/stacklets/test_archivist_welcome.py. - bot._send_room_welcome_if_needed = lambda *_a, **_kw: _none_coro() - return bot, calls @pytest.mark.asyncio async def test_mention_in_group_room_routes_to_search(self, bot_with_recorder): @@ -913,3 +916,97 @@ async def _none_coro(): async def _resp(event): """A `room_get_event` response wrapping one event.""" return SimpleNamespace(event=event) + + +class TestSpokenMessagesAreDeliberate: + """A voice memo is a deliberate handover, the way a link or an upload + is. Holding the mic button for thirty seconds is not a turn in a + conversation, so the ambient-capture gate that silences long *typed* + messages in a shared room must not silence it. + + Routing is otherwise identical to typed text — that is the whole + point of decoding speech in the transport. What differs is only the + "is this material or chatter?" heuristic, which is the same judgement + the gate already makes in favour of links and files. + """ + + @staticmethod + def _event(body, *, transcribed, sender="@homer:server"): + content: dict = {} + if transcribed: + content[voice.TRANSCRIPT_KEY] = { + "source_msgtype": "m.audio", + "url": "mxc://server/abc", + } + return SimpleNamespace( + body=body, sender=sender, event_id="$evt:server", + server_timestamp=1, source={"content": content}, + ) + + @staticmethod + def _shared_room(): + """A topic room with two humans in it — the shape that closes the + ambient gate for typed text.""" + return SimpleNamespace( + room_id="!room:server", + canonical_alias="#topic-camping:server", + name="Topic: Camping", + users={uid: object() for uid in + (BOT_ID, "@homer:server", "@marge:server")}, + ) + + # Comfortably under the 100-char paste threshold, so nothing but the + # spoken-ness can be carrying it past the gate. + SHORT = "Book the campsite for July." + + @pytest.mark.asyncio + async def test_a_memo_is_captured_in_a_room_with_other_people( + self, bot_with_recorder, + ): + bot, calls = bot_with_recorder + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=True), + ) + assert calls == [("capture_text", self.SHORT)] + + @pytest.mark.asyncio + async def test_the_same_words_typed_stay_chatter(self, bot_with_recorder): + """The contrast that makes the rule honest: it is the deliberate + act being recognised, not the words.""" + bot, calls = bot_with_recorder + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=False), + ) + assert calls == [] + + @pytest.mark.asyncio + async def test_the_capture_is_told_the_words_were_transcribed( + self, bot_with_recorder, + ): + """So the reply quotes them back — the sender's only chance to + catch whisper mishearing them without opening the vault.""" + bot, seen = bot_with_recorder + flags: list[bool] = [] + + async def _record(room_id, text, sender, reply_to=None, *, + capture_id=None, transcribed=False): + flags.append(transcribed) + + bot._handle_text_capture = _record + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=True), + ) + assert flags == [True] + + @pytest.mark.asyncio + async def test_a_mention_still_asks_rather_than_files( + self, bot_with_recorder, + ): + """Speaking to the bot is asking it something, exactly as typing + to it is. Deliberateness decides capture, never who is addressed.""" + bot, calls = bot_with_recorder + event = self._event("Archivist, where is the campsite booking?", + transcribed=True) + event.source["content"]["m.mentions"] = {"user_ids": [BOT_ID]} + await bot._on_text(self._shared_room(), event) + assert [c[0] for c in calls] == ["search"] diff --git a/tests/stacklets/test_capture_pipeline.py b/tests/stacklets/test_capture_pipeline.py index a5c704fe..66b759da 100644 --- a/tests/stacklets/test_capture_pipeline.py +++ b/tests/stacklets/test_capture_pipeline.py @@ -120,10 +120,11 @@ async def acknowledge(self): def _pipeline(*, mirror, classifier=None, capture_keep_body=False, - transcriber=None, llm=None): + llm=None, text_extractor=None): return CapturePipeline( url_extractor=FakeExtractor(_source(source_uri="http://src")), - text_extractor=FakeExtractor(_source(source_uri="http://embedded")), + text_extractor=text_extractor + or FakeExtractor(_source(source_uri="http://embedded")), classifier=classifier or FakeClassifier(), mirror=mirror, capture_tags=FakeTags(), @@ -132,37 +133,10 @@ def _pipeline(*, mirror, classifier=None, capture_keep_body=False, classify_max_chars=10000, capture_keep_body=capture_keep_body, capture_tag_prompt_size=50, - transcriber=transcriber, llm=llm, ) -class FakeTranscriber: - """Stand-in for stack.ai.client.Transcriber — records calls and - returns a configured transcript or raises a configured error. - - Mirrors the real signature including ``cleanup_with`` so the capture - pipeline can pass an LLM through and we can assert the routing. - """ - - def __init__(self, transcript: str = "I forgot to renew the boiler service", - error: Exception | None = None): - self.transcript = transcript - self.error = error - self.calls: list[dict] = [] - - async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", - model: str | None = None, - cleanup_with=None) -> str: - self.calls.append({ - "audio": audio, "filename": filename, - "cleanup_with": cleanup_with, - }) - if self.error is not None: - raise self.error - return self.transcript - - class TestCaptureUrl: @pytest.mark.asyncio @@ -236,88 +210,6 @@ async def test_empty_text_is_dropped(self): assert out.status == "empty" -class TestCaptureVoiceBatch: - """The ( ... ) batch flow: N voice memos arrive, each gets transcribed - on its way in (with LLM cleanup if available), then `)` combines them - into one note via capture_voice_batch.""" - - @pytest.mark.asyncio - async def test_concatenates_transcripts_with_paragraph_breaks(self): - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=[ - "First memo about the boiler.", - "Second memo about Bart's prescription.", - "Third memo with shopping list.", - ], - primary_mxc="mxc://server/first", - sender_mxid="@homer:s", - ) - assert out.status == "captured" - body = mirror.captures[0]["body_text"] - assert "First memo about the boiler." in body - assert "Second memo about Bart's prescription." in body - assert "Third memo with shopping list." in body - # Paragraph breaks between memos so the LLM and the human reader - # both see them as distinct thoughts. - assert "boiler.\n\nSecond" in body - - @pytest.mark.asyncio - async def test_empty_transcripts_returns_empty_status(self): - """A `(` ... `)` with no voice memos (or all silent ones) drops - out as empty -- the orchestrator silently no-ops, matching the - existing scan_cancelled semantics for empty PDF batches.""" - pipe = _pipeline(mirror=FakeMirror()) - out = await pipe.capture_voice_batch( - transcripts=[], primary_mxc=None, sender_mxid="@homer:s", - ) - assert out.status == "empty" - - @pytest.mark.asyncio - async def test_blank_transcripts_filtered(self): - """Empty / whitespace-only transcripts are dropped silently so - one silent memo in the middle of a batch doesn't add a hole - of blank lines to the combined body.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["real one", " ", "", "another real one"], - primary_mxc="mxc://server/first", - sender_mxid="@homer:s", - ) - assert out.status == "captured" - body = mirror.captures[0]["body_text"] - assert body == "real one\n\nanother real one" - - @pytest.mark.asyncio - async def test_primary_mxc_threads_through_as_source_uri(self): - """The first memo's mxc becomes the vault note's link -- the - wiki entry points back to the start of the conversation.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["just one"], - primary_mxc="mxc://server/start-of-batch", - sender_mxid="@homer:s", - ) - assert mirror.captures[0]["source_uri"] == "mxc://server/start-of-batch" - assert out.display_link == "mxc://server/start-of-batch" - - @pytest.mark.asyncio - async def test_outcome_carries_transcript_for_reply_echo(self): - """The mime=audio/ogg on the synthetic SourceContent triggers - the same transcript-in-reply behaviour single-memo captures - get -- so the sender sees the combined text quoted back.""" - pipe = _pipeline(mirror=FakeMirror()) - out = await pipe.capture_voice_batch( - transcripts=["one", "two"], - primary_mxc="mxc://server/x", - sender_mxid="@homer:s", - ) - assert out.transcript == "one\n\ntwo" - - class TestTagList: """`_tag_list` mixes the classifier's free-form tags with a `Person: X` tag per attributed person, normalising types and whitespace.""" @@ -391,9 +283,9 @@ def test_image_returns_single_image_no_text(self): assert kind == "bookmark" def test_unsupported_mime_returns_none(self): - # _source_from_binary itself doesn't handle audio; capture_binary - # routes audio through _source_from_audio upstream. Video and - # archives still land here and fall through to None. + # Video and archives land here and fall through to None. (Audio + # never reaches the capture pipeline at all: the transport + # decodes it to text before any handler sees it.) pipe = self._pipe() source, images, _kind = pipe._source_from_binary( file_data=b"riff-wave", @@ -441,140 +333,16 @@ def test_txt_file_by_extension_when_mime_unknown(self): # ── Audio capture (voice memos) ──────────────────────────────────────── -class TestSourceFromAudio: - """The audio branch: transcribe via the injected Transcriber, return - a note-shaped SourceContent. When no transcriber is wired, soft-skip - with None so the orchestrator surfaces a friendly extract_failed.""" - - @pytest.mark.asyncio - async def test_transcribes_audio_into_note(self): - tr = FakeTranscriber(transcript="Reminder: book the boiler service") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice-2026-06-09.ogg", - source_uri="mxc://server/abc", - ) - assert len(tr.calls) == 1 - assert tr.calls[0]["audio"] == b"opus-bytes" - assert tr.calls[0]["filename"] == "voice-2026-06-09.ogg" - assert source is not None - assert source.text == "Reminder: book the boiler service" - assert source.mime == "audio/ogg" - assert source.title_hint == "voice-2026-06-09.ogg" - assert source.source_uri == "mxc://server/abc" - assert images == [] - assert kind == "note" - - @pytest.mark.asyncio - async def test_no_transcriber_soft_skips(self): - # WHISPER_URL unset at bootstrap -> the pipeline drops audio - # rather than raising. Bot replies with extract_failed. - pipe = _pipeline(mirror=FakeMirror(), transcriber=None) - source, images, _kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - @pytest.mark.asyncio - async def test_transcriber_error_soft_skips(self): - # Whisper is misconfigured or down -> log + drop, same shape as - # the no-transcriber case so the user-facing reply is uniform. - from stack.ai.client import LLMUnavailableError - tr = FakeTranscriber(error=LLMUnavailableError("whisper down")) - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, _kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - @pytest.mark.asyncio - async def test_empty_transcript_soft_skips(self): - # whisper.cpp sometimes returns "" for a silent clip; treat it - # like an unreadable scan -- no point classifying empty bytes. - tr = FakeTranscriber(transcript=" \n ") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, _kind = await pipe._source_from_audio( - file_data=b"silence", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - -class TestCaptureBinaryAudioRouting: - """capture_binary peeks at the mime: audio/* -> _source_from_audio, - everything else -> _source_from_binary. The classify + mirror tail - runs identically afterwards, so a transcribed voice memo lands in - the mirror as a fully classified note.""" - - @pytest.mark.asyncio - async def test_audio_mime_routes_through_transcriber(self): - mirror = FakeMirror() - tr = FakeTranscriber(transcript="Pick up Bart's prescription Friday") - pipe = _pipeline(mirror=mirror, transcriber=tr) - out = await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice-2026-06-09.ogg", - source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - # The transcript reached the mirror via the classify tail. - assert len(tr.calls) == 1 - assert tr.calls[0]["audio"] == b"opus-data" - assert tr.calls[0]["filename"] == "voice-2026-06-09.ogg" - assert out.status == "captured" - assert mirror.captures, "expected the transcript to be mirrored as a note" - published = mirror.captures[0] - # The body the mirror writes IS the transcript -- voice memo as a - # searchable note in the sender's bucket. - assert "Pick up Bart's prescription Friday" in published["body_text"] - assert published["kind"] == "note" - # The outcome carries the transcript so the reply renderer can - # echo it back to the sender; PDF/URL/note captures get None. - assert out.transcript == "Pick up Bart's prescription Friday" - - @pytest.mark.asyncio - async def test_non_audio_capture_outcome_has_no_transcript(self): - """A markdown note capture must not surface a transcript field -- - only audio captures get one, so the reply layer can branch on - presence rather than mime-sniffing again.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror, transcriber=FakeTranscriber()) - out = await pipe.capture_binary( - file_data=b"# Hello\n\nbody", - mime="text/markdown", filename="note.md", - source_uri="mxc://server/md1", sender_mxid="@homer:s", - ) - assert out.status == "captured" - assert out.transcript is None - - @pytest.mark.asyncio - async def test_audio_without_transcriber_returns_extract_failed(self): - # Mirrors a bot booted before `stack up ai` -- the rest of the - # bot is alive, but voice messages can't be processed yet. - pipe = _pipeline(mirror=FakeMirror(), transcriber=None) - out = await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert out.status == "extract_failed" - # The reason qualifies the failure so the reply layer can render - # a voice-shaped message ('Couldn't transcribe...') rather than - # the URL-shaped message ('Couldn't read that link...'). - assert out.failure_reason == "transcription" +class TestCaptureBinaryFailures: + """capture_binary handles the shapes it can read and says which + shape failed, so the reply layer can word the error correctly.""" @pytest.mark.asyncio async def test_unreadable_binary_returns_binary_failure_reason(self): - """A non-audio mime that the binary path can't extract (e.g. - an unsupported video format) returns failure_reason='binary' so - the user gets the file-shaped error message, not the link one.""" - pipe = _pipeline(mirror=FakeMirror(), transcriber=FakeTranscriber()) + """A mime the binary path can't extract (e.g. an unsupported + video format) returns failure_reason='binary' so the user gets + the file-shaped error message, not the link one.""" + pipe = _pipeline(mirror=FakeMirror()) out = await pipe.capture_binary( file_data=b"random-bytes", mime="video/mp4", filename="clip.mp4", source_uri="mxc://server/v", @@ -583,56 +351,6 @@ async def test_unreadable_binary_returns_binary_failure_reason(self): assert out.status == "extract_failed" assert out.failure_reason == "binary" - @pytest.mark.asyncio - async def test_non_audio_mime_bypasses_transcriber(self): - # An image upload must not get sent to whisper -- the transcriber - # is for audio only, so non-audio mimes never touch it. - tr = FakeTranscriber() - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - await pipe.capture_binary( - file_data=b"\xff\xd8jpg", mime="image/jpeg", - filename="recipe.jpg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls == [] - - @pytest.mark.asyncio - async def test_llm_passed_to_transcriber_as_cleanup_with(self): - """The CapturePipeline.llm kwarg threads through to the Transcriber's - cleanup_with on every audio capture. This is the wiring contract - that powers transcript cleanup -- the Transcriber owns the prompt; - the pipeline only has to hand it the LLM.""" - # Sentinel object: not actually called by FakeTranscriber, but we - # assert it shows up in the recorded call. - sentinel_llm = object() - tr = FakeTranscriber(transcript="raw words") - pipe = _pipeline( - mirror=FakeMirror(), transcriber=tr, llm=sentinel_llm, - ) - await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls[0]["cleanup_with"] is sentinel_llm - - @pytest.mark.asyncio - async def test_no_llm_means_no_cleanup(self): - """When the LLM isn't wired (e.g. archivist boots before - OPENAI_URL is set), cleanup_with is None and the Transcriber - returns the raw transcript verbatim.""" - tr = FakeTranscriber(transcript="raw words") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr, llm=None) - await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls[0]["cleanup_with"] is None - - -# ── Topic-room tag-seed invariant ──────────────────────────────────────── - class TestTopicSeedMerge: """The static merge helper pins the tag-seed semantics: seed first, @@ -1034,21 +752,6 @@ async def test_capture_url_with_seed_keeps_classifier_additions(self): non_person = [t for t in tags if not t.startswith("Person: ")] assert non_person == ["van-life", "gear", "vans"] - @pytest.mark.asyncio - async def test_capture_voice_batch_with_seed_files_seed_tag(self): - """Voice batches go through `_publish` like every other - capture; the seed applies the same way.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["first memo", "second memo"], - primary_mxc="mxc://server/abc", - sender_mxid="@homer:s", - seed_topics=["camping"], - ) - assert out.status == "captured" - assert "camping" in mirror.captures[0]["tags"] - @pytest.mark.asyncio async def test_seed_recorded_in_capture_tag_cache(self): """The capture-tag cache feeds the next capture's prompt as @@ -1275,3 +978,30 @@ async def test_empty_body_is_empty_outcome(self): assert out.status == "empty" assert mirror.emails == [] assert mirror.captures == [] + + +class TestCaptureOrigin: + """The footer tells the family where a capture came from. Speech and + a paste are not the same provenance and must not read the same.""" + + # Spoken words rarely contain a URL, so the footer falls through to + # the origin label — which is the whole point of the label. + NO_LINK = staticmethod(lambda: FakeExtractor(_source(source_uri=None))) + + @pytest.mark.asyncio + async def test_a_transcribed_note_reads_as_a_voice_message(self): + pipe = _pipeline(mirror=FakeMirror(), text_extractor=self.NO_LINK()) + out = await pipe.capture_text( + text="Sort out the loft before Christmas.", + sender_mxid="@homer:s", transcribed=True, + ) + assert out.display_link == "(voice message)" + + @pytest.mark.asyncio + async def test_a_typed_note_still_reads_as_pasted_text(self): + pipe = _pipeline(mirror=FakeMirror(), text_extractor=self.NO_LINK()) + out = await pipe.capture_text( + text="Sort out the loft before Christmas.", + sender_mxid="@homer:s", + ) + assert out.display_link == "(pasted text)" diff --git a/tests/stacklets/test_microbot_voice.py b/tests/stacklets/test_microbot_voice.py new file mode 100644 index 00000000..99fef183 --- /dev/null +++ b/tests/stacklets/test_microbot_voice.py @@ -0,0 +1,519 @@ +"""Voice messages arrive at handlers as text. + +Holding the mic button and typing are the same act: putting words in the +room. Only the encoding differs, so only the transport should know about +it. These tests pin that contract from the caller's side — a bot registers +an ordinary text handler and receives spoken words through it, with the +speaker still the sender, having passed every gate a typed message passes. + +The whisper call itself is stubbed; what is under test is the framework +wiring, not the speech model. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from nio.events.room_events import RoomMessage, RoomMessageAudio, RoomMessageText + +_RUNNER = Path(__file__).resolve().parent.parent.parent / "stacklets" / "core" / "bot-runner" +sys.path.insert(0, str(_RUNNER)) +# `lib/` hosts `stack.ai.client`, which the framework imports for the +# Transcriber type. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "lib")) + +import voice # noqa: E402 +from microbot import MicroBot # noqa: E402 + + +# ── Fixtures ───────────────────────────────────────────────────────────── + +VOICE_SOURCE = { + "type": "m.room.message", + "event_id": "$voice1", + "sender": "@homer:simpson", + "origin_server_ts": 1700000000000, + "content": { + "msgtype": "m.audio", + "body": "Voice message", + "url": "mxc://simpson/abc123", + "info": {"mimetype": "audio/ogg", "duration": 4200, "size": 9001}, + }, +} + + +def _voice_event(**overrides) -> RoomMessageAudio: + src = {**VOICE_SOURCE, **overrides} + if "content" in overrides: + src["content"] = {**VOICE_SOURCE["content"], **overrides["content"]} + return RoomMessage.parse_event(src) + + +def _text_event(body: str = "typed words") -> RoomMessageText: + return RoomMessage.parse_event({ + "type": "m.room.message", + "event_id": "$typed1", + "sender": "@marge:simpson", + "origin_server_ts": 1700000000001, + "content": {"msgtype": "m.text", "body": body}, + }) + + +class _Room: + room_id = "!kitchen:simpson" + + +class _FakeClient: + def __init__(self): + self.rooms = {"!kitchen:simpson": _Room()} + self.typing: list[bool] = [] + + async def room_typing(self, room_id, typing_state, timeout=None): + self.typing.append(typing_state) + + +class _StubTranscriber: + """Stand-in for `stack.ai.client.Transcriber` — the raw whisper half.""" + + def __init__(self, result: str = "buy milk on the way home", + error: Exception | None = None): + self.result = result + self.error = error + self.calls: list[dict] = [] + + async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", + model: str | None = None, cleanup_with=None) -> str: + self.calls.append({"audio": audio, "filename": filename, + "cleanup_with": cleanup_with}) + if self.error is not None: + raise self.error + return self.result + + +class _StubLLM: + """Stand-in for the polish pass: records what it was asked to fix.""" + + def __init__(self, result: str = "Buy milk on the way home."): + self.result = result + self.seen: list[str] = [] + self.kwargs: list[dict] = [] + + async def complete(self, role: str, prompt: str, **kw) -> str: + self.seen.append(prompt) + self.kwargs.append(kw) + return self.result + + +class _Bot(MicroBot): + name = "test-bot" + + def register_callbacks(self, client): + pass + + +def _bot(tmp_path, *, transcriber=None, name="test-bot", cleanup=None) -> _Bot: + bot = _Bot(homeserver="http://hs", user_id="@test-bot:simpson", + password="x", session_dir=str(tmp_path)) + bot.name = name + bot._client = _FakeClient() + bot._transcriber = transcriber + bot._transcript_cleanup = cleanup + + async def _download(mxc_url): + return b"OggS-fake-audio" + + bot._download_media = _download + return bot + + +@pytest.fixture(autouse=True) +def _isolated_store(tmp_path, monkeypatch): + """The store is process-global and durable by design; give each test + its own directory so they do not inherit each other's transcripts.""" + monkeypatch.setattr( + voice, "TRANSCRIPTS", voice.TranscriptStore(tmp_path / "transcripts"), + ) + + +def _collect(bot, event_type=RoomMessageText) -> list: + """Register a handler of `event_type` and return the list it fills.""" + seen: list = [] + + async def handler(room, event): + seen.append(event) + + bot.add_event_callback(handler, event_type) + return seen + + +# ── The contract ───────────────────────────────────────────────────────── + +class TestVoiceIsJustAMessage: + """A handler that was written for typed text handles speech too, + without knowing speech exists.""" + + @pytest.mark.asyncio + async def test_a_text_handler_receives_the_spoken_words(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk")) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert len(seen) == 1 + assert isinstance(seen[0], RoomMessageText) + assert seen[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_the_speaker_remains_the_sender(self, tmp_path): + """Replies, reactions and thread ownership all key off the sender + and event id. If the framework re-attributed the words to itself, + a bot would answer its own message and corrections would break.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].sender == "@homer:simpson" + assert seen[0].event_id == "$voice1" + assert seen[0].server_timestamp == 1700000000000 + + @pytest.mark.asyncio + async def test_a_memo_sent_into_a_thread_stays_in_it(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + relates = {"rel_type": "m.thread", "event_id": "$root9", + "is_falling_back": True, + "m.in_reply_to": {"event_id": "$root9"}} + + await bot._dispatch( + "!kitchen:simpson", + _voice_event(content={"m.relates_to": relates}), + ) + + assert seen[0].source["content"]["m.relates_to"] == relates + + @pytest.mark.asyncio + async def test_the_words_are_marked_as_transcribed(self, tmp_path): + """Whisper is lossy in a way a keyboard is not, so provenance has + to survive: what was said, and the audio it came from, so a bad + transcript is visible and correctable.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + content = seen[0].source["content"] + assert voice.was_transcribed(content) + block = content[voice.TRANSCRIPT_KEY] + assert block["url"] == "mxc://simpson/abc123" + assert block["source_msgtype"] == "m.audio" + assert block["duration"] == 4200 + # The audio payload itself must not masquerade as a text body. + assert "url" not in content + assert content["msgtype"] == "m.text" + + @pytest.mark.asyncio + async def test_audio_handlers_no_longer_see_it(self, tmp_path): + """The decode is total, not a fan-out: nothing downstream is left + holding raw audio, so there is no second place to transcribe.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + audio_seen = _collect(bot, RoomMessageAudio) + text_seen = _collect(bot, RoomMessageText) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert audio_seen == [] + assert len(text_seen) == 1 + + @pytest.mark.asyncio + async def test_typed_messages_pass_through_untouched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + typed = _text_event("typed words") + await bot._dispatch("!kitchen:simpson", typed) + + assert seen == [typed] + assert not voice.was_transcribed(typed.source["content"]) + + +class TestWhenTheWordsCannotBeRecovered: + """Undecodable speech is dispatched to nobody. It is not an error to + answer: the family sees their own voice message sitting in the room, + the same as a message in a language the stack cannot read.""" + + @pytest.mark.asyncio + async def test_without_whisper_nothing_is_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=None) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_transcription_failure_is_not_dispatched(self, tmp_path): + from stack.ai.client import LLMError + + bot = _bot(tmp_path, transcriber=_StubTranscriber( + error=LLMError("whisper unreachable"))) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_silence_is_not_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber(" ")) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_failed_download_is_not_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + async def _no_bytes(mxc_url): + return None + + bot._download_media = _no_bytes + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + +class TestOneWhisperRunPerMessage: + """Every bot in a room drains the same timeline in the same process. + Without sharing, a five-minute memo would be sent to whisper once per + bot listening.""" + + @pytest.mark.asyncio + async def test_two_bots_transcribe_the_same_memo_once(self, tmp_path): + shared = _StubTranscriber("buy milk") + archivist = _bot(tmp_path / "a", transcriber=shared, name="archivist-bot") + stacker = _bot(tmp_path / "b", transcriber=shared, name="stacker-bot") + heard_a, heard_b = _collect(archivist), _collect(stacker) + + event = _voice_event() + await archivist._dispatch("!kitchen:simpson", event) + await stacker._dispatch("!kitchen:simpson", event) + + assert len(shared.calls) == 1 + # Both still get the words: sharing the work, not the delivery. + assert heard_a[0].body == "buy milk" + assert heard_b[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_a_stored_transcript_is_not_produced_again(self, tmp_path): + """The backfill and the drain are different processes over one + store. Whatever already cost GPU time must never cost it twice.""" + first = _StubTranscriber("buy milk") + bot = _bot(tmp_path, transcriber=first) + _collect(bot) + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert len(first.calls) == 1 + + # A fresh bot, as after a restart — same store on disk. + second = _StubTranscriber("should never run") + restarted = _bot(tmp_path / "restarted", transcriber=second) + seen = _collect(restarted) + await restarted._dispatch("!kitchen:simpson", _voice_event()) + + assert second.calls == [] + assert seen[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_a_failure_is_not_remembered(self, tmp_path): + """A whisper outage must not poison the message forever — the + drain is at-least-once, so the retry has to be able to succeed.""" + from stack.ai.client import LLMError + + broken = _StubTranscriber(error=LLMError("whisper down")) + bot = _bot(tmp_path, transcriber=broken) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert seen == [] + + bot._transcriber = _StubTranscriber("buy milk") + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert [e.body for e in seen] == ["buy milk"] + + +class TestTranscriberWiring: + """Built from the environment at construction, like every other + framework capability; absent whisper is a normal state, not a crash.""" + + def test_absent_whisper_leaves_the_bot_running(self, tmp_path, monkeypatch): + monkeypatch.delenv("WHISPER_URL", raising=False) + bot = _Bot(homeserver="http://hs", user_id="@t:simpson", + password="x", session_dir=str(tmp_path)) + assert bot._transcriber is None + + def test_whisper_url_builds_a_transcriber(self, tmp_path, monkeypatch): + monkeypatch.setenv("WHISPER_URL", "http://localhost:42062/v1") + bot = _Bot(homeserver="http://hs", user_id="@t:simpson", + password="x", session_dir=str(tmp_path)) + assert bot._transcriber is not None + + +class TestTranscribedSource: + """The pure rewrite, exercised directly: given a voice event's raw + source dict and the words, produce the text event it becomes.""" + + def test_keeps_identity_and_replaces_the_body(self): + out = voice.transcribed_source(VOICE_SOURCE, "buy milk") + assert out["event_id"] == VOICE_SOURCE["event_id"] + assert out["sender"] == VOICE_SOURCE["sender"] + assert out["content"]["body"] == "buy milk" + assert out["content"]["msgtype"] == "m.text" + + def test_does_not_mutate_the_original(self): + before = VOICE_SOURCE["content"]["msgtype"] + voice.transcribed_source(VOICE_SOURCE, "buy milk") + assert VOICE_SOURCE["content"]["msgtype"] == before + assert "url" in VOICE_SOURCE["content"] + + +class TestIsVoice: + def test_audio_with_a_payload(self): + assert voice.is_voice(_voice_event()) + + def test_plain_text(self): + assert not voice.is_voice(_text_event()) + + def test_audio_without_a_payload_is_not_decodable(self): + assert not voice.is_voice(_voice_event(content={"url": ""})) + + +class TestThePolishPass: + """whisper.cpp emits one unbroken lowercase run of words. The polish + pass puts the sentences back. It is kept deliberately word-preserving: + the memories room holds things people said to their children, and a + model "improving" those is not a transcript any more.""" + + @pytest.mark.asyncio + async def test_handlers_receive_the_polished_text(self, tmp_path): + bot = _bot( + tmp_path, + transcriber=_StubTranscriber("buy milk on the way home"), + cleanup=_StubLLM("Buy milk on the way home."), + ) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].body == "Buy milk on the way home." + + @pytest.mark.asyncio + async def test_the_raw_transcript_is_kept_alongside_the_polished_one( + self, tmp_path, + ): + """Polishing gets better with better models and costs almost + nothing; whisper does not and is not. Keeping the raw text means + years of recordings can be re-polished without the audio.""" + bot = _bot( + tmp_path, + transcriber=_StubTranscriber("buy milk on the way home"), + cleanup=_StubLLM("Buy milk on the way home."), + ) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + record = voice.TRANSCRIPTS.read("$voice1") + assert record["raw"] == "buy milk on the way home" + assert record["text"] == "Buy milk on the way home." + assert record["url"] == "mxc://simpson/abc123" + + @pytest.mark.asyncio + async def test_without_an_llm_the_raw_transcript_still_lands(self, tmp_path): + """A rough transcript beats no transcript. Polish is never a gate.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk"), + cleanup=None) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].body == "buy milk" + assert voice.TRANSCRIPTS.read("$voice1")["raw"] == "buy milk" + + +class TestTranscriptStore: + """Durable because a backfill over the memories room is years of + recordings, and a separate process from the bot that reads them.""" + + def test_a_record_survives_a_new_store_over_the_same_directory(self, tmp_path): + store = voice.TranscriptStore(tmp_path / "t") + store.write("$evt1", {"raw": "hello", "text": "Hello."}) + + reopened = voice.TranscriptStore(tmp_path / "t") + assert reopened.read("$evt1")["text"] == "Hello." + + def test_an_unknown_event_reads_as_none(self, tmp_path): + assert voice.TranscriptStore(tmp_path / "t").read("$nope") is None + + def test_event_ids_with_awkward_characters_round_trip(self, tmp_path): + """Matrix event ids are base64 and carry `/` and `+`, which would + otherwise land as directory separators.""" + store = voice.TranscriptStore(tmp_path / "t") + awkward = "$a/b+c=d:simpson" + store.write(awkward, {"text": "fine"}) + assert store.read(awkward)["text"] == "fine" + + def test_an_unwritable_store_does_not_break_the_message(self, tmp_path): + """A store we cannot write costs us the transcript again later. + It must never cost the family the message now.""" + blocked = tmp_path / "afile" + blocked.write_text("not a directory") + store = voice.TranscriptStore(blocked / "t") + store.write("$evt1", {"text": "hi"}) + assert store.read("$evt1") is None + + +class TestTheWorkingIndicator: + """Whisper runs before any handler can say it is working, so the + framework raises the indicator itself. Whoever raises it owns putting + it down — a bot left "typing" at a room it is not going to answer + sits there for the full five-minute timeout.""" + + @pytest.mark.asyncio + async def test_typing_stops_when_no_handler_wants_the_message(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk")) + # A bot that only handles audio now matches nothing: the message + # reaches dispatch as text. + _collect(bot, RoomMessageAudio) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert bot._client.typing == [True, False] + + @pytest.mark.asyncio + async def test_typing_stops_when_the_words_cannot_be_recovered(self, tmp_path): + from stack.ai.client import LLMError + + bot = _bot(tmp_path, transcriber=_StubTranscriber( + error=LLMError("whisper down"))) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert bot._client.typing == [True, False] + + @pytest.mark.asyncio + async def test_a_typed_message_never_raises_it(self, tmp_path): + """Only the decode needs the indicator; ordinary text is fast and + the handler decides for itself. (The handler wrap still clears it + on the way out, as it does for every message.)""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _text_event()) + + assert True not in bot._client.typing diff --git a/tests/stacklets/test_scribe.py b/tests/stacklets/test_scribe.py index ac4007f4..292e3e3d 100644 --- a/tests/stacklets/test_scribe.py +++ b/tests/stacklets/test_scribe.py @@ -1,285 +1,150 @@ -"""ScribeBot voice handling. - -Scribe used to call nio's media download, `room_typing`, and a hand- -built `room_send` directly. After the transport consolidation it goes -through the framework: `_download_media` (authenticated endpoint), -`_set_typing`, and `_send` (formatted, threaded). These pin that the -handler drives the framework methods rather than the raw client. - -The transcription HTTP call is delegated to the shared `Transcriber` -capability on the AI client; these tests inject a stub Transcriber so -the bot's wiring is exercised without a whisper server. +"""Scribe retires itself. + +Transcription moved into the transport, so Scribe has no job left. It +ships for one more release as a shell that explains itself and leaves, +because the framework has no way to deprovision a removed bot: delete the +declaration and the Matrix account simply survives, still joined to +whatever room someone invited it to, answering nothing forever. + +Scribe declared no room of its own, so the only installs affected are the +ones where a person went looking for it and invited it by hand. Those are +exactly the people who would notice it going quiet, which is why it says +goodbye rather than just stopping. """ from __future__ import annotations import sys from pathlib import Path -from types import SimpleNamespace import pytest -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "core" / "bot-runner")) -sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "messages" / "bot")) -# `lib/` hosts `stack.ai.client`, which the bot now imports at module load. -sys.path.insert(0, str(_REPO_ROOT / "lib")) +_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_ROOT / "stacklets" / "core" / "bot-runner")) +sys.path.insert(0, str(_ROOT / "stacklets" / "messages" / "bot")) +sys.path.insert(0, str(_ROOT / "lib")) from scribe import ScribeBot # noqa: E402 -from stack.ai.client import LLMUnavailableError # noqa: E402 - - -class _FakeClient: - """Supports the legacy surface so pre-migration code runs to a clean - assertion failure rather than crashing.""" - - def __init__(self): - self.sends: list[tuple] = [] - def add_event_callback(self, cb, event_type): - pass - async def room_typing(self, room_id, typing_state, timeout=None): - pass +class _Room: + def __init__(self, room_id): + self.room_id = room_id - async def room_send(self, room_id, message_type, content): - self.sends.append((room_id, message_type, content)) - - async def download(self, url): - # Not a DownloadResponse — legacy path bails here, so the spies - # below stay empty and the assertions fail cleanly (RED). - return SimpleNamespace(body=b"") - - -class _StubTranscriber: - """Stand-in for `stack.ai.client.Transcriber` — records calls and - returns a configured transcript (or raises a configured error).""" - - def __init__(self, result: str = "hello world", error: Exception | None = None): - self.result = result - self.error = error - self.calls: list[tuple[bytes, str]] = [] - - async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", - model: str | None = None, - cleanup_with=None) -> str: - self.calls.append({ - "audio": audio, "filename": filename, - "cleanup_with": cleanup_with, - }) - if self.error is not None: - raise self.error - return self.result - - -def _build(tmp_path, monkeypatch, *, transcriber: _StubTranscriber | None = None): - # Transcriber.from_env reads WHISPER_URL at construction; we don't - # care what URL the stub is "pointed at", but the constructor refuses - # an empty value, so pin a benign placeholder. - monkeypatch.setenv("WHISPER_URL", "http://test.local/v1") - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - bot._transcriber = transcriber or _StubTranscriber() - return bot +class _FakeClient: + def __init__(self, *room_ids, leave_error: Exception | None = None): + self.rooms = {r: _Room(r) for r in room_ids} + self.sent: list[dict] = [] + self.left: list[str] = [] + self._leave_error = leave_error -@pytest.mark.asyncio -async def test_voice_routes_through_framework(tmp_path, monkeypatch): - """A voice message is downloaded via `_download_media`, transcribed, - and posted via `_send` threaded to the voice event; typing toggles - via `_set_typing`.""" - bot = _build(tmp_path, monkeypatch) - - calls = {"download": [], "send": [], "typing": []} - - async def fake_download(mxc): - calls["download"].append(mxc) - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - calls["send"].append((room_id, text, reply_to)) - - async def fake_typing(room_id, on=True): - calls["typing"].append((room_id, on)) - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc123", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - assert calls["download"] == ["mxc://server/abc123"] - # The Transcriber got the downloaded bytes and the caption filename. - assert len(bot._transcriber.calls) == 1 - assert bot._transcriber.calls[0]["audio"] == b"audio-bytes" - assert bot._transcriber.calls[0]["filename"] == "voice.ogg" - # The bot built no LLM (no OPENAI_URL in the test env), so cleanup - # is skipped: cleanup_with comes through as None. - assert bot._transcriber.calls[0]["cleanup_with"] is None - assert len(calls["send"]) == 1 - room_id, text, reply_to = calls["send"][0] - assert room_id == "!r:server" - assert "hello world" in text - assert reply_to == "$v:server" - # Typing was driven via the framework helper (on at least once). - assert ("!r:server", True) in calls["typing"] - - -@pytest.mark.asyncio -async def test_failed_download_sends_nothing(tmp_path, monkeypatch): - """When media can't be downloaded, scribe bails without posting.""" - bot = _build(tmp_path, monkeypatch) - sent = [] - - async def fake_download(mxc): - return None - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - sent.append(text) - - async def fake_typing(room_id, on=True): + def add_event_callback(self, cb, event_type): pass - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/missing", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - assert sent == [] - # The Transcriber must not be called when there's nothing to transcribe. - assert bot._transcriber.calls == [] - - -@pytest.mark.asyncio -async def test_llm_threaded_into_transcribe_when_configured(tmp_path, monkeypatch): - """When OPENAI_URL is set, scribe builds an LLM and passes it as - cleanup_with so whisper's raw output gets polished. The LLM itself - isn't called in this test -- we use a stub transcriber that just - records what kwarg it received.""" - monkeypatch.setenv("WHISPER_URL", "http://test.local/v1") - monkeypatch.setenv("OPENAI_URL", "http://test.local/v1") - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - assert bot._llm is not None, "scribe should construct an LLM when OPENAI_URL is set" - - stub = _StubTranscriber(result="Cleaned text.") - bot._transcriber = stub - - async def fake_download(mxc): - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - pass + async def room_send(self, room_id, message_type, content, **kw): + self.sent.append({"room_id": room_id, "content": content}) - async def fake_typing(room_id, on=True): - pass + async def room_leave(self, room_id): + if self._leave_error is not None: + raise self._leave_error + self.left.append(room_id) - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - # The bot's own LLM instance came through as cleanup_with -- not a - # different one, not None. - assert stub.calls[0]["cleanup_with"] is bot._llm - - -@pytest.mark.asyncio -async def test_no_whisper_url_does_not_crash_construction(tmp_path, monkeypatch): - """WHISPER_URL missing -> the bot still constructs, sets transcriber=None, - and logs a warning. The family server should boot scribe even when AI - hasn't been installed yet.""" - monkeypatch.delenv("WHISPER_URL", raising=False) - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - assert bot._transcriber is None - - -@pytest.mark.asyncio -async def test_voice_no_op_when_transcriber_disabled(tmp_path, monkeypatch): - """Without a transcriber the bot silently ignores audio: no typing - indicator, no apology reply, no download. The warning at startup is - the one and only signal.""" - monkeypatch.delenv("WHISPER_URL", raising=False) - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - assert bot._transcriber is None - - calls = {"download": [], "send": [], "typing": []} - - async def fake_download(mxc): - calls["download"].append(mxc) - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - calls["send"].append(text) - - async def fake_typing(room_id, on=True): - calls["typing"].append((room_id, on)) - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - assert calls == {"download": [], "send": [], "typing": []} - - -@pytest.mark.asyncio -async def test_transcriber_error_falls_back_to_apology(tmp_path, monkeypatch): - """If whisper is down (Transcriber raises LLMError) the bot tells the - sender it couldn't do it — silent failure leaves people wondering.""" - failing = _StubTranscriber(error=LLMUnavailableError("whisper offline")) - bot = _build(tmp_path, monkeypatch, transcriber=failing) - sent: list[str] = [] - - async def fake_download(mxc): - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - sent.append(text) - - async def fake_typing(room_id, on=True): - pass - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing +def _bot(tmp_path, client) -> ScribeBot: + bot = ScribeBot(homeserver="http://hs", user_id="@scribe-bot:simpson", + password="x", session_dir=str(tmp_path)) + bot._client = client + return bot - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - assert len(sent) == 1 - assert "couldn't transcribe" in sent[0].lower() +class TestScribeRetires: + """The one behaviour it has left.""" + + @pytest.mark.asyncio + async def test_it_says_goodbye_and_leaves_every_room(self, tmp_path): + client = _FakeClient("!kitchen:simpson", "!notes:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert client.left == ["!kitchen:simpson", "!notes:simpson"] + assert [s["room_id"] for s in client.sent] == [ + "!kitchen:simpson", "!notes:simpson", + ] + + @pytest.mark.asyncio + async def test_the_goodbye_explains_itself(self, tmp_path): + """A member that vanishes without a word is a mystery to debug. + It has to say what replaced it and that nothing is lost.""" + client = _FakeClient("!kitchen:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + body = client.sent[0]["content"]["body"].lower() + assert "voice" in body + assert "automatic" in body or "automatically" in body + # A notice, not a chat message: this is the machine talking. + assert client.sent[0]["content"]["msgtype"] == "m.notice" + + @pytest.mark.asyncio + async def test_it_leaves_a_room_it_is_freshly_invited_to(self, tmp_path): + """Someone following an older guide invites it. Same answer, so + the invite does not leave a silent member behind.""" + client = _FakeClient("!new:simpson") + bot = _bot(tmp_path, client) + + await bot.on_room_joined("!new:simpson") + + assert client.left == ["!new:simpson"] + assert len(client.sent) == 1 + + @pytest.mark.asyncio + async def test_a_room_it_cannot_leave_does_not_stop_the_others( + self, tmp_path, + ): + """The sweep runs on every launch, so a failure is retried next + boot. It must not strand the rooms behind it in the meantime.""" + client = _FakeClient("!stuck:simpson", "!fine:simpson", + leave_error=RuntimeError("homeserver said no")) + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert len(client.sent) == 2 + + @pytest.mark.asyncio + async def test_it_speaks_the_household_language(self, tmp_path, monkeypatch): + monkeypatch.setenv("LANGUAGE", "de") + client = _FakeClient("!kueche:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert "Sprachnachrichten" in client.sent[0]["content"]["body"] + + +class TestScribeAnswersNothing: + """It transcribes nothing and replies to nothing. The framework does + the transcribing now, and a second transcriber in the same process is + the exact thing this release removed.""" + + @pytest.mark.asyncio + async def test_it_registers_no_message_handlers(self, tmp_path): + # Async because `register_callbacks` schedules the retirement + # sweep, and the framework always calls it from inside `start()`'s + # running loop. + client = _FakeClient() + bot = _bot(tmp_path, client) + bot.register_callbacks(client) + assert bot._handlers == [] + + def test_it_builds_no_transcriber_of_its_own(self, tmp_path, monkeypatch): + """MicroBot gives every bot one for the transport decode; Scribe + must not reach for it. Nothing here should ever call whisper.""" + monkeypatch.setenv("WHISPER_URL", "http://localhost:42062/v1") + bot = _bot(tmp_path, _FakeClient()) + assert not hasattr(bot, "_scribe_transcriber") + import scribe + assert "Transcriber" not in scribe.__dict__