From 32cc7996f5eb2117799fa3a63fe04d5e525fda45 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 11 Sep 2026 11:55:26 +0200 Subject: [PATCH 1/4] fix(ai): stop a cut-off whisper download breaking voice for good The installer only checked that the model file existed, so a download the network interrupted was reported as "Whisper model ready" with a green checkmark. whisper-server then loaded 47 of 587 tensors, aborted, and every `stack up ai` afterwards skipped the re-download because the file was still there. Voice transcription stayed broken with no way to recover short of deleting the file by hand. The model is ~1.5 GB, so anything far under that is a download that did not finish: re-fetch it and fail loudly if it is still short. Adds -f to the curl so an HTTP error page is never written out as a model. --- stacklets/ai/hooks/on_install.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/stacklets/ai/hooks/on_install.py b/stacklets/ai/hooks/on_install.py index 3481abe3..ad63a66b 100644 --- a/stacklets/ai/hooks/on_install.py +++ b/stacklets/ai/hooks/on_install.py @@ -28,6 +28,13 @@ OMLX_TAP_URL = "https://github.com/jundot/omlx" WHISPER_MODEL = "ggml-large-v3-turbo.bin" WHISPER_MODEL_URL = f"https://huggingface.co/ggerganov/whisper.cpp/resolve/main/{WHISPER_MODEL}" +# Floor for "this download finished". The model is ~1.5 GB; anything far +# under that is a download the network cut short. Existence alone is not +# enough: a half-file loads far enough to report "not all tensors loaded" +# and then abort, so the install would keep saying "model ready" while +# whisper died on every boot. Deliberately loose so a slightly smaller +# re-quantisation upstream does not trip it. +WHISPER_MODEL_MIN_BYTES = 1_000_000_000 WHISPER_PORT = 42062 PLIST_LABEL = "dev.famstack.whisper" @@ -200,11 +207,25 @@ def _install_whisper(ctx, data_dir: Path, state_dir: Path): # Download model model_path = model_dir / WHISPER_MODEL - if not model_path.exists(): + have = model_path.exists() and model_path.stat().st_size >= WHISPER_MODEL_MIN_BYTES + if not have: + if model_path.exists(): + size_mb = model_path.stat().st_size // (1024 * 1024) + dim(f" Existing model is only {size_mb} MB — re-downloading.") + model_path.unlink() ctx.step("Downloading Whisper model: large-v3-turbo (~1.5 GB)...") dim(" One-time download. OpenAI's distilled model — nearly") dim(" identical accuracy to large-v3, but 6x faster on Metal GPU.") - ctx.shell_live(f'curl -L --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"') + # -f so an HTTP error page is never written out as a model file. + ctx.shell_live( + f'curl -fL --progress-bar -o "{model_path}" "{WHISPER_MODEL_URL}"' + ) + if (not model_path.exists() + or model_path.stat().st_size < WHISPER_MODEL_MIN_BYTES): + raise RuntimeError( + "Whisper model download did not complete — run 'stack up ai' " + "again to resume. Voice transcription stays off until it does." + ) done("Model downloaded") else: done("Whisper model ready (large-v3-turbo)") From 94d6b5762bc45746e4fe21234a20c5af2ac0e24d Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 11 Sep 2026 11:55:35 +0200 Subject: [PATCH 2/4] fix(ai): stop a thinking model overwriting what you said A voice memo is cleaned up by the LLM to restore punctuation. Pointed at a Qwen3 with reasoning on, that call came back with a page of the model's own deliberation and no transcript at all, and we wrote it straight into the vault as what somebody had spoken. The audio is the only other copy. Three changes, smallest first: - Ask models not to think out loud. Verified against oMLX that only `chat_template_kwargs.enable_thinking=false` works; reasoning_effort, a top-level enable_thinking, a reasoning object and the /no_think prompt suffix all had no effect. Sent on every completion, because no famstack prompt wants deliberation in the answer. - Pin the cleanup to temperature 0. It is a transformation, not a generation: the same words should always get the same punctuation. - Verify the result instead of trusting it. The prompt permits punctuation, capitalization and line breaks and nothing else, so the letters are an exact invariant. A polish that summarises, truncates, reorders or leaks reasoning fails it and the raw transcript is kept. The check compares letters rather than words on purpose: whisper writes "theres" and "well known", and restoring the apostrophe or hyphen is punctuation. A word-level check rejected every real polish. --- lib/stack/ai/client.py | 89 ++++++++++++++-- tests/framework/test_ai_client.py | 164 +++++++++++++++++++++++++++++- 2 files changed, 244 insertions(+), 9 deletions(-) diff --git a/lib/stack/ai/client.py b/lib/stack/ai/client.py index 52bd5700..1ab1ee4d 100644 --- a/lib/stack/ai/client.py +++ b/lib/stack/ai/client.py @@ -23,6 +23,7 @@ import base64 import os +import re from dataclasses import dataclass import openai @@ -245,6 +246,7 @@ async def complete(self, role: str, prompt: str, *, resp = await self._client.chat.completions.create( model=model, messages=[{"role": "user", "content": content}], + extra_body=_NO_THINKING, **kwargs, ) # Order matters: APITimeoutError < APIConnectionError, and @@ -299,6 +301,23 @@ async def aclose(self) -> None: await self._client.close() +# Every prompt famstack sends wants an answer, not a monologue: classify +# this, summarise that, punctuate these words. Reasoning models default to +# writing their deliberation into `content`, which at best burns tokens and +# at worst *is* the output — a Qwen3 with thinking on answered the +# transcript-polish prompt with a page of its own analysis and never +# reached the transcript. +# +# `chat_template_kwargs` is the switch that actually works. Verified +# against oMLX: `reasoning_effort`, a top-level `enable_thinking`, a +# `reasoning` object and the `/no_think` prompt suffix all had no effect, +# while this returned clean output with finish_reason=stop. vLLM, SGLang +# and Ollama take the same key; servers that don't know it ignore unknown +# body fields. Sent on every call rather than hidden behind a setting, +# because there is no famstack prompt that wants the other behaviour. +_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}} + + def _content_parts(prompt: str, images: list) -> list[dict]: """OpenAI-style multimodal content: one text part + N image_url parts. @@ -451,27 +470,81 @@ async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", raw = (getattr(resp, "text", "") or "").strip() if not raw or cleanup_with is None: return raw - return await self._cleanup(raw, cleanup_with) + return await self.polish(raw, cleanup_with) @staticmethod - async def _cleanup(raw: str, llm: "LLM") -> str: - """Run the LLM-cleanup pass; fall back to raw on any failure. + def _comparable(text: str) -> str: + """`text` reduced to what the polish is not allowed to change. + + Letters and digits only, lowercased. Punctuation is dropped + rather than treated as a separator, because the polish is + *supposed* to add it — including inside words. Splitting on + non-word characters instead would read a restored apostrophe + ("theres" -> "there's") or a restored hyphen ("well known" -> + "well-known") as an added word and reject a correct polish. + That is not hypothetical: it rejected every real polish on the + first run against a live model. + + The failures this exists to catch all move far more than a + boundary — reasoning leaking into the reply, a summary, a + truncation, reordered words — so comparing the letter stream + catches them while leaving legitimate punctuation alone. + """ + return re.sub(r"[^a-z0-9]+", "", text.lower()) - Cleanup is best-effort: if the LLM is down or the model returns - empty output, the caller still gets a usable transcript. We log - a warning so the admin can see drift between raw and clean if - they ever want to investigate model quality. + @staticmethod + async def polish(raw: str, llm: "LLM") -> str: + """Restore punctuation and sentence breaks in a raw transcript. + + whisper.cpp emits one unbroken lowercase run of words. This pass + makes it readable again without changing what was said — the + prompt forbids adding, removing, reordering or rephrasing, so the + result is still verbatim. That restraint is deliberate: the + memories room holds things people said to their children, and a + model "improving" those is not a transcript any more. + + Public because the polish and the transcription are separately + useful: a backfill keeps the raw text so a better model can + re-polish years of recordings without paying for whisper twice. + + Best-effort: if the LLM is down or returns nothing, the caller + still gets a usable transcript. We log a warning so the admin can + see drift between raw and polished if they want to investigate + model quality. """ try: cleaned = await llm.complete( _CLEANUP_ROLE, _CLEANUP_PROMPT.format(raw=raw), + # A transformation, not a generation: the same words + # should always get the same punctuation. Sampling only + # invites the rephrasing the word check below rejects, + # which turns a paid model call into a raw transcript. + temperature=0.0, ) except LLMError as e: logger.warning("[transcriber] cleanup failed, returning raw: {}", e) return raw cleaned = (cleaned or "").strip() # A model that returned nothing is no better than no model. - return cleaned or raw + if not cleaned: + return raw + # Verify rather than trust. The prompt permits punctuation, + # capitalization and line breaks and nothing else, so the word + # sequence is an exact, checkable invariant — and the failures + # worth catching all break it: reasoning leaking into `content`, + # a summary instead of a transcript, a polish truncated at the + # token limit, words reordered. Any of those would otherwise be + # written into the vault as what somebody said, with the audio + # as the only surviving copy of the truth. + raw_cmp, clean_cmp = Transcriber._comparable(raw), Transcriber._comparable(cleaned) + if raw_cmp != clean_cmp: + logger.warning( + "[transcriber] polish changed the words, keeping raw " + "(raw={} chars, polished={} chars)", + len(raw_cmp), len(clean_cmp), + ) + return raw + return cleaned async def aclose(self) -> None: """Close the underlying HTTP client. Owned by us via from_env().""" diff --git a/tests/framework/test_ai_client.py b/tests/framework/test_ai_client.py index 3e80f72e..42bcf247 100644 --- a/tests/framework/test_ai_client.py +++ b/tests/framework/test_ai_client.py @@ -554,9 +554,11 @@ def __init__(self, result: str = "Cleaned text.", self.result = result self.error = error self.calls: list[tuple[str, str]] = [] + self.kwargs: list[dict] = [] - async def complete(self, role: str, prompt: str) -> str: + async def complete(self, role: str, prompt: str, **kw) -> str: self.calls.append((role, prompt)) + self.kwargs.append(kw) if self.error is not None: raise self.error return self.result @@ -634,3 +636,163 @@ async def test_empty_cleanup_response_falls_back_to_raw(self, httpserver: HTTPSe tr = _make_transcriber(httpserver) assert await tr.transcribe(b"audio", cleanup_with=empty_llm) == "raw words present" await tr.aclose() + + +class TestPolishKeepsTheWords: + """The polish pass may add punctuation, capitalization and sentence + breaks. It may not change what was said. + + This is not a theoretical guard. Pointed at a reasoning model with + thinking left on, the cleanup prompt came back with a page of the + model's own deliberation and no transcript at all. Without a check, + that monologue is what gets written into the family's vault in place + of the words somebody actually spoke — and the audio it came from is + the only remaining copy. + + So the polish is verified against its own contract rather than + trusted: same words, same order, or we keep the raw transcript. + """ + + async def test_punctuation_and_capitalization_are_accepted( + self, httpserver: HTTPServer, + ): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "hey i forgot to renew the boiler"}) + llm = _StubLLM(result="Hey, I forgot to renew the boiler.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "Hey, I forgot to renew the boiler." + ) + await tr.aclose() + + async def test_a_model_thinking_out_loud_is_rejected( + self, httpserver: HTTPServer, + ): + """The failure that prompted this: reasoning leaks into content.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite for july"}) + llm = _StubLLM(result=( + "Here's a thinking process:\n\n1. **Analyze User Input:** The " + "user wants punctuation restored. Let me check the constraints." + )) + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "book the campsite for july" + ) + await tr.aclose() + + async def test_a_summary_is_rejected(self, httpserver: HTTPServer): + """Dropping words is the quiet failure: it reads fine and is wrong.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({ + "text": "remember to book the campsite by the lake for july", + }) + llm = _StubLLM(result="Book the campsite.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "remember to book the campsite by the lake for july" + ) + await tr.aclose() + + async def test_a_truncated_polish_is_rejected(self, httpserver: HTTPServer): + """Hitting the token limit mid-sentence loses the tail silently.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "one two three four five six"}) + llm = _StubLLM(result="One, two, three.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "one two three four five six" + ) + await tr.aclose() + + async def test_reordered_words_are_rejected(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "marge called about bart"}) + llm = _StubLLM(result="Bart called about Marge.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "marge called about bart" + ) + await tr.aclose() + + async def test_whitespace_and_line_breaks_are_accepted( + self, httpserver: HTTPServer, + ): + """whisper emits segment-per-line with leading spaces; rejoining + those into paragraphs is exactly what the polish is for.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite.\n by the lake."}) + llm = _StubLLM(result="Book the campsite, by the lake.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "Book the campsite, by the lake." + ) + await tr.aclose() + + +class TestPolishIsDeterministic: + """Polishing is a transformation, not a generation: the same words in + should give the same punctuation out. Sampling only invites the model + to rephrase, which the word check then rejects — so a run that costs a + model call ends up returning the raw transcript anyway.""" + + async def test_the_cleanup_call_pins_temperature(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "book the campsite"}) + llm = _StubLLM(result="Book the campsite.") + tr = _make_transcriber(httpserver) + await tr.transcribe(b"a", cleanup_with=llm) + assert llm.kwargs[0]["temperature"] == 0.0 + await tr.aclose() + + +class TestPolishMayAddPunctuationInsideWords: + """Regression: the first word-level check rejected every real polish. + + Whisper writes "theres" and "well known"; restoring the apostrophe and + the hyphen is punctuation, which the prompt permits. A check that + split on non-word characters counted those as new words and threw the + polish away, so the feature silently did nothing against a live model + while every stubbed test passed. + """ + + async def test_a_restored_apostrophe_is_accepted(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "theres decorations in the loft"}) + llm = _StubLLM(result="There's decorations in the loft.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "There's decorations in the loft." + ) + await tr.aclose() + + async def test_a_restored_hyphen_is_accepted(self, httpserver: HTTPServer): + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "its a well known campsite"}) + llm = _StubLLM(result="It's a well-known campsite.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "It's a well-known campsite." + ) + await tr.aclose() + + async def test_a_changed_word_is_still_rejected(self, httpserver: HTTPServer): + """The loosening must not swallow an actual edit.""" + httpserver.expect_request( + "/v1/audio/transcriptions", method="POST", + ).respond_with_json({"text": "theres decorations in the loft"}) + llm = _StubLLM(result="There are decorations in the attic.") + tr = _make_transcriber(httpserver) + assert await tr.transcribe(b"a", cleanup_with=llm) == ( + "theres decorations in the loft" + ) + await tr.aclose() From 7170dd24113ebbb7ac60417329816047b28aeb1c Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 11 Sep 2026 11:56:11 +0200 Subject: [PATCH 3/4] feat(voice): treat a voice message like any other message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Holding the mic button and typing are the same act: putting words in the room. Only the encoding differed, and until now every bot had to know about it. Transcription was a capability each one reached for, which grew an audio branch in the archivist, a second whisper client in the same process, a third transcribe call for batch scans, and a whole bot whose only job was the capability. Now the transport decodes speech before dispatch, beside decryption: below everyone, done once. A handler written for typed text handles speech without knowing speech exists, and every gate a typed message passes — room mode, thread ownership, mention, routing, corrections — applies to it unchanged. Speaking "find the insurance policy" in the documents room now searches, the way typing it always did. The message keeps its identity through the decode: same sender, event id and thread, so replies, reactions and ownership still land on the voice message the family can see. What it gains is provenance, because whisper is lossy in a way a keyboard is not, and the reply quotes the words back so a mishearing is visible without opening the vault. Two judgement calls worth knowing: - A memo counts as deliberately handed over, the way a link or an upload does. The ambient gate that silences long typed messages in a shared room reads length as a stand-in for intent, and it cannot read speech. Nobody holds the mic button for thirty seconds as a turn in a chat. On the rig, without this, voice capture in a normal family room went from always working to never working. - Transcripts are cached on disk, keyed by event id. They are derived data, not a record — the audio on the timeline is the record — but whisper costs minutes of GPU per memo and a backfill over years of recordings must never pay for the same one twice. The raw text is kept beside the polished text so a better model can re-polish without touching the audio again. Removes scribe-bot, whose only job this was. It had been in zero rooms since it shipped: it declares no room, and only bots that declare one get joined anywhere. `stack messages upload` can now send audio as m.audio, so a voice message can be driven from the terminal at all. --- stacklets/core/bot-runner/Dockerfile | 2 +- stacklets/core/bot-runner/microbot.py | 102 +++- stacklets/core/bot-runner/voice.py | 217 ++++++++ stacklets/core/docker-compose.yml | 1 + stacklets/core/stacklet.toml | 5 + stacklets/docs/bot/archivist.py | 197 ++----- stacklets/docs/bot/capture_pipeline.py | 146 +---- stacklets/docs/bot/cli/capture.py | 1 - stacklets/docs/bot/messages/archivist.yml | 12 - stacklets/messages/bot/bot.toml | 3 - stacklets/messages/bot/scribe.py | 103 ---- stacklets/messages/cli/upload.py | 13 +- tests/stacklets/test_archivist_corrections.py | 3 +- tests/stacklets/test_archivist_routing.py | 183 ++++-- tests/stacklets/test_capture_pipeline.py | 350 ++---------- tests/stacklets/test_microbot_voice.py | 519 ++++++++++++++++++ tests/stacklets/test_scribe.py | 285 ---------- 17 files changed, 1094 insertions(+), 1048 deletions(-) create mode 100644 stacklets/core/bot-runner/voice.py delete mode 100644 stacklets/messages/bot/bot.toml delete mode 100644 stacklets/messages/bot/scribe.py create mode 100644 tests/stacklets/test_microbot_voice.py delete mode 100644 tests/stacklets/test_scribe.py diff --git a/stacklets/core/bot-runner/Dockerfile b/stacklets/core/bot-runner/Dockerfile index 7f0c8bcf..6b0cf8d7 100644 --- a/stacklets/core/bot-runner/Dockerfile +++ b/stacklets/core/bot-runner/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py microbot.py room_context.py accounts.py ./ +COPY main.py microbot.py room_context.py accounts.py voice.py ./ RUN adduser --disabled-password --uid 1000 botrunner RUN mkdir -p /botdata && chown botrunner:botrunner /botdata diff --git a/stacklets/core/bot-runner/microbot.py b/stacklets/core/bot-runner/microbot.py index 01a31a3c..ff5e2ed2 100644 --- a/stacklets/core/bot-runner/microbot.py +++ b/stacklets/core/bot-runner/microbot.py @@ -67,12 +67,15 @@ LoginResponse, MegolmEvent, MessageDirection, + RoomMessage, RoomMessagesResponse, SyncResponse, ) from nio.api import RelationshipType +import voice from room_context import RoomContext, context_for +from stack.ai.client import LLM, LLMError, LLMUnavailableError, Transcriber # The framework's "I picked this up and I'm working on it" signal. A # bot reacts with 👀 on the source message the moment it starts a @@ -126,6 +129,24 @@ def __init__(self, homeserver: str, user_id: str, password: str, session_dir: st # via sync. The SyncResponse callback drains this set and fires # `on_room_joined` once nio has populated the room. self._pending_room_joins: set[str] = set() + # Speech-to-text for the decode in `_dispatch`. Built here rather + # than on demand because a missing whisper is a normal state, not + # an error: without it, audio simply stays audio and reaches no + # handler, the same as a message in a language we cannot read. + try: + self._transcriber = Transcriber.from_env(namespace=self.name) + except LLMUnavailableError as e: + logger.info( + "[{}] no transcription: {} — voice messages will be left " + "as audio", self.name, e, + ) + self._transcriber = None + # Optional polish on raw whisper output (punctuation, sentence + # breaks). Absent, the transcript still lands, just rougher. + try: + self._transcript_cleanup = LLM.from_env(namespace=self.name) + except LLMUnavailableError: + self._transcript_cleanup = None self._client: AsyncClient | None = None # Lazily-created shared aiohttp session for non-nio HTTP (media # download, and subclasses' own API calls). Owned by the @@ -493,13 +514,92 @@ async def _set_read_receipt(self, room_id: str, event_id: str | None) -> None: self.name, room_id, e) async def _dispatch(self, room_id: str, event) -> None: - """Invoke every handler whose registered type matches the event.""" + """Invoke every handler whose registered type matches the event. + + Speech is decoded first, so what handlers match against is the + text of the message rather than its encoding. + """ room = self.client.rooms.get(room_id) if room is None: return + + decoding = voice.is_voice(event) + if decoding: + # Transcription is the one part of handling a message that can + # run for minutes, and it happens before any handler gets to + # signal it is working. A matched handler's wrap clears the + # indicator in its `finally`; we clear it ourselves on the two + # paths where no handler ever runs. + await self._set_typing(room_id, on=True) + event = await self._decode_voice(room_id, event) + if event is None: + await self._set_typing(room_id, on=False) + return + + handled = False for event_type, handler in self._handlers: if isinstance(event, event_type): + handled = True await handler(room, event) + if decoding and not handled: + await self._set_typing(room_id, on=False) + + async def _decode_voice(self, room_id: str, event): + """Turn a voice message into the text event it is, or None. + + None means the words could not be recovered — whisper is absent, + unreachable, or heard nothing. That is dispatched to nobody + rather than answered with an apology: the family can see their + own voice message sitting in the room, and a bot volunteering + "I could not hear that" in every room it is in, for audio nobody + was addressing to it, is noise. + """ + if self._transcriber is None: + logger.debug( + "[{}] voice from {} left as audio (no whisper)", + self.name, event.sender, + ) + return None + + content = event.source.get("content", {}) + url = content.get("url", "") + filename = content.get("filename") or content.get("body") or "voice.ogg" + + async def run() -> dict: + audio = await self._download_media(url) + if not audio: + raise LLMError(f"could not download {url}") + raw = await self._transcriber.transcribe(audio, filename=filename) + # whisper emits one unbroken lowercase run of words; the polish + # pass puts the sentences back without changing them. Both are + # kept: polishing again with a better model is cheap, and + # transcribing again is not. + text = raw + if raw.strip() and self._transcript_cleanup is not None: + text = await Transcriber.polish(raw, self._transcript_cleanup) + return {"raw": raw, "text": text, "url": url, "filename": filename} + + try: + transcript = (await voice.TRANSCRIPTS.run(event.event_id, run))["text"] + except LLMError as e: + logger.warning( + "[{}] transcription failed for {}: {}", + self.name, event.event_id, e, + ) + return None + if not transcript.strip(): + logger.info( + "[{}] no speech found in {}", self.name, event.event_id, + ) + return None + + logger.info( + "[{}] heard {} in {}: {}...", + self.name, event.sender, room_id, transcript[:80], + ) + return RoomMessage.parse_event( + voice.transcribed_source(event.source, transcript) + ) # ── Typing + error response ────────────────────────────────────────── # diff --git a/stacklets/core/bot-runner/voice.py b/stacklets/core/bot-runner/voice.py new file mode 100644 index 00000000..d8cf9854 --- /dev/null +++ b/stacklets/core/bot-runner/voice.py @@ -0,0 +1,217 @@ +"""Voice messages are messages. The transport decodes them, not the bots. + +Someone holding the mic button and someone typing are doing the same +thing: putting words in the room. Only the encoding differs, so only the +transport should know about it. By the time a handler runs, an `m.audio` +event has already become an ordinary text event — same sender, same event +id, same thread — and every gate a typed message passes through (room +mode, thread ownership, mention, routing) applies to it unchanged. + +This is deliberately not a capability bots reach for. That was the earlier +design, and it grew an audio branch in every consumer, two whisper clients +in one process, and three places that could transcribe the same bytes. +Decoding belongs beside decryption: below everyone, done once. + +What survives the decode is provenance. Whisper is lossy in a way a +keyboard is not, so the text event carries a `dev.famstack.transcript` +block naming the audio it came from. Routing ignores it; the bot that +acts on the words uses it to echo them back, which is the only way a +mistranscription is visible without opening the vault. The audio itself +stays on the timeline as the reproducibility anchor (ADR-010) — we never +need to keep a second copy of the bytes. + +The module owns the pure half: recognising a voice event and rewriting its +raw source dict. `MicroBot._dispatch` owns the I/O half (download, +whisper, share). +""" + +from __future__ import annotations + +import asyncio +import base64 +import copy +import json +import os +import time +from collections.abc import Awaitable, Callable +from pathlib import Path + +from loguru import logger + + +# Provenance on a decoded message: which audio event these words came +# from. Its presence is the answer to "were these words guessed at by a +# machine?" — a question the reply layer asks and the routing layer must not. +TRANSCRIPT_KEY = "dev.famstack.transcript" + +# Fields of the audio event that describe the payload rather than the +# message. They move into the provenance block; leaving them on a text +# event would let a consumer treat it as an upload and re-file the bytes. +_PAYLOAD_FIELDS = ("url", "file", "info", "filename") + + +def is_voice(event) -> bool: + """Whether this timeline event is speech we can decode. + + Any `m.audio` with a plain mxc payload counts. We do not try to tell + a voice memo from a music file first: whisper answering "no words + here" is a better detector than a guess at the sender's intent, and + the clients families use mark both the same way. + + Encrypted media (`file` rather than `url`) is deliberately not + claimed. The framework already tells encrypted rooms it cannot read + them, so saying yes here would only produce a download that fails. + """ + content = (getattr(event, "source", None) or {}).get("content") or {} + if content.get("msgtype") != "m.audio": + return False + return bool(content.get("url")) + + +def was_transcribed(content: dict) -> bool: + """Whether a message's words were transcribed rather than typed. + + Reads the framework contract, so it works for any consumer without + knowing which component decoded the audio. Consumers ask this to + decide whether to show the words back for checking, never to decide + routing — routing is the whole thing that must not care. + """ + return isinstance(content.get(TRANSCRIPT_KEY), dict) + + +def transcribed_source(source: dict, transcript: str) -> dict: + """Rewrite a voice event's raw source dict as the text event it is. + + Identity is preserved wholesale — event id, sender, timestamp, + thread relation — because this is the same message, read aloud + rather than typed. Bots reply to it, react on it and claim thread + ownership of it by that identity, so a synthesised id would detach + every one of those from the message the family can actually see. + """ + out = copy.deepcopy(source) + content = out.setdefault("content", {}) + + provenance = {"source_msgtype": content.get("msgtype", "m.audio")} + if url := content.get("url"): + provenance["url"] = url + if filename := (content.get("filename") or content.get("body")): + provenance["filename"] = filename + info = content.get("info") + if isinstance(info, dict): + if (duration := info.get("duration")) is not None: + provenance["duration"] = duration + if mimetype := info.get("mimetype"): + provenance["mimetype"] = mimetype + + for field in _PAYLOAD_FIELDS: + content.pop(field, None) + content["msgtype"] = "m.text" + content["body"] = transcript + content[TRANSCRIPT_KEY] = provenance + return out + + +class TranscriptStore: + """Transcripts on disk, keyed by Matrix event id. + + Transcription is the most expensive thing the stack does per message: + minutes of GPU for a long memo. Three callers want the same answer and + must not each pay for it. + + * Every bot in the room drains the same timeline in the same + process, so each reaches the same audio independently. + * The drain is at-least-once, so a handler that dies mid-flight + brings its event back around. + * A backfill walks a room's whole history in a *separate* process. + The memories room holds years of recordings; re-transcribing that + because the answer was only ever in RAM is not acceptable once, + let alone every time something wants to read it. + + So the store is durable and shared, and the in-flight map on top of it + collapses concurrent askers onto one run. One file per voice message, + written atomically, because the backfill and the bot runner are + different processes writing the same directory. + + Each record keeps the raw whisper output next to the polished text. + Polishing is cheap and improves with better models; whisper is not and + does not. Keeping both means years of recordings can be re-polished + without touching the audio again. + """ + + def __init__(self, path: str | Path | None = None): + self.path = Path( + path or os.environ.get("TRANSCRIPT_DIR", "/data/core/transcripts") + ) + self._inflight: dict[str, asyncio.Future] = {} + + # ── Durable half ───────────────────────────────────────────────── + + def _file(self, event_id: str) -> Path: + # Event ids carry `$`, `/` and `+`; base64url keeps one file per + # id without inventing a collision-prone slug. + name = base64.urlsafe_b64encode(event_id.encode()).decode().rstrip("=") + return self.path / f"{name}.json" + + def read(self, event_id: str) -> dict | None: + """The stored record for `event_id`, or None if we never ran it.""" + try: + return json.loads(self._file(event_id).read_text()) + except FileNotFoundError: + return None + except (OSError, ValueError) as e: + logger.warning("[voice] unreadable transcript for {}: {}", event_id, e) + return None + + def write(self, event_id: str, record: dict) -> None: + """Persist a record. A store we cannot write is not fatal — the + transcript still reaches the handler, it just costs again later.""" + target = self._file(event_id) + try: + self.path.mkdir(parents=True, exist_ok=True) + tmp = target.with_suffix(".tmp") + tmp.write_text(json.dumps(record, ensure_ascii=False, indent=2)) + os.replace(tmp, target) + except OSError as e: + logger.warning("[voice] could not store transcript {}: {}", event_id, e) + + # ── Single-flight half ─────────────────────────────────────────── + + async def run( + self, event_id: str, produce: Callable[[], Awaitable[dict]], + ) -> dict: + """The record for `event_id`, produced at most once across callers. + + A failure is never remembered. The drain is at-least-once and + whisper outages are transient, so caching an error would turn a + restartable service into a permanently silent message. + """ + if (stored := self.read(event_id)) is not None: + return stored + if (pending := self._inflight.get(event_id)) is not None: + return await pending + + loop = asyncio.get_running_loop() + future: asyncio.Future = loop.create_future() + self._inflight[event_id] = future + try: + record = await produce() + except BaseException as e: + self._inflight.pop(event_id, None) + if not future.done(): + future.set_exception(e) + # Retrieve it so an unawaited future does not warn. + future.exception() + raise + record.setdefault("at", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + # Publish before releasing the slot, so a caller arriving in + # between finds the answer rather than starting a second run. + self.write(event_id, record) + if not future.done(): + future.set_result(record) + self._inflight.pop(event_id, None) + return record + + +# Process-global: the point is to share across the bots in this runner, +# and with whatever process backfills history. +TRANSCRIPTS = TranscriptStore() diff --git a/stacklets/core/docker-compose.yml b/stacklets/core/docker-compose.yml index b7a2d9ca..959e3565 100644 --- a/stacklets/core/docker-compose.yml +++ b/stacklets/core/docker-compose.yml @@ -52,6 +52,7 @@ services: - ../../stacklets/core/bot-runner/microbot.py:/app/microbot.py:ro - ../../stacklets/core/bot-runner/room_context.py:/app/room_context.py:ro - ../../stacklets/core/bot-runner/accounts.py:/app/accounts.py:ro + - ../../stacklets/core/bot-runner/voice.py:/app/voice.py:ro # Per-stacklet bot data (sessions, crypto stores) - ${DATA_DIR}:/data # API is TCP on host.docker.internal:42001 — no socket mount needed diff --git a/stacklets/core/stacklet.toml b/stacklets/core/stacklet.toml index 182eae51..b8cb9156 100644 --- a/stacklets/core/stacklet.toml +++ b/stacklets/core/stacklet.toml @@ -45,6 +45,11 @@ STACK_ADMIN_USER_IDS = "{admin_user_ids}" OPENAI_URL = "{ai_openai_url_docker}" OPENAI_KEY = "{ai_openai_key}" WHISPER_URL = "{ai_whisper_url_docker}/audio/transcriptions" +# Transcripts are a cache over the audio, not a record: the mxc on the +# timeline stays the anchor. Kept on disk because whisper costs minutes +# of GPU per memo and a backfill over the memories room must never pay +# for the same recording twice. +TRANSCRIPT_DIR = "/data/core/transcripts" # Paperless — document search for archivist bot and tools server PAPERLESS_URL = "http://stack-docs-paperless:8000" diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py index f7c11c37..232bedc9 100644 --- a/stacklets/docs/bot/archivist.py +++ b/stacklets/docs/bot/archivist.py @@ -50,6 +50,7 @@ from capture_tags import CaptureTagCache from extractors import TextExtractor, UrlExtractor from git_mirror import GitMirror +import voice from microbot import CHECK, CROSS, EYES, MicroBot from pdf_analysis import ( DEFAULT_REFORMAT_MAX_PDF_PAGES, @@ -65,10 +66,7 @@ from stack.email_message import defang_links from stack.links import go_docs, go_topic, public from stack.ai.client import ( - LLMError, - LLMUnavailableError, ModelCapabilities, - Transcriber, ) # Make sibling stacklets importable. In the bot-runner container, @@ -161,7 +159,7 @@ def _llm_error_for_chat( # ── Constants ──────────────────────────────────────────────────────────────── -SUPPORTED_MSGTYPES = {"m.file", "m.image", "m.audio"} +SUPPORTED_MSGTYPES = {"m.file", "m.image"} SCAN_BEGIN = {"scan", "("} SCAN_END = {"done", "fertig", ")"} @@ -223,8 +221,6 @@ def _guess_mime(filename: str, msgtype: str) -> str: return _MIME_BY_EXT[ext] if msgtype == "m.image": return "image/jpeg" - if msgtype == "m.audio": - return "audio/ogg" return "application/octet-stream" @@ -301,10 +297,6 @@ def __init__(self, homeserver, user_id, password, session_dir, **settings): self.link_base_url = os.environ.get("LINK_BASE_URL", "") self.openai_url = os.environ.get("OPENAI_URL", "") self.openai_key = os.environ.get("OPENAI_KEY", "") - # Voice transcription endpoint. When unset (e.g. the AI stacklet - # isn't installed yet), the archivist still files PDFs/images; only - # the audio-capture path becomes a soft-skip with a friendly reply. - self.whisper_url = os.environ.get("WHISPER_URL", "") self.language = os.environ.get("LANGUAGE", "en") # Per-bot settings from stacklet.toml [bots.archivist.settings] self.classify_enabled = settings.get("classify", True) @@ -378,7 +370,6 @@ def __init__(self, homeserver, user_id, password, session_dir, **settings): self._pipeline: DocumentPipeline | None = None self._search: SearchService | None = None self._capture: CapturePipeline | None = None - self._transcriber: Transcriber | None = None self._vault: VaultContext | None = None self._paperless_version: str = "" @@ -474,27 +465,6 @@ async def start(self) -> None: shared_bucket=self.shared_bucket, vault=self._vault, ) - # Whisper speaks /v1/audio/transcriptions on its own port, so the - # Transcriber gets its own client. When WHISPER_URL is unset the - # archivist still works for PDFs/images; only the voice-capture - # branch in the pipeline soft-skips. - if self.whisper_url: - try: - self._transcriber = Transcriber.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning("[archivist] no transcription: {}", e) - self._transcriber = None - else: - logger.info( - "[archivist] WHISPER_URL unset — voice messages will be ignored " - "(set up AI with 'stack up ai' to enable transcription)" - ) - - # The capture pipeline borrows the classifier's framework LLM for - # transcript cleanup -- punctuating raw whisper output with the - # model that's already running. When the classifier isn't built - # (AI not configured), cleanup soft-skips and the raw transcript - # falls through. capture_llm = self._classifier.llm if self._classifier is not None else None self._capture = CapturePipeline( url_extractor=self._url_extractor, @@ -508,7 +478,6 @@ async def start(self) -> None: capture_keep_body=self.capture_keep_body, capture_tag_prompt_size=self.capture_tag_prompt_size, vision_max_pdf_pages=self.vision_max_pdf_pages, - transcriber=self._transcriber, llm=capture_llm, ) # Warm the vision-capability cache on every boot. Previously @@ -1508,17 +1477,6 @@ async def _on_file(self, room, event) -> None: if msgtype not in SUPPORTED_MSGTYPES: return - # Voice messages depend on the optional whisper service. When the - # transcriber wasn't wired (WHISPER_URL unset or whisper missing - # at boot), silently ignore audio so the bot still files PDFs and - # images normally; the startup log already warned the admin. - if msgtype == "m.audio" and self._transcriber is None: - logger.debug( - "[archivist] ignoring voice from {} (transcription disabled)", - event.sender, - ) - return - url = content.get("url", "") if not url or not url.startswith("mxc://"): return @@ -1552,20 +1510,12 @@ async def _on_file(self, room, event) -> None: display_name = _clean_filename(raw_filename, msgtype) reply_to = event.event_id - # Multi-page scan / multi-message batch mode. PDFs and images - # take the existing page-accumulator path; voice memos take the - # transcription-and-accumulate path so a single `(` ... `)` - # session can collect either kind (or both, filed as separate - # captures on close). + # Multi-page scan mode: accumulate pages until `)` closes the + # session and the batch is filed as one document. if event.sender in self._scan_sessions: - if msgtype == "m.audio": - await self._handle_voice_batch_message( - room.room_id, event, url, raw_filename, - ) - else: - await self._handle_scan_page( - room.room_id, event, url, raw_filename, caption, - ) + await self._handle_scan_page( + room.room_id, event, url, raw_filename, caption, + ) return file_data = await self._download_media(url) @@ -1592,11 +1542,7 @@ async def _on_file(self, room, event) -> None: # capture room: PDFs and images become visual bookmarks in the # sender's bucket; Matrix already stores the binary, so we link # to the mxc URL and don't re-archive the bytes. - # - # Voice memos always take the capture path, even in #documents: - # a transcript belongs in the sender's own notes, not in Paperless - # alongside scanned invoices and IDs. - if self._is_documents_room(ctx) and msgtype != "m.audio": + if self._is_documents_room(ctx): await self._process_document( room.room_id, raw_filename, display_name, file_data, reply_to, date_filed=self._event_date(event), @@ -1660,6 +1606,11 @@ async def _on_text(self, room, event: RoomMessageText) -> None: return query_lower = query.lower() reply_to = event.event_id + # Whether these words were spoken rather than typed. Used twice + # below, and for nothing else: never to route, only to decide + # whether the message was handed to us deliberately and whether + # to quote it back for checking. + transcribed = voice.was_transcribed(event.source.get("content", {})) ctx = self._room_context(room) # First-encounter welcome -- idempotent, gated by a per-room @@ -1738,8 +1689,7 @@ async def _on_text(self, room, event: RoomMessageText) -> None: elif (begin := _split_scan_command(query, SCAN_BEGIN))[0]: sender_name = event.sender.split(":")[0].replace("@", "").capitalize() self._scan_sessions[event.sender] = { - "files": [], "voice_inputs": [], - "room_id": room.room_id, "caption": begin[1], + "files": [], "room_id": room.room_id, "caption": begin[1], } await self._send(room.room_id, self.t("scan_started", sender=sender_name), reply_to) @@ -1811,7 +1761,9 @@ async def _on_text(self, room, event: RoomMessageText) -> None: sender=event.sender, ) - elif self._looks_like_paste(query) and self._count_humans_in_room(room) < 2: + elif transcribed or ( + self._looks_like_paste(query) and self._count_humans_in_room(room) < 2 + ): # One person in the room, paste-shaped message, no mention → # file as text capture. Nobody is being talked *to* here, so # a long message is material dropped for us to keep. @@ -1826,10 +1778,21 @@ async def _on_text(self, room, event: RoomMessageText) -> None: # # Links and files are unaffected in either case: pasting a URL # or dropping a PDF is a deliberate act, not a turn in a - # conversation. + # conversation. A voice memo is the same kind of act: nobody + # holds the mic button for thirty seconds as a turn in a chat, + # and the length heuristic that stands in for deliberateness + # elsewhere cannot read it. So speech carries itself past this + # gate, in any room, exactly as an upload does. + # + # This is not speech being routed differently. Everything + # above still applies to it unchanged -- a memo addressed to + # the bot is a question, in the documents room it is a search, + # in a thread it obeys ownership. Only "was this handed to us + # on purpose?" is answered another way, because for speech + # there is another way to answer it. await self._handle_text_capture( room.room_id, query, event.sender, reply_to, - capture_id=event.event_id, + capture_id=event.event_id, transcribed=transcribed, ) else: @@ -2153,82 +2116,21 @@ async def _handle_scan_page( # the timeline clean; the scan-complete reply is the closure. await self._react(room_id, reply_to, EYES) - async def _handle_voice_batch_message( - self, room_id: str, event, url: str, raw_filename: str, - ): - """Transcribe a voice memo and stash it on the open batch session. - - Mirrors _handle_scan_page: download, normalise input, append to - the session, ack the count. The transcript is computed eagerly - (with the LLM cleanup pass) so by the time `)` arrives we just - concatenate ready text -- no extra round-trip on close. - """ - reply_to = event.event_id - if self._transcriber is None: - await self._send( - room_id, self.t("scan_voice_no_transcriber"), reply_to, - ) - return - try: - audio = await self._download_media(url) - except Exception as e: - await self._send( - room_id, self.t("scan_voice_failed", error=str(e)), reply_to, - ) - return - if not audio: - await self._send( - room_id, self.t("scan_voice_failed_matrix"), reply_to, - ) - return - - cleanup_llm = self._classifier.llm if self._classifier is not None else None - try: - transcript = await self._transcriber.transcribe( - audio, filename=raw_filename or "voice.ogg", - cleanup_with=cleanup_llm, - ) - except LLMError as e: - logger.warning("[archivist] batch voice transcribe failed: {}", e) - await self._send( - room_id, self.t("scan_voice_failed", error=str(e)), reply_to, - ) - return - - if not transcript.strip(): - await self._send(room_id, self.t("scan_voice_empty"), reply_to) - return - - session = self._scan_sessions[event.sender] - session["voice_inputs"].append({ - "transcript": transcript, - "mxc": url, - "event_id": event.event_id, - }) - # 👀 acknowledges the memo landed in the batch; the scan-complete - # reply is the closure (mirrors _handle_scan_page). - await self._react(room_id, reply_to, EYES) - async def _handle_scan_complete( self, room_id: str, sender: str, reply_to: str | None = None, *, date_filed: str | None = None, ): session = self._scan_sessions.pop(sender) files = session["files"] - voice_inputs = session.get("voice_inputs") or [] caption = session.get("caption", "").strip() sender_name = sender.split(":")[0].replace("@", "").capitalize() # Nothing accumulated -- the user opened a session and closed it # without sending anything. Treat as cancelled, same as today. - if not files and not voice_inputs: + if not files: await self._send(room_id, self.t("scan_cancelled"), reply_to) return - # Files (PDFs / images) take the existing PDF combine path. A - # mixed batch files both: the PDF as a document and the voice - # memos as a separate note below, since the vault data model - # has one body per capture. if files: if len(files) == 1: filename, file_data = files[0] @@ -2267,40 +2169,6 @@ async def _handle_scan_complete( submitter_mxid=sender, ) - if voice_inputs: - await self._handle_voice_batch_complete( - room_id, sender, voice_inputs, reply_to, - ) - - async def _handle_voice_batch_complete( - self, room_id: str, sender: str, voice_inputs: list[dict], - reply_to: str | None, - ): - """File the accumulated voice memos as one combined note. - - Each input already carries a cleaned transcript (the LLM pass - ran at message time), so this just hands the list to the - capture pipeline and renders the resulting reply. We post a - progress line first because the classify call is the slowest - step and the sender deserves to know we're working. - """ - n = len(voice_inputs) - await self._send( - room_id, self.t("scan_voice_complete", count=n), reply_to, - ) - binding = await self._topic_binding( - self._room_by_id(room_id), sender, - ) - outcome = await self._capture.capture_voice_batch( - transcripts=[v["transcript"] for v in voice_inputs], - primary_mxc=voice_inputs[0].get("mxc"), - sender_mxid=sender, - capture_id=voice_inputs[0].get("event_id"), - seed_topics=binding.seed_topics if binding else None, - bucket=binding.bucket if binding else None, - ) - await self._reply_for_capture(room_id, outcome, reply_to) - # ── URL capture (knowledge rooms, DMs, per-person notes rooms) ─────── # # Capture mode is what runs when the bot sees a URL in any room that @@ -2353,6 +2221,7 @@ async def _handle_capture( async def _handle_text_capture( self, room_id: str, text: str, sender_mxid: str, reply_to: str | None = None, *, capture_id: str | None = None, + transcribed: bool = False, ) -> None: # 👀 before the work, like every other capture shape: a link gets # it from the pipeline's notifier, an upload from the handler. A @@ -2370,6 +2239,7 @@ async def _handle_text_capture( capture_id=capture_id, seed_topics=binding.seed_topics if binding else None, bucket=binding.bucket if binding else None, + transcribed=transcribed, ) await self._reply_for_capture(room_id, outcome, reply_to) @@ -2447,7 +2317,6 @@ async def _reply_for_capture( # back to the original URL-shaped message for legacy callers. failure_keys = { "url": "capture_failed", - "transcription": "capture_failed_transcription", "binary": "capture_failed_binary", } key = failure_keys.get(o.failure_reason or "", "capture_failed") diff --git a/stacklets/docs/bot/capture_pipeline.py b/stacklets/docs/bot/capture_pipeline.py index 856cb677..d3a5b0bc 100644 --- a/stacklets/docs/bot/capture_pipeline.py +++ b/stacklets/docs/bot/capture_pipeline.py @@ -41,7 +41,6 @@ LLMUnavailableError, ) from stack import resolve_model -from stack.ai.client import LLMError try: from pypdf import PdfReader # type: ignore @@ -65,11 +64,6 @@ } TEXT_EXTS = {"md", "markdown", "txt"} -# Voice memos: when whisper is wired, the transcript IS the note body. -# Element and most Matrix clients upload Opus-in-OGG; the other formats -# show up when people attach a file picker rather than the in-app recorder. -AUDIO_MIME_PREFIX = "audio/" - # Ratio that the reformatted body must be of the input for the # reformat to count as a useful rewrite. A well-behaved reformat # tightens whitespace and stitches broken lines but keeps essentially @@ -131,7 +125,6 @@ def __init__( capture_keep_body: bool, capture_tag_prompt_size: int, vision_max_pdf_pages: int = DEFAULT_VISION_MAX_PDF_PAGES, - transcriber=None, llm=None, ): self._url_extractor = url_extractor @@ -145,13 +138,6 @@ def __init__( self.capture_keep_body = capture_keep_body self.capture_tag_prompt_size = capture_tag_prompt_size self.vision_max_pdf_pages = vision_max_pdf_pages - # Optional: when None, audio uploads soft-skip with extract_failed - # so the bot tells the sender it can't transcribe right now. - self._transcriber = transcriber - # Optional: when present, the transcriber uses it to polish raw - # whisper output (add punctuation + sentence breaks). The same - # LLM the classifier already runs on -- no extra HTTP client. - # Missing -> raw transcript falls through, never a hard failure. self._llm = llm async def capture_url( @@ -205,23 +191,32 @@ async def capture_text( capture_id: str | None = None, seed_topics: list[str] | None = None, bucket: str | None = None, + transcribed: bool = False, ) -> CaptureOutcome: - """File a pasted body as a note. Nothing is fetched — the text is + """File a message body as a note. Nothing is fetched — the text is the source; TextExtractor surfaces any embedded URL as the link. - The body is always kept (the user typed those exact bytes). + The body is always kept (those are the exact words sent). ``seed_topics`` and ``bucket`` carry the topic-room guarantees through; see ``capture_url``. + + ``transcribed`` marks a body that whisper produced rather than a + keyboard. It changes nothing about how the note is filed — only + whether the reply quotes the words back, which is the sender's + one chance to catch a mishearing. """ source = await self._text_extractor.extract(text) if source is None: return CaptureOutcome(status="empty") + # The footer says where a capture came from. "(pasted text)" is a + # lie about something somebody said out loud. + origin = "(voice message)" if transcribed else "(pasted text)" return await self._publish( source=source, kind="note", sender_mxid=sender_mxid, - display_link=source.source_uri or "(pasted text)", + display_link=source.source_uri or origin, actor=sender_mxid, capture_id=capture_id, seed_topics=seed_topics, - bucket=bucket, + bucket=bucket, transcribed=transcribed, ) async def capture_email( @@ -267,48 +262,6 @@ async def capture_email( default_person=False, ) - async def capture_voice_batch( - self, *, - transcripts: list[str], - primary_mxc: str | None, - sender_mxid: str, - capture_id: str | None = None, - seed_topics: list[str] | None = None, - bucket: str | None = None, - ) -> CaptureOutcome: - """File N voice memos as a single combined note. - - Used by the `( ... )` batch flow: each voice memo has already - been transcribed (and LLM-cleaned) on arrival, so this just - concatenates them with paragraph breaks and ships the result - through the standard publish path. ``primary_mxc`` is the - first memo's media URL -- the vault note links back to the - start of the conversation; other memos stay discoverable via - chat scrollback. - - Marking the SourceContent as ``audio/ogg`` makes ``_publish`` - echo the combined transcript in the capture reply, same as - single-memo captures, so the sender sees the whole batch's - text quoted back to them. - """ - bodies = [t.strip() for t in transcripts if t and t.strip()] - if not bodies: - return CaptureOutcome(status="empty") - combined = "\n\n".join(bodies) - source = SourceContent( - text=combined, - mime="audio/ogg", - title_hint=None, - source_uri=primary_mxc, - ) - return await self._publish( - source=source, kind="note", sender_mxid=sender_mxid, - display_link=primary_mxc or "(voice batch)", - actor=sender_mxid, - capture_id=capture_id, seed_topics=seed_topics, - bucket=bucket, - ) - async def capture_binary( self, *, file_data: bytes, @@ -333,21 +286,13 @@ async def capture_binary( wiki entry links back to the original binary -- we don't re-store the bytes; Matrix already has them. """ - is_audio = bool(mime and mime.startswith(AUDIO_MIME_PREFIX)) - if is_audio: - source, images, kind = await self._source_from_audio( - file_data=file_data, mime=mime, filename=filename, - source_uri=source_uri, - ) - else: - source, images, kind = self._source_from_binary( - file_data=file_data, mime=mime, filename=filename, - source_uri=source_uri, - ) + source, images, kind = self._source_from_binary( + file_data=file_data, mime=mime, filename=filename, + source_uri=source_uri, + ) if source is None: return CaptureOutcome( - status="extract_failed", - failure_reason="transcription" if is_audio else "binary", + status="extract_failed", failure_reason="binary", ) source = self._cap_pdf_body(source) source = await self._maybe_reformat_pdf(source, kind) @@ -422,49 +367,6 @@ async def _maybe_reformat_pdf( source_uri=source.source_uri, ) - async def _source_from_audio( - self, *, file_data: bytes, mime: str, filename: str, - source_uri: str | None, - ) -> tuple[SourceContent | None, list[ImageAttachment], str]: - """Transcribe a voice memo into a note-shaped SourceContent. - - The transcript IS the note body: same kind ("note") that md/txt - uploads use, so the classifier sees the transcribed text and the - mirror writes the words verbatim. Returning ``None`` signals the - capture as extract_failed -- the orchestrator already renders a - friendly reply for that case, so a missing/down whisper service - surfaces uniformly across PDFs and audio. - """ - if self._transcriber is None: - logger.info( - "[capture] audio dropped: no transcriber configured " - "(WHISPER_URL unset?)" - ) - return (None, [], "note") - try: - transcript = await self._transcriber.transcribe( - file_data, filename=filename or "voice.ogg", - cleanup_with=self._llm, - ) - except LLMError as e: - # Same failure shape the LLM client uses for chat outages; - # the orchestrator already maps extract_failed to a user- - # visible "I couldn't process that" reply. - logger.warning("[capture] transcription failed: {}", e) - return (None, [], "note") - if not transcript.strip(): - return (None, [], "note") - return ( - SourceContent( - text=transcript, - mime=mime, - title_hint=filename or None, - source_uri=source_uri, - ), - [], - "note", - ) - def _source_from_binary( self, *, file_data: bytes, mime: str, filename: str, source_uri: str | None, @@ -651,6 +553,7 @@ async def _publish( bucket: str | None = None, email_meta: dict | None = None, default_person: bool = True, + transcribed: bool = False, ) -> CaptureOutcome: """Shared tail: classify, mirror, record tags, return the outcome. @@ -777,12 +680,11 @@ async def _publish( if user_hint: envelope["data"]["user_hint"] = user_hint - # Voice captures echo the transcript back to the sender so they - # can spot a mistranscription without opening the vault. Other - # capture shapes (PDFs, images, URLs, notes) leave it None. - source_mime = getattr(source, "mime", None) or "" - is_audio_source = source_mime.startswith(AUDIO_MIME_PREFIX) - transcript = source.text if is_audio_source else None + # Transcribed captures echo the words back to the sender so they + # can spot a mishearing without opening the vault. The caller says + # so; by the time the text reaches here it is an ordinary note, + # indistinguishable from a typed one, which is the point. + transcript = source.text if transcribed else None return CaptureOutcome( status="reclassified" if reclassified else "captured", diff --git a/stacklets/docs/bot/cli/capture.py b/stacklets/docs/bot/cli/capture.py index 04632996..0b6102ae 100644 --- a/stacklets/docs/bot/cli/capture.py +++ b/stacklets/docs/bot/cli/capture.py @@ -177,7 +177,6 @@ def render_receipt(outcome) -> str: what = { "url": "that link", - "transcription": "that voice memo", "binary": "that file", }.get(getattr(outcome, "failure_reason", None), "the content") return f"Nothing captured: could not read {what}." diff --git a/stacklets/docs/bot/messages/archivist.yml b/stacklets/docs/bot/messages/archivist.yml index fef7d140..ce326967 100644 --- a/stacklets/docs/bot/messages/archivist.yml +++ b/stacklets/docs/bot/messages/archivist.yml @@ -71,11 +71,6 @@ en: scan_complete_single: "\U0001F4CB Scan complete (1 page) — analyzing..." scan_complete_multi: "\U0001F4CB Scan complete — combining {count} pages into PDF..." scan_combine_failed: "\u274C Failed to combine pages into PDF: {error}" - scan_voice_failed: "\u274C Failed to download voice memo: {error}" - scan_voice_failed_matrix: "\u274C Failed to download voice memo from Matrix." - scan_voice_no_transcriber: "\U0001F3A4 Voice transcription is not configured. Set up AI with 'stack up ai'." - scan_voice_empty: "\U0001F3A4 Couldn't hear anything in that voice memo." - scan_voice_complete: "\U0001F3A4 Combining {count} voice memos into one note. Analyzing..." no_active_scan: "No active scan. Send ( or 'scan' to start one." # URL archiving @@ -89,7 +84,6 @@ en: # URL capture (knowledge rooms — links become summarized notes, not Paperless docs) capture_failed: "\u274C Couldn't read that link. Either the host is unreachable or there's no article body to extract." - capture_failed_transcription: "\U0001F3A4 Couldn't transcribe that voice memo. Whisper may be down or misconfigured." capture_failed_binary: "\u274C Couldn't read that file." capture_llm_failed: "\U0001F4DD Captured the page, but classification skipped: {error}" capture_no_mirror: "\u274C Captures need the `code` stacklet up so I can write to the memory vault. Run `stack up code`." @@ -280,11 +274,6 @@ de: scan_complete_single: "\U0001F4CB Scan abgeschlossen (1 Seite) — wird analysiert..." scan_complete_multi: "\U0001F4CB Scan abgeschlossen — {count} Seiten werden zu PDF zusammengefasst..." scan_combine_failed: "\u274C Seiten konnten nicht zu PDF zusammengefasst werden: {error}" - scan_voice_failed: "\u274C Sprachnachricht konnte nicht heruntergeladen werden: {error}" - scan_voice_failed_matrix: "\u274C Sprachnachricht konnte nicht aus Matrix heruntergeladen werden." - scan_voice_no_transcriber: "\U0001F3A4 Sprach-Transkription ist nicht eingerichtet. 'stack up ai' ausführen." - scan_voice_empty: "\U0001F3A4 Konnte nichts in der Sprachnachricht hören." - scan_voice_complete: "\U0001F3A4 Kombiniere {count} Sprachnachrichten zu einer Notiz. Wird analysiert..." no_active_scan: "Kein aktiver Scan. Sende ( oder 'scan' um einen zu starten." # URL archiving @@ -298,7 +287,6 @@ de: # URL-Capture (Wissensräume — Links werden zu zusammengefassten Notizen, nicht zu Paperless-Dokumenten) capture_failed: "\u274C Konnte den Link nicht lesen. Entweder ist die Seite nicht erreichbar, oder es gibt keinen Artikel-Inhalt zum Extrahieren." - capture_failed_transcription: "\U0001F3A4 Konnte die Sprachnachricht nicht transkribieren. Whisper ist möglicherweise nicht erreichbar oder falsch konfiguriert." capture_failed_binary: "\u274C Konnte diese Datei nicht lesen." capture_llm_failed: "\U0001F4DD Seite gespeichert, aber Klassifizierung übersprungen: {error}" capture_no_mirror: "\u274C Captures brauchen das `code`-Stacklet, damit ich in den Memory-Vault schreiben kann. Starte `stack up code`." diff --git a/stacklets/messages/bot/bot.toml b/stacklets/messages/bot/bot.toml deleted file mode 100644 index 90727671..00000000 --- a/stacklets/messages/bot/bot.toml +++ /dev/null @@ -1,3 +0,0 @@ -id = "scribe-bot" -name = "Scribe" -description = "Transcribes voice messages" diff --git a/stacklets/messages/bot/scribe.py b/stacklets/messages/bot/scribe.py deleted file mode 100644 index 3a0d99d1..00000000 --- a/stacklets/messages/bot/scribe.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Scribe — voice message transcription bot. - -Send a voice message in any room where Scribe is present, and it replies -with the transcribed text. Uses whisper.cpp running natively on the host -for Metal GPU acceleration — a 5-minute voice memo transcribes in ~20 -seconds, vs 10+ minutes CPU-only inside Docker. - -The transcription API call is delegated to the shared `Transcriber` -capability on the AI client; the bot is just the Matrix-side wiring — -download audio, hand bytes to the Transcriber, post the result. -""" - -from loguru import logger -from nio import ( - AsyncClient, - RoomMessageAudio, -) - -from microbot import MicroBot -from stack.ai.client import LLM, LLMError, LLMUnavailableError, Transcriber - - -class ScribeBot(MicroBot): - name = "scribe-bot" - - def __init__(self, homeserver, user_id, password, session_dir, **config): - super().__init__(homeserver, user_id, password, session_dir, **config) - # The Transcriber owns the HTTP client; we don't have a clean - # shutdown hook in the MicroBot base, so we accept that the - # underlying client lives for the lifetime of the bot process. - # - # When WHISPER_URL isn't set (e.g. the AI stacklet hasn't been - # installed yet) the constructor would otherwise crash here. We - # degrade to a no-op bot instead: still in the room, still - # joinable, but silent on voice messages until whisper is wired. - try: - self._transcriber = Transcriber.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning( - "[scribe] transcription disabled: {} — " - "voice messages will be ignored until WHISPER_URL is set", - e, - ) - self._transcriber = None - - # The LLM is optional: when present, it polishes raw whisper - # output into a punctuated paragraph (the Transcriber owns the - # prompt). When absent, scribe still posts the raw transcript -- - # less readable but never a blocker. - try: - self._llm = LLM.from_env(namespace=self.name) - except LLMUnavailableError as e: - logger.warning( - "[scribe] transcript cleanup disabled: {} — " - "voice messages will post the raw whisper output", - e, - ) - self._llm = None - - def register_callbacks(self, client: AsyncClient) -> None: - self.add_event_callback(self._on_voice, RoomMessageAudio) - - async def _on_voice(self, room, event: RoomMessageAudio) -> None: - if event.sender == self.user_id: - return - if self._transcriber is None: - # The startup warning already told the admin why; don't - # spam a per-message error reply or a typing indicator that - # leads nowhere. - return - - logger.info("[scribe] Voice from {} in {}", event.sender, room.room_id) - await self._set_typing(room.room_id, on=True) - - audio = await self._download_media(event.url) - if audio is None: - await self._set_typing(room.room_id, on=False) - logger.error("[scribe] Download failed for {}", event.url) - return - - filename = event.body if event.body else "voice.ogg" - text = "" - try: - text = await self._transcriber.transcribe( - audio, filename=filename, cleanup_with=self._llm, - ) - except LLMError as e: - # The Transcriber maps every transport / API failure to an - # LLMError; we log and fall through to the empty-text branch - # so the family sees a friendly reply instead of silence. - logger.error("[scribe] Transcription failed: {}", e) - await self._set_typing(room.room_id, on=False) - - if text: - logger.info("[scribe] Transcribed: {}...", text[:80]) - await self._send( - room.room_id, f"**Transcription:**\n\n{text}", - reply_to=event.event_id, - ) - else: - await self._send( - room.room_id, "Sorry, I couldn't transcribe that audio.", - ) diff --git a/stacklets/messages/cli/upload.py b/stacklets/messages/cli/upload.py index e695ba7d..8e5128b5 100644 --- a/stacklets/messages/cli/upload.py +++ b/stacklets/messages/cli/upload.py @@ -14,7 +14,8 @@ The user's password is read from the secrets store (the same place the account was created), so the sender shows up correctly in the timeline and the archivist's "received from " reply. Images (.png/.jpg) are sent -as m.image; everything else as m.file. +as m.image, audio (.ogg/.opus/.m4a/.mp3/.wav) as m.audio so it is treated +as a voice message and transcribed; everything else as m.file. """ HELP = "Upload a file to a chat room" @@ -27,7 +28,10 @@ from _matrix import MatrixClient, resolve_login # Extension → (mime type, Matrix msgtype). Images go as m.image so the -# archivist takes its vision path; everything else is m.file. +# archivist takes its vision path. Audio goes as m.audio, which is what a +# phone's mic button sends, so the framework transcribes it and the bots +# see the words — the only way to exercise voice from the terminal. +# Everything else is m.file. _MIME = { ".pdf": ("application/pdf", "m.file"), ".png": ("image/png", "m.image"), @@ -36,6 +40,11 @@ ".webp": ("image/webp", "m.image"), ".gif": ("image/gif", "m.image"), ".txt": ("text/plain", "m.file"), + ".ogg": ("audio/ogg", "m.audio"), + ".opus": ("audio/ogg", "m.audio"), + ".m4a": ("audio/mp4", "m.audio"), + ".mp3": ("audio/mpeg", "m.audio"), + ".wav": ("audio/wav", "m.audio"), } diff --git a/tests/stacklets/test_archivist_corrections.py b/tests/stacklets/test_archivist_corrections.py index 8bd9d9b8..d8b7ab21 100644 --- a/tests/stacklets/test_archivist_corrections.py +++ b/tests/stacklets/test_archivist_corrections.py @@ -189,7 +189,8 @@ async def _capture_reprocess(room_id, vault_path, user_hint, sender_mxid, reply_to, *, initial_classification=None): bot.routed.append(("capture_reprocess", vault_path, user_hint)) - async def _text_capture(room_id, text, sender, reply_to=None, *, capture_id=None): + async def _text_capture(room_id, text, sender, reply_to=None, *, + capture_id=None, transcribed=False): bot.routed.append(("capture_text", text)) async def _send(room_id, text, *a, **kw): diff --git a/tests/stacklets/test_archivist_routing.py b/tests/stacklets/test_archivist_routing.py index af70d3fe..12055b4a 100644 --- a/tests/stacklets/test_archivist_routing.py +++ b/tests/stacklets/test_archivist_routing.py @@ -27,6 +27,7 @@ sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "core" / "bot-runner")) sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "docs" / "bot")) +import voice # noqa: E402 from archivist import ArchivistBot # noqa: E402 @@ -712,6 +713,51 @@ def test_only_quoted_block_returns_empty(self): # ── Mention as routing signal ───────────────────────────────────────────── +@pytest.fixture +def bot_with_recorder(tmp_path): + """Build a bot wired with stubs that record handler dispatches. + + Avoids any I/O. The handlers under test (_handle_search, + _handle_capture, _handle_text_capture) are replaced with + recorders so we can assert which branch fired with which query. + """ + bot = _build_bot(tmp_path) + calls: list[tuple[str, str]] = [] + + async def _record_search(room_id, query, reply_to=None, *, sender=None): + calls.append(("search", query)) + + async def _record_capture(room_id, url, sender, reply_to=None, + *, capture_id=None): + calls.append(("capture_url", url)) + + async def _record_text_capture(room_id, text, sender, reply_to=None, + *, capture_id=None, transcribed=False): + calls.append(("capture_text", text)) + + async def _record_url(room_id, url, reply_to=None, **kw): + calls.append(("paperless_url", url)) + + async def _record_send(*a, **kw): + calls.append(("send", a[1] if len(a) > 1 else "")) + + bot._handle_search = _record_search + bot._handle_capture = _record_capture + bot._handle_text_capture = _record_text_capture + bot._handle_url = _record_url + bot._send = _record_send + # Correction lookup needs the client; short-circuit it. What it + # resolves is pinned in test_archivist_corrections.py. + bot._correction_anchor = lambda *_a, **_kw: _none_coro() + # The per-room welcome path runs ahead of routing decisions in + # `_on_text` / `_on_file`. These tests focus on the routing + # dispatch, not the welcome -- stub it out so the recorded + # calls list stays clean. The welcome itself has its own test + # file: tests/stacklets/test_archivist_welcome.py. + bot._send_room_welcome_if_needed = lambda *_a, **_kw: _none_coro() + return bot, calls + + class TestMentionRoutesToSearch: """Mention promotes any free-text message to a search query, in any room. The mxid is stripped first so command matching and search see @@ -744,49 +790,6 @@ def _room_obj(*, alias="#family-chat:server", members=None): users={uid: object() for uid in (members or [BOT_ID, "@homer:server", "@marge:server"])}, ) - @pytest.fixture - def bot_with_recorder(self, tmp_path): - """Build a bot wired with stubs that record handler dispatches. - - Avoids any I/O. The handlers under test (_handle_search, - _handle_capture, _handle_text_capture) are replaced with - recorders so we can assert which branch fired with which query. - """ - bot = _build_bot(tmp_path) - calls: list[tuple[str, str]] = [] - - async def _record_search(room_id, query, reply_to=None, *, sender=None): - calls.append(("search", query)) - - async def _record_capture(room_id, url, sender, reply_to=None, - *, capture_id=None): - calls.append(("capture_url", url)) - - async def _record_text_capture(room_id, text, sender, reply_to=None, - *, capture_id=None): - calls.append(("capture_text", text)) - - async def _record_url(room_id, url, reply_to=None, **kw): - calls.append(("paperless_url", url)) - - async def _record_send(*a, **kw): - calls.append(("send", a[1] if len(a) > 1 else "")) - - bot._handle_search = _record_search - bot._handle_capture = _record_capture - bot._handle_text_capture = _record_text_capture - bot._handle_url = _record_url - bot._send = _record_send - # Correction lookup needs the client; short-circuit it. What it - # resolves is pinned in test_archivist_corrections.py. - bot._correction_anchor = lambda *_a, **_kw: _none_coro() - # The per-room welcome path runs ahead of routing decisions in - # `_on_text` / `_on_file`. These tests focus on the routing - # dispatch, not the welcome -- stub it out so the recorded - # calls list stays clean. The welcome itself has its own test - # file: tests/stacklets/test_archivist_welcome.py. - bot._send_room_welcome_if_needed = lambda *_a, **_kw: _none_coro() - return bot, calls @pytest.mark.asyncio async def test_mention_in_group_room_routes_to_search(self, bot_with_recorder): @@ -913,3 +916,97 @@ async def _none_coro(): async def _resp(event): """A `room_get_event` response wrapping one event.""" return SimpleNamespace(event=event) + + +class TestSpokenMessagesAreDeliberate: + """A voice memo is a deliberate handover, the way a link or an upload + is. Holding the mic button for thirty seconds is not a turn in a + conversation, so the ambient-capture gate that silences long *typed* + messages in a shared room must not silence it. + + Routing is otherwise identical to typed text — that is the whole + point of decoding speech in the transport. What differs is only the + "is this material or chatter?" heuristic, which is the same judgement + the gate already makes in favour of links and files. + """ + + @staticmethod + def _event(body, *, transcribed, sender="@homer:server"): + content: dict = {} + if transcribed: + content[voice.TRANSCRIPT_KEY] = { + "source_msgtype": "m.audio", + "url": "mxc://server/abc", + } + return SimpleNamespace( + body=body, sender=sender, event_id="$evt:server", + server_timestamp=1, source={"content": content}, + ) + + @staticmethod + def _shared_room(): + """A topic room with two humans in it — the shape that closes the + ambient gate for typed text.""" + return SimpleNamespace( + room_id="!room:server", + canonical_alias="#topic-camping:server", + name="Topic: Camping", + users={uid: object() for uid in + (BOT_ID, "@homer:server", "@marge:server")}, + ) + + # Comfortably under the 100-char paste threshold, so nothing but the + # spoken-ness can be carrying it past the gate. + SHORT = "Book the campsite for July." + + @pytest.mark.asyncio + async def test_a_memo_is_captured_in_a_room_with_other_people( + self, bot_with_recorder, + ): + bot, calls = bot_with_recorder + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=True), + ) + assert calls == [("capture_text", self.SHORT)] + + @pytest.mark.asyncio + async def test_the_same_words_typed_stay_chatter(self, bot_with_recorder): + """The contrast that makes the rule honest: it is the deliberate + act being recognised, not the words.""" + bot, calls = bot_with_recorder + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=False), + ) + assert calls == [] + + @pytest.mark.asyncio + async def test_the_capture_is_told_the_words_were_transcribed( + self, bot_with_recorder, + ): + """So the reply quotes them back — the sender's only chance to + catch whisper mishearing them without opening the vault.""" + bot, seen = bot_with_recorder + flags: list[bool] = [] + + async def _record(room_id, text, sender, reply_to=None, *, + capture_id=None, transcribed=False): + flags.append(transcribed) + + bot._handle_text_capture = _record + await bot._on_text( + self._shared_room(), self._event(self.SHORT, transcribed=True), + ) + assert flags == [True] + + @pytest.mark.asyncio + async def test_a_mention_still_asks_rather_than_files( + self, bot_with_recorder, + ): + """Speaking to the bot is asking it something, exactly as typing + to it is. Deliberateness decides capture, never who is addressed.""" + bot, calls = bot_with_recorder + event = self._event("Archivist, where is the campsite booking?", + transcribed=True) + event.source["content"]["m.mentions"] = {"user_ids": [BOT_ID]} + await bot._on_text(self._shared_room(), event) + assert [c[0] for c in calls] == ["search"] diff --git a/tests/stacklets/test_capture_pipeline.py b/tests/stacklets/test_capture_pipeline.py index a5c704fe..66b759da 100644 --- a/tests/stacklets/test_capture_pipeline.py +++ b/tests/stacklets/test_capture_pipeline.py @@ -120,10 +120,11 @@ async def acknowledge(self): def _pipeline(*, mirror, classifier=None, capture_keep_body=False, - transcriber=None, llm=None): + llm=None, text_extractor=None): return CapturePipeline( url_extractor=FakeExtractor(_source(source_uri="http://src")), - text_extractor=FakeExtractor(_source(source_uri="http://embedded")), + text_extractor=text_extractor + or FakeExtractor(_source(source_uri="http://embedded")), classifier=classifier or FakeClassifier(), mirror=mirror, capture_tags=FakeTags(), @@ -132,37 +133,10 @@ def _pipeline(*, mirror, classifier=None, capture_keep_body=False, classify_max_chars=10000, capture_keep_body=capture_keep_body, capture_tag_prompt_size=50, - transcriber=transcriber, llm=llm, ) -class FakeTranscriber: - """Stand-in for stack.ai.client.Transcriber — records calls and - returns a configured transcript or raises a configured error. - - Mirrors the real signature including ``cleanup_with`` so the capture - pipeline can pass an LLM through and we can assert the routing. - """ - - def __init__(self, transcript: str = "I forgot to renew the boiler service", - error: Exception | None = None): - self.transcript = transcript - self.error = error - self.calls: list[dict] = [] - - async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", - model: str | None = None, - cleanup_with=None) -> str: - self.calls.append({ - "audio": audio, "filename": filename, - "cleanup_with": cleanup_with, - }) - if self.error is not None: - raise self.error - return self.transcript - - class TestCaptureUrl: @pytest.mark.asyncio @@ -236,88 +210,6 @@ async def test_empty_text_is_dropped(self): assert out.status == "empty" -class TestCaptureVoiceBatch: - """The ( ... ) batch flow: N voice memos arrive, each gets transcribed - on its way in (with LLM cleanup if available), then `)` combines them - into one note via capture_voice_batch.""" - - @pytest.mark.asyncio - async def test_concatenates_transcripts_with_paragraph_breaks(self): - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=[ - "First memo about the boiler.", - "Second memo about Bart's prescription.", - "Third memo with shopping list.", - ], - primary_mxc="mxc://server/first", - sender_mxid="@homer:s", - ) - assert out.status == "captured" - body = mirror.captures[0]["body_text"] - assert "First memo about the boiler." in body - assert "Second memo about Bart's prescription." in body - assert "Third memo with shopping list." in body - # Paragraph breaks between memos so the LLM and the human reader - # both see them as distinct thoughts. - assert "boiler.\n\nSecond" in body - - @pytest.mark.asyncio - async def test_empty_transcripts_returns_empty_status(self): - """A `(` ... `)` with no voice memos (or all silent ones) drops - out as empty -- the orchestrator silently no-ops, matching the - existing scan_cancelled semantics for empty PDF batches.""" - pipe = _pipeline(mirror=FakeMirror()) - out = await pipe.capture_voice_batch( - transcripts=[], primary_mxc=None, sender_mxid="@homer:s", - ) - assert out.status == "empty" - - @pytest.mark.asyncio - async def test_blank_transcripts_filtered(self): - """Empty / whitespace-only transcripts are dropped silently so - one silent memo in the middle of a batch doesn't add a hole - of blank lines to the combined body.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["real one", " ", "", "another real one"], - primary_mxc="mxc://server/first", - sender_mxid="@homer:s", - ) - assert out.status == "captured" - body = mirror.captures[0]["body_text"] - assert body == "real one\n\nanother real one" - - @pytest.mark.asyncio - async def test_primary_mxc_threads_through_as_source_uri(self): - """The first memo's mxc becomes the vault note's link -- the - wiki entry points back to the start of the conversation.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["just one"], - primary_mxc="mxc://server/start-of-batch", - sender_mxid="@homer:s", - ) - assert mirror.captures[0]["source_uri"] == "mxc://server/start-of-batch" - assert out.display_link == "mxc://server/start-of-batch" - - @pytest.mark.asyncio - async def test_outcome_carries_transcript_for_reply_echo(self): - """The mime=audio/ogg on the synthetic SourceContent triggers - the same transcript-in-reply behaviour single-memo captures - get -- so the sender sees the combined text quoted back.""" - pipe = _pipeline(mirror=FakeMirror()) - out = await pipe.capture_voice_batch( - transcripts=["one", "two"], - primary_mxc="mxc://server/x", - sender_mxid="@homer:s", - ) - assert out.transcript == "one\n\ntwo" - - class TestTagList: """`_tag_list` mixes the classifier's free-form tags with a `Person: X` tag per attributed person, normalising types and whitespace.""" @@ -391,9 +283,9 @@ def test_image_returns_single_image_no_text(self): assert kind == "bookmark" def test_unsupported_mime_returns_none(self): - # _source_from_binary itself doesn't handle audio; capture_binary - # routes audio through _source_from_audio upstream. Video and - # archives still land here and fall through to None. + # Video and archives land here and fall through to None. (Audio + # never reaches the capture pipeline at all: the transport + # decodes it to text before any handler sees it.) pipe = self._pipe() source, images, _kind = pipe._source_from_binary( file_data=b"riff-wave", @@ -441,140 +333,16 @@ def test_txt_file_by_extension_when_mime_unknown(self): # ── Audio capture (voice memos) ──────────────────────────────────────── -class TestSourceFromAudio: - """The audio branch: transcribe via the injected Transcriber, return - a note-shaped SourceContent. When no transcriber is wired, soft-skip - with None so the orchestrator surfaces a friendly extract_failed.""" - - @pytest.mark.asyncio - async def test_transcribes_audio_into_note(self): - tr = FakeTranscriber(transcript="Reminder: book the boiler service") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice-2026-06-09.ogg", - source_uri="mxc://server/abc", - ) - assert len(tr.calls) == 1 - assert tr.calls[0]["audio"] == b"opus-bytes" - assert tr.calls[0]["filename"] == "voice-2026-06-09.ogg" - assert source is not None - assert source.text == "Reminder: book the boiler service" - assert source.mime == "audio/ogg" - assert source.title_hint == "voice-2026-06-09.ogg" - assert source.source_uri == "mxc://server/abc" - assert images == [] - assert kind == "note" - - @pytest.mark.asyncio - async def test_no_transcriber_soft_skips(self): - # WHISPER_URL unset at bootstrap -> the pipeline drops audio - # rather than raising. Bot replies with extract_failed. - pipe = _pipeline(mirror=FakeMirror(), transcriber=None) - source, images, _kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - @pytest.mark.asyncio - async def test_transcriber_error_soft_skips(self): - # Whisper is misconfigured or down -> log + drop, same shape as - # the no-transcriber case so the user-facing reply is uniform. - from stack.ai.client import LLMUnavailableError - tr = FakeTranscriber(error=LLMUnavailableError("whisper down")) - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, _kind = await pipe._source_from_audio( - file_data=b"opus-bytes", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - @pytest.mark.asyncio - async def test_empty_transcript_soft_skips(self): - # whisper.cpp sometimes returns "" for a silent clip; treat it - # like an unreadable scan -- no point classifying empty bytes. - tr = FakeTranscriber(transcript=" \n ") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - source, images, _kind = await pipe._source_from_audio( - file_data=b"silence", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - ) - assert source is None - assert images == [] - - -class TestCaptureBinaryAudioRouting: - """capture_binary peeks at the mime: audio/* -> _source_from_audio, - everything else -> _source_from_binary. The classify + mirror tail - runs identically afterwards, so a transcribed voice memo lands in - the mirror as a fully classified note.""" - - @pytest.mark.asyncio - async def test_audio_mime_routes_through_transcriber(self): - mirror = FakeMirror() - tr = FakeTranscriber(transcript="Pick up Bart's prescription Friday") - pipe = _pipeline(mirror=mirror, transcriber=tr) - out = await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice-2026-06-09.ogg", - source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - # The transcript reached the mirror via the classify tail. - assert len(tr.calls) == 1 - assert tr.calls[0]["audio"] == b"opus-data" - assert tr.calls[0]["filename"] == "voice-2026-06-09.ogg" - assert out.status == "captured" - assert mirror.captures, "expected the transcript to be mirrored as a note" - published = mirror.captures[0] - # The body the mirror writes IS the transcript -- voice memo as a - # searchable note in the sender's bucket. - assert "Pick up Bart's prescription Friday" in published["body_text"] - assert published["kind"] == "note" - # The outcome carries the transcript so the reply renderer can - # echo it back to the sender; PDF/URL/note captures get None. - assert out.transcript == "Pick up Bart's prescription Friday" - - @pytest.mark.asyncio - async def test_non_audio_capture_outcome_has_no_transcript(self): - """A markdown note capture must not surface a transcript field -- - only audio captures get one, so the reply layer can branch on - presence rather than mime-sniffing again.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror, transcriber=FakeTranscriber()) - out = await pipe.capture_binary( - file_data=b"# Hello\n\nbody", - mime="text/markdown", filename="note.md", - source_uri="mxc://server/md1", sender_mxid="@homer:s", - ) - assert out.status == "captured" - assert out.transcript is None - - @pytest.mark.asyncio - async def test_audio_without_transcriber_returns_extract_failed(self): - # Mirrors a bot booted before `stack up ai` -- the rest of the - # bot is alive, but voice messages can't be processed yet. - pipe = _pipeline(mirror=FakeMirror(), transcriber=None) - out = await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert out.status == "extract_failed" - # The reason qualifies the failure so the reply layer can render - # a voice-shaped message ('Couldn't transcribe...') rather than - # the URL-shaped message ('Couldn't read that link...'). - assert out.failure_reason == "transcription" +class TestCaptureBinaryFailures: + """capture_binary handles the shapes it can read and says which + shape failed, so the reply layer can word the error correctly.""" @pytest.mark.asyncio async def test_unreadable_binary_returns_binary_failure_reason(self): - """A non-audio mime that the binary path can't extract (e.g. - an unsupported video format) returns failure_reason='binary' so - the user gets the file-shaped error message, not the link one.""" - pipe = _pipeline(mirror=FakeMirror(), transcriber=FakeTranscriber()) + """A mime the binary path can't extract (e.g. an unsupported + video format) returns failure_reason='binary' so the user gets + the file-shaped error message, not the link one.""" + pipe = _pipeline(mirror=FakeMirror()) out = await pipe.capture_binary( file_data=b"random-bytes", mime="video/mp4", filename="clip.mp4", source_uri="mxc://server/v", @@ -583,56 +351,6 @@ async def test_unreadable_binary_returns_binary_failure_reason(self): assert out.status == "extract_failed" assert out.failure_reason == "binary" - @pytest.mark.asyncio - async def test_non_audio_mime_bypasses_transcriber(self): - # An image upload must not get sent to whisper -- the transcriber - # is for audio only, so non-audio mimes never touch it. - tr = FakeTranscriber() - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr) - await pipe.capture_binary( - file_data=b"\xff\xd8jpg", mime="image/jpeg", - filename="recipe.jpg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls == [] - - @pytest.mark.asyncio - async def test_llm_passed_to_transcriber_as_cleanup_with(self): - """The CapturePipeline.llm kwarg threads through to the Transcriber's - cleanup_with on every audio capture. This is the wiring contract - that powers transcript cleanup -- the Transcriber owns the prompt; - the pipeline only has to hand it the LLM.""" - # Sentinel object: not actually called by FakeTranscriber, but we - # assert it shows up in the recorded call. - sentinel_llm = object() - tr = FakeTranscriber(transcript="raw words") - pipe = _pipeline( - mirror=FakeMirror(), transcriber=tr, llm=sentinel_llm, - ) - await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls[0]["cleanup_with"] is sentinel_llm - - @pytest.mark.asyncio - async def test_no_llm_means_no_cleanup(self): - """When the LLM isn't wired (e.g. archivist boots before - OPENAI_URL is set), cleanup_with is None and the Transcriber - returns the raw transcript verbatim.""" - tr = FakeTranscriber(transcript="raw words") - pipe = _pipeline(mirror=FakeMirror(), transcriber=tr, llm=None) - await pipe.capture_binary( - file_data=b"opus-data", mime="audio/ogg", - filename="voice.ogg", source_uri="mxc://server/abc", - sender_mxid="@homer:s", - ) - assert tr.calls[0]["cleanup_with"] is None - - -# ── Topic-room tag-seed invariant ──────────────────────────────────────── - class TestTopicSeedMerge: """The static merge helper pins the tag-seed semantics: seed first, @@ -1034,21 +752,6 @@ async def test_capture_url_with_seed_keeps_classifier_additions(self): non_person = [t for t in tags if not t.startswith("Person: ")] assert non_person == ["van-life", "gear", "vans"] - @pytest.mark.asyncio - async def test_capture_voice_batch_with_seed_files_seed_tag(self): - """Voice batches go through `_publish` like every other - capture; the seed applies the same way.""" - mirror = FakeMirror() - pipe = _pipeline(mirror=mirror) - out = await pipe.capture_voice_batch( - transcripts=["first memo", "second memo"], - primary_mxc="mxc://server/abc", - sender_mxid="@homer:s", - seed_topics=["camping"], - ) - assert out.status == "captured" - assert "camping" in mirror.captures[0]["tags"] - @pytest.mark.asyncio async def test_seed_recorded_in_capture_tag_cache(self): """The capture-tag cache feeds the next capture's prompt as @@ -1275,3 +978,30 @@ async def test_empty_body_is_empty_outcome(self): assert out.status == "empty" assert mirror.emails == [] assert mirror.captures == [] + + +class TestCaptureOrigin: + """The footer tells the family where a capture came from. Speech and + a paste are not the same provenance and must not read the same.""" + + # Spoken words rarely contain a URL, so the footer falls through to + # the origin label — which is the whole point of the label. + NO_LINK = staticmethod(lambda: FakeExtractor(_source(source_uri=None))) + + @pytest.mark.asyncio + async def test_a_transcribed_note_reads_as_a_voice_message(self): + pipe = _pipeline(mirror=FakeMirror(), text_extractor=self.NO_LINK()) + out = await pipe.capture_text( + text="Sort out the loft before Christmas.", + sender_mxid="@homer:s", transcribed=True, + ) + assert out.display_link == "(voice message)" + + @pytest.mark.asyncio + async def test_a_typed_note_still_reads_as_pasted_text(self): + pipe = _pipeline(mirror=FakeMirror(), text_extractor=self.NO_LINK()) + out = await pipe.capture_text( + text="Sort out the loft before Christmas.", + sender_mxid="@homer:s", + ) + assert out.display_link == "(pasted text)" diff --git a/tests/stacklets/test_microbot_voice.py b/tests/stacklets/test_microbot_voice.py new file mode 100644 index 00000000..99fef183 --- /dev/null +++ b/tests/stacklets/test_microbot_voice.py @@ -0,0 +1,519 @@ +"""Voice messages arrive at handlers as text. + +Holding the mic button and typing are the same act: putting words in the +room. Only the encoding differs, so only the transport should know about +it. These tests pin that contract from the caller's side — a bot registers +an ordinary text handler and receives spoken words through it, with the +speaker still the sender, having passed every gate a typed message passes. + +The whisper call itself is stubbed; what is under test is the framework +wiring, not the speech model. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from nio.events.room_events import RoomMessage, RoomMessageAudio, RoomMessageText + +_RUNNER = Path(__file__).resolve().parent.parent.parent / "stacklets" / "core" / "bot-runner" +sys.path.insert(0, str(_RUNNER)) +# `lib/` hosts `stack.ai.client`, which the framework imports for the +# Transcriber type. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "lib")) + +import voice # noqa: E402 +from microbot import MicroBot # noqa: E402 + + +# ── Fixtures ───────────────────────────────────────────────────────────── + +VOICE_SOURCE = { + "type": "m.room.message", + "event_id": "$voice1", + "sender": "@homer:simpson", + "origin_server_ts": 1700000000000, + "content": { + "msgtype": "m.audio", + "body": "Voice message", + "url": "mxc://simpson/abc123", + "info": {"mimetype": "audio/ogg", "duration": 4200, "size": 9001}, + }, +} + + +def _voice_event(**overrides) -> RoomMessageAudio: + src = {**VOICE_SOURCE, **overrides} + if "content" in overrides: + src["content"] = {**VOICE_SOURCE["content"], **overrides["content"]} + return RoomMessage.parse_event(src) + + +def _text_event(body: str = "typed words") -> RoomMessageText: + return RoomMessage.parse_event({ + "type": "m.room.message", + "event_id": "$typed1", + "sender": "@marge:simpson", + "origin_server_ts": 1700000000001, + "content": {"msgtype": "m.text", "body": body}, + }) + + +class _Room: + room_id = "!kitchen:simpson" + + +class _FakeClient: + def __init__(self): + self.rooms = {"!kitchen:simpson": _Room()} + self.typing: list[bool] = [] + + async def room_typing(self, room_id, typing_state, timeout=None): + self.typing.append(typing_state) + + +class _StubTranscriber: + """Stand-in for `stack.ai.client.Transcriber` — the raw whisper half.""" + + def __init__(self, result: str = "buy milk on the way home", + error: Exception | None = None): + self.result = result + self.error = error + self.calls: list[dict] = [] + + async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", + model: str | None = None, cleanup_with=None) -> str: + self.calls.append({"audio": audio, "filename": filename, + "cleanup_with": cleanup_with}) + if self.error is not None: + raise self.error + return self.result + + +class _StubLLM: + """Stand-in for the polish pass: records what it was asked to fix.""" + + def __init__(self, result: str = "Buy milk on the way home."): + self.result = result + self.seen: list[str] = [] + self.kwargs: list[dict] = [] + + async def complete(self, role: str, prompt: str, **kw) -> str: + self.seen.append(prompt) + self.kwargs.append(kw) + return self.result + + +class _Bot(MicroBot): + name = "test-bot" + + def register_callbacks(self, client): + pass + + +def _bot(tmp_path, *, transcriber=None, name="test-bot", cleanup=None) -> _Bot: + bot = _Bot(homeserver="http://hs", user_id="@test-bot:simpson", + password="x", session_dir=str(tmp_path)) + bot.name = name + bot._client = _FakeClient() + bot._transcriber = transcriber + bot._transcript_cleanup = cleanup + + async def _download(mxc_url): + return b"OggS-fake-audio" + + bot._download_media = _download + return bot + + +@pytest.fixture(autouse=True) +def _isolated_store(tmp_path, monkeypatch): + """The store is process-global and durable by design; give each test + its own directory so they do not inherit each other's transcripts.""" + monkeypatch.setattr( + voice, "TRANSCRIPTS", voice.TranscriptStore(tmp_path / "transcripts"), + ) + + +def _collect(bot, event_type=RoomMessageText) -> list: + """Register a handler of `event_type` and return the list it fills.""" + seen: list = [] + + async def handler(room, event): + seen.append(event) + + bot.add_event_callback(handler, event_type) + return seen + + +# ── The contract ───────────────────────────────────────────────────────── + +class TestVoiceIsJustAMessage: + """A handler that was written for typed text handles speech too, + without knowing speech exists.""" + + @pytest.mark.asyncio + async def test_a_text_handler_receives_the_spoken_words(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk")) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert len(seen) == 1 + assert isinstance(seen[0], RoomMessageText) + assert seen[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_the_speaker_remains_the_sender(self, tmp_path): + """Replies, reactions and thread ownership all key off the sender + and event id. If the framework re-attributed the words to itself, + a bot would answer its own message and corrections would break.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].sender == "@homer:simpson" + assert seen[0].event_id == "$voice1" + assert seen[0].server_timestamp == 1700000000000 + + @pytest.mark.asyncio + async def test_a_memo_sent_into_a_thread_stays_in_it(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + relates = {"rel_type": "m.thread", "event_id": "$root9", + "is_falling_back": True, + "m.in_reply_to": {"event_id": "$root9"}} + + await bot._dispatch( + "!kitchen:simpson", + _voice_event(content={"m.relates_to": relates}), + ) + + assert seen[0].source["content"]["m.relates_to"] == relates + + @pytest.mark.asyncio + async def test_the_words_are_marked_as_transcribed(self, tmp_path): + """Whisper is lossy in a way a keyboard is not, so provenance has + to survive: what was said, and the audio it came from, so a bad + transcript is visible and correctable.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + content = seen[0].source["content"] + assert voice.was_transcribed(content) + block = content[voice.TRANSCRIPT_KEY] + assert block["url"] == "mxc://simpson/abc123" + assert block["source_msgtype"] == "m.audio" + assert block["duration"] == 4200 + # The audio payload itself must not masquerade as a text body. + assert "url" not in content + assert content["msgtype"] == "m.text" + + @pytest.mark.asyncio + async def test_audio_handlers_no_longer_see_it(self, tmp_path): + """The decode is total, not a fan-out: nothing downstream is left + holding raw audio, so there is no second place to transcribe.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + audio_seen = _collect(bot, RoomMessageAudio) + text_seen = _collect(bot, RoomMessageText) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert audio_seen == [] + assert len(text_seen) == 1 + + @pytest.mark.asyncio + async def test_typed_messages_pass_through_untouched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + typed = _text_event("typed words") + await bot._dispatch("!kitchen:simpson", typed) + + assert seen == [typed] + assert not voice.was_transcribed(typed.source["content"]) + + +class TestWhenTheWordsCannotBeRecovered: + """Undecodable speech is dispatched to nobody. It is not an error to + answer: the family sees their own voice message sitting in the room, + the same as a message in a language the stack cannot read.""" + + @pytest.mark.asyncio + async def test_without_whisper_nothing_is_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=None) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_transcription_failure_is_not_dispatched(self, tmp_path): + from stack.ai.client import LLMError + + bot = _bot(tmp_path, transcriber=_StubTranscriber( + error=LLMError("whisper unreachable"))) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_silence_is_not_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber(" ")) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + @pytest.mark.asyncio + async def test_a_failed_download_is_not_dispatched(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + seen = _collect(bot) + + async def _no_bytes(mxc_url): + return None + + bot._download_media = _no_bytes + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen == [] + + +class TestOneWhisperRunPerMessage: + """Every bot in a room drains the same timeline in the same process. + Without sharing, a five-minute memo would be sent to whisper once per + bot listening.""" + + @pytest.mark.asyncio + async def test_two_bots_transcribe_the_same_memo_once(self, tmp_path): + shared = _StubTranscriber("buy milk") + archivist = _bot(tmp_path / "a", transcriber=shared, name="archivist-bot") + stacker = _bot(tmp_path / "b", transcriber=shared, name="stacker-bot") + heard_a, heard_b = _collect(archivist), _collect(stacker) + + event = _voice_event() + await archivist._dispatch("!kitchen:simpson", event) + await stacker._dispatch("!kitchen:simpson", event) + + assert len(shared.calls) == 1 + # Both still get the words: sharing the work, not the delivery. + assert heard_a[0].body == "buy milk" + assert heard_b[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_a_stored_transcript_is_not_produced_again(self, tmp_path): + """The backfill and the drain are different processes over one + store. Whatever already cost GPU time must never cost it twice.""" + first = _StubTranscriber("buy milk") + bot = _bot(tmp_path, transcriber=first) + _collect(bot) + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert len(first.calls) == 1 + + # A fresh bot, as after a restart — same store on disk. + second = _StubTranscriber("should never run") + restarted = _bot(tmp_path / "restarted", transcriber=second) + seen = _collect(restarted) + await restarted._dispatch("!kitchen:simpson", _voice_event()) + + assert second.calls == [] + assert seen[0].body == "buy milk" + + @pytest.mark.asyncio + async def test_a_failure_is_not_remembered(self, tmp_path): + """A whisper outage must not poison the message forever — the + drain is at-least-once, so the retry has to be able to succeed.""" + from stack.ai.client import LLMError + + broken = _StubTranscriber(error=LLMError("whisper down")) + bot = _bot(tmp_path, transcriber=broken) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert seen == [] + + bot._transcriber = _StubTranscriber("buy milk") + await bot._dispatch("!kitchen:simpson", _voice_event()) + assert [e.body for e in seen] == ["buy milk"] + + +class TestTranscriberWiring: + """Built from the environment at construction, like every other + framework capability; absent whisper is a normal state, not a crash.""" + + def test_absent_whisper_leaves_the_bot_running(self, tmp_path, monkeypatch): + monkeypatch.delenv("WHISPER_URL", raising=False) + bot = _Bot(homeserver="http://hs", user_id="@t:simpson", + password="x", session_dir=str(tmp_path)) + assert bot._transcriber is None + + def test_whisper_url_builds_a_transcriber(self, tmp_path, monkeypatch): + monkeypatch.setenv("WHISPER_URL", "http://localhost:42062/v1") + bot = _Bot(homeserver="http://hs", user_id="@t:simpson", + password="x", session_dir=str(tmp_path)) + assert bot._transcriber is not None + + +class TestTranscribedSource: + """The pure rewrite, exercised directly: given a voice event's raw + source dict and the words, produce the text event it becomes.""" + + def test_keeps_identity_and_replaces_the_body(self): + out = voice.transcribed_source(VOICE_SOURCE, "buy milk") + assert out["event_id"] == VOICE_SOURCE["event_id"] + assert out["sender"] == VOICE_SOURCE["sender"] + assert out["content"]["body"] == "buy milk" + assert out["content"]["msgtype"] == "m.text" + + def test_does_not_mutate_the_original(self): + before = VOICE_SOURCE["content"]["msgtype"] + voice.transcribed_source(VOICE_SOURCE, "buy milk") + assert VOICE_SOURCE["content"]["msgtype"] == before + assert "url" in VOICE_SOURCE["content"] + + +class TestIsVoice: + def test_audio_with_a_payload(self): + assert voice.is_voice(_voice_event()) + + def test_plain_text(self): + assert not voice.is_voice(_text_event()) + + def test_audio_without_a_payload_is_not_decodable(self): + assert not voice.is_voice(_voice_event(content={"url": ""})) + + +class TestThePolishPass: + """whisper.cpp emits one unbroken lowercase run of words. The polish + pass puts the sentences back. It is kept deliberately word-preserving: + the memories room holds things people said to their children, and a + model "improving" those is not a transcript any more.""" + + @pytest.mark.asyncio + async def test_handlers_receive_the_polished_text(self, tmp_path): + bot = _bot( + tmp_path, + transcriber=_StubTranscriber("buy milk on the way home"), + cleanup=_StubLLM("Buy milk on the way home."), + ) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].body == "Buy milk on the way home." + + @pytest.mark.asyncio + async def test_the_raw_transcript_is_kept_alongside_the_polished_one( + self, tmp_path, + ): + """Polishing gets better with better models and costs almost + nothing; whisper does not and is not. Keeping the raw text means + years of recordings can be re-polished without the audio.""" + bot = _bot( + tmp_path, + transcriber=_StubTranscriber("buy milk on the way home"), + cleanup=_StubLLM("Buy milk on the way home."), + ) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + record = voice.TRANSCRIPTS.read("$voice1") + assert record["raw"] == "buy milk on the way home" + assert record["text"] == "Buy milk on the way home." + assert record["url"] == "mxc://simpson/abc123" + + @pytest.mark.asyncio + async def test_without_an_llm_the_raw_transcript_still_lands(self, tmp_path): + """A rough transcript beats no transcript. Polish is never a gate.""" + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk"), + cleanup=None) + seen = _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert seen[0].body == "buy milk" + assert voice.TRANSCRIPTS.read("$voice1")["raw"] == "buy milk" + + +class TestTranscriptStore: + """Durable because a backfill over the memories room is years of + recordings, and a separate process from the bot that reads them.""" + + def test_a_record_survives_a_new_store_over_the_same_directory(self, tmp_path): + store = voice.TranscriptStore(tmp_path / "t") + store.write("$evt1", {"raw": "hello", "text": "Hello."}) + + reopened = voice.TranscriptStore(tmp_path / "t") + assert reopened.read("$evt1")["text"] == "Hello." + + def test_an_unknown_event_reads_as_none(self, tmp_path): + assert voice.TranscriptStore(tmp_path / "t").read("$nope") is None + + def test_event_ids_with_awkward_characters_round_trip(self, tmp_path): + """Matrix event ids are base64 and carry `/` and `+`, which would + otherwise land as directory separators.""" + store = voice.TranscriptStore(tmp_path / "t") + awkward = "$a/b+c=d:simpson" + store.write(awkward, {"text": "fine"}) + assert store.read(awkward)["text"] == "fine" + + def test_an_unwritable_store_does_not_break_the_message(self, tmp_path): + """A store we cannot write costs us the transcript again later. + It must never cost the family the message now.""" + blocked = tmp_path / "afile" + blocked.write_text("not a directory") + store = voice.TranscriptStore(blocked / "t") + store.write("$evt1", {"text": "hi"}) + assert store.read("$evt1") is None + + +class TestTheWorkingIndicator: + """Whisper runs before any handler can say it is working, so the + framework raises the indicator itself. Whoever raises it owns putting + it down — a bot left "typing" at a room it is not going to answer + sits there for the full five-minute timeout.""" + + @pytest.mark.asyncio + async def test_typing_stops_when_no_handler_wants_the_message(self, tmp_path): + bot = _bot(tmp_path, transcriber=_StubTranscriber("buy milk")) + # A bot that only handles audio now matches nothing: the message + # reaches dispatch as text. + _collect(bot, RoomMessageAudio) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert bot._client.typing == [True, False] + + @pytest.mark.asyncio + async def test_typing_stops_when_the_words_cannot_be_recovered(self, tmp_path): + from stack.ai.client import LLMError + + bot = _bot(tmp_path, transcriber=_StubTranscriber( + error=LLMError("whisper down"))) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _voice_event()) + + assert bot._client.typing == [True, False] + + @pytest.mark.asyncio + async def test_a_typed_message_never_raises_it(self, tmp_path): + """Only the decode needs the indicator; ordinary text is fast and + the handler decides for itself. (The handler wrap still clears it + on the way out, as it does for every message.)""" + bot = _bot(tmp_path, transcriber=_StubTranscriber()) + _collect(bot) + + await bot._dispatch("!kitchen:simpson", _text_event()) + + assert True not in bot._client.typing diff --git a/tests/stacklets/test_scribe.py b/tests/stacklets/test_scribe.py deleted file mode 100644 index ac4007f4..00000000 --- a/tests/stacklets/test_scribe.py +++ /dev/null @@ -1,285 +0,0 @@ -"""ScribeBot voice handling. - -Scribe used to call nio's media download, `room_typing`, and a hand- -built `room_send` directly. After the transport consolidation it goes -through the framework: `_download_media` (authenticated endpoint), -`_set_typing`, and `_send` (formatted, threaded). These pin that the -handler drives the framework methods rather than the raw client. - -The transcription HTTP call is delegated to the shared `Transcriber` -capability on the AI client; these tests inject a stub Transcriber so -the bot's wiring is exercised without a whisper server. -""" - -from __future__ import annotations - -import sys -from pathlib import Path -from types import SimpleNamespace - -import pytest - -_REPO_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "core" / "bot-runner")) -sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "messages" / "bot")) -# `lib/` hosts `stack.ai.client`, which the bot now imports at module load. -sys.path.insert(0, str(_REPO_ROOT / "lib")) - -from scribe import ScribeBot # noqa: E402 -from stack.ai.client import LLMUnavailableError # noqa: E402 - - -class _FakeClient: - """Supports the legacy surface so pre-migration code runs to a clean - assertion failure rather than crashing.""" - - def __init__(self): - self.sends: list[tuple] = [] - - def add_event_callback(self, cb, event_type): - pass - - async def room_typing(self, room_id, typing_state, timeout=None): - pass - - async def room_send(self, room_id, message_type, content): - self.sends.append((room_id, message_type, content)) - - async def download(self, url): - # Not a DownloadResponse — legacy path bails here, so the spies - # below stay empty and the assertions fail cleanly (RED). - return SimpleNamespace(body=b"") - - -class _StubTranscriber: - """Stand-in for `stack.ai.client.Transcriber` — records calls and - returns a configured transcript (or raises a configured error).""" - - def __init__(self, result: str = "hello world", error: Exception | None = None): - self.result = result - self.error = error - self.calls: list[tuple[bytes, str]] = [] - - async def transcribe(self, audio: bytes, *, filename: str = "voice.ogg", - model: str | None = None, - cleanup_with=None) -> str: - self.calls.append({ - "audio": audio, "filename": filename, - "cleanup_with": cleanup_with, - }) - if self.error is not None: - raise self.error - return self.result - - -def _build(tmp_path, monkeypatch, *, transcriber: _StubTranscriber | None = None): - # Transcriber.from_env reads WHISPER_URL at construction; we don't - # care what URL the stub is "pointed at", but the constructor refuses - # an empty value, so pin a benign placeholder. - monkeypatch.setenv("WHISPER_URL", "http://test.local/v1") - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - bot._transcriber = transcriber or _StubTranscriber() - return bot - - -@pytest.mark.asyncio -async def test_voice_routes_through_framework(tmp_path, monkeypatch): - """A voice message is downloaded via `_download_media`, transcribed, - and posted via `_send` threaded to the voice event; typing toggles - via `_set_typing`.""" - bot = _build(tmp_path, monkeypatch) - - calls = {"download": [], "send": [], "typing": []} - - async def fake_download(mxc): - calls["download"].append(mxc) - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - calls["send"].append((room_id, text, reply_to)) - - async def fake_typing(room_id, on=True): - calls["typing"].append((room_id, on)) - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc123", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - assert calls["download"] == ["mxc://server/abc123"] - # The Transcriber got the downloaded bytes and the caption filename. - assert len(bot._transcriber.calls) == 1 - assert bot._transcriber.calls[0]["audio"] == b"audio-bytes" - assert bot._transcriber.calls[0]["filename"] == "voice.ogg" - # The bot built no LLM (no OPENAI_URL in the test env), so cleanup - # is skipped: cleanup_with comes through as None. - assert bot._transcriber.calls[0]["cleanup_with"] is None - assert len(calls["send"]) == 1 - room_id, text, reply_to = calls["send"][0] - assert room_id == "!r:server" - assert "hello world" in text - assert reply_to == "$v:server" - # Typing was driven via the framework helper (on at least once). - assert ("!r:server", True) in calls["typing"] - - -@pytest.mark.asyncio -async def test_failed_download_sends_nothing(tmp_path, monkeypatch): - """When media can't be downloaded, scribe bails without posting.""" - bot = _build(tmp_path, monkeypatch) - sent = [] - - async def fake_download(mxc): - return None - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - sent.append(text) - - async def fake_typing(room_id, on=True): - pass - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/missing", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - assert sent == [] - # The Transcriber must not be called when there's nothing to transcribe. - assert bot._transcriber.calls == [] - - -@pytest.mark.asyncio -async def test_llm_threaded_into_transcribe_when_configured(tmp_path, monkeypatch): - """When OPENAI_URL is set, scribe builds an LLM and passes it as - cleanup_with so whisper's raw output gets polished. The LLM itself - isn't called in this test -- we use a stub transcriber that just - records what kwarg it received.""" - monkeypatch.setenv("WHISPER_URL", "http://test.local/v1") - monkeypatch.setenv("OPENAI_URL", "http://test.local/v1") - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - assert bot._llm is not None, "scribe should construct an LLM when OPENAI_URL is set" - - stub = _StubTranscriber(result="Cleaned text.") - bot._transcriber = stub - - async def fake_download(mxc): - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - pass - - async def fake_typing(room_id, on=True): - pass - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - # The bot's own LLM instance came through as cleanup_with -- not a - # different one, not None. - assert stub.calls[0]["cleanup_with"] is bot._llm - - -@pytest.mark.asyncio -async def test_no_whisper_url_does_not_crash_construction(tmp_path, monkeypatch): - """WHISPER_URL missing -> the bot still constructs, sets transcriber=None, - and logs a warning. The family server should boot scribe even when AI - hasn't been installed yet.""" - monkeypatch.delenv("WHISPER_URL", raising=False) - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - assert bot._transcriber is None - - -@pytest.mark.asyncio -async def test_voice_no_op_when_transcriber_disabled(tmp_path, monkeypatch): - """Without a transcriber the bot silently ignores audio: no typing - indicator, no apology reply, no download. The warning at startup is - the one and only signal.""" - monkeypatch.delenv("WHISPER_URL", raising=False) - bot = ScribeBot( - homeserver="http://x", user_id="@scribe-bot:server", - password="x", session_dir=str(tmp_path), - ) - bot._client = _FakeClient() - assert bot._transcriber is None - - calls = {"download": [], "send": [], "typing": []} - - async def fake_download(mxc): - calls["download"].append(mxc) - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - calls["send"].append(text) - - async def fake_typing(room_id, on=True): - calls["typing"].append((room_id, on)) - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - assert calls == {"download": [], "send": [], "typing": []} - - -@pytest.mark.asyncio -async def test_transcriber_error_falls_back_to_apology(tmp_path, monkeypatch): - """If whisper is down (Transcriber raises LLMError) the bot tells the - sender it couldn't do it — silent failure leaves people wondering.""" - failing = _StubTranscriber(error=LLMUnavailableError("whisper offline")) - bot = _build(tmp_path, monkeypatch, transcriber=failing) - sent: list[str] = [] - - async def fake_download(mxc): - return b"audio-bytes" - - async def fake_send(room_id, text, reply_to=None, *, metadata=None): - sent.append(text) - - async def fake_typing(room_id, on=True): - pass - - bot._download_media = fake_download - bot._send = fake_send - bot._set_typing = fake_typing - - event = SimpleNamespace( - sender="@homer:server", url="mxc://server/abc", - event_id="$v:server", body="voice.ogg", - ) - await bot._on_voice(SimpleNamespace(room_id="!r:server"), event) - - assert len(sent) == 1 - assert "couldn't transcribe" in sent[0].lower() From a643e0db57579cc6e1e23f258f84f95e3c2ce37c Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 11 Sep 2026 13:05:27 +0200 Subject: [PATCH 4/4] feat(messages): let Scribe say goodbye instead of going quiet Deleting the bot declaration stops the runner launching it, but the Matrix account survives: still joined to whatever room someone invited it to, still listed as a member, answering nothing, forever. There is no deprovisioning path in the framework, so removing Scribe outright would strand exactly the people who had gone looking for it. So it ships one more release as a shell with one behaviour. It explains that voice messages are transcribed automatically now, that nothing is lost and there is nothing to set up, and then it leaves the room. It registers no message handlers and builds no transcriber; the transport does that work now. The sweep runs on every launch rather than once, so a room it could not leave is retried next boot instead of keeping a silent member. A fresh invite gets the same answer, so following an older guide does not leave one behind either. Delete this one release after it ships. The real gap it papers over is that famstack cannot retire a bot at all, which is worth fixing before the next one goes away. --- stacklets/messages/bot/bot.toml | 6 ++ stacklets/messages/bot/scribe.py | 101 +++++++++++++++++++++ tests/stacklets/test_scribe.py | 150 +++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 stacklets/messages/bot/bot.toml create mode 100644 stacklets/messages/bot/scribe.py create mode 100644 tests/stacklets/test_scribe.py diff --git a/stacklets/messages/bot/bot.toml b/stacklets/messages/bot/bot.toml new file mode 100644 index 00000000..dbbae6d7 --- /dev/null +++ b/stacklets/messages/bot/bot.toml @@ -0,0 +1,6 @@ +# Deprecated. Transcription moved into the framework; this bot exists only +# to tell rooms it was invited to that it is going away, and then leave. +# Remove this declaration and scribe.py one release after it ships. +id = "scribe-bot" +name = "Scribe" +description = "Retired — voice messages are transcribed automatically now" diff --git a/stacklets/messages/bot/scribe.py b/stacklets/messages/bot/scribe.py new file mode 100644 index 00000000..3dc022be --- /dev/null +++ b/stacklets/messages/bot/scribe.py @@ -0,0 +1,101 @@ +"""Scribe — retired, and saying so. + +Transcribing voice messages used to be this bot's whole job. It is the +transport's job now: a voice message is decoded before any handler sees +it, so every bot in every room gets the words without asking, and nothing +needs Scribe to be present. + +This shell exists for one release only, because the framework has no way +to deprovision a bot that goes away. Deleting the declaration stops the +runner from launching it, but the Matrix account survives — still joined +to whatever rooms it was invited to, listed as a member, answering +nothing, forever. Somebody would eventually go looking for why. + +So instead of vanishing it explains itself once per room and leaves. The +people this reaches are the only ones it can have affected: Scribe +declared no room of its own, so it was never in a room unless a person +went and invited it by hand. Those are exactly the people who would +notice it go quiet. + +Delete this file and its bot.toml one release after it ships. +""" + +import asyncio +import os + +from loguru import logger +from nio import AsyncClient + +from microbot import MicroBot + + +# Kept inline rather than in a message catalogue: this is two strings with +# a known expiry, and a catalogue would outlive the bot that uses it. +_GOODBYE = { + "en": ( + "**Voice messages are transcribed automatically now.**\n\n" + "famstack does it for every room, so you no longer need me here. " + "Keep sending voice messages exactly as you always have. They are " + "transcribed the moment they arrive, and the other bots read the " + "words rather than the recording.\n\n" + "Nothing is lost and there is nothing to set up. I am leaving this " + "room; you can remove my account whenever you like." + ), + "de": ( + "**Sprachnachrichten werden jetzt automatisch transkribiert.**\n\n" + "famstack übernimmt das für jeden Raum, ihr braucht mich hier also " + "nicht mehr. Schickt Sprachnachrichten weiter wie bisher. Sie " + "werden sofort transkribiert, und die anderen Bots lesen den Text " + "statt der Aufnahme.\n\n" + "Es geht nichts verloren und es ist nichts einzurichten. Ich " + "verlasse diesen Raum; mein Konto könnt ihr jederzeit löschen." + ), +} + + +class ScribeBot(MicroBot): + name = "scribe-bot" + + def register_callbacks(self, client: AsyncClient) -> None: + """Register nothing, and start the retirement sweep. + + No message handlers at all: this bot answers nothing. The sweep + runs on every launch rather than once, so a room it could not + leave (homeserver hiccup, lost network) is retried next boot + instead of keeping a silent member forever. + + Scheduled as a task because `register_callbacks` is sync and runs + inside `start()`'s event loop, after the initial sync has + populated `client.rooms`. + """ + asyncio.create_task(self.retire_everywhere()) + + async def on_room_joined(self, room_id: str) -> None: + """Someone followed an older guide and invited it. Same answer, + so an invite never leaves a silent member behind.""" + await self._retire_from(room_id) + + async def retire_everywhere(self) -> None: + """Say goodbye in every room this account is still in, and leave.""" + room_ids = list(self.client.rooms.keys()) + if room_ids: + logger.info( + "[{}] retiring from {} room(s)", self.name, len(room_ids), + ) + for room_id in room_ids: + await self._retire_from(room_id) + + async def _retire_from(self, room_id: str) -> None: + """Explain, then leave. A failure to leave is logged and dropped: + the goodbye still landed, and the next launch tries again.""" + lang = os.environ.get("LANGUAGE", "en") + await self._send( + room_id, _GOODBYE.get(lang, _GOODBYE["en"]), msgtype="m.notice", + ) + try: + await self.client.room_leave(room_id) + except Exception as e: + logger.warning( + "[{}] could not leave {}: {} — will retry on next launch", + self.name, room_id, e, + ) diff --git a/tests/stacklets/test_scribe.py b/tests/stacklets/test_scribe.py new file mode 100644 index 00000000..292e3e3d --- /dev/null +++ b/tests/stacklets/test_scribe.py @@ -0,0 +1,150 @@ +"""Scribe retires itself. + +Transcription moved into the transport, so Scribe has no job left. It +ships for one more release as a shell that explains itself and leaves, +because the framework has no way to deprovision a removed bot: delete the +declaration and the Matrix account simply survives, still joined to +whatever room someone invited it to, answering nothing forever. + +Scribe declared no room of its own, so the only installs affected are the +ones where a person went looking for it and invited it by hand. Those are +exactly the people who would notice it going quiet, which is why it says +goodbye rather than just stopping. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_ROOT / "stacklets" / "core" / "bot-runner")) +sys.path.insert(0, str(_ROOT / "stacklets" / "messages" / "bot")) +sys.path.insert(0, str(_ROOT / "lib")) + +from scribe import ScribeBot # noqa: E402 + + +class _Room: + def __init__(self, room_id): + self.room_id = room_id + + +class _FakeClient: + def __init__(self, *room_ids, leave_error: Exception | None = None): + self.rooms = {r: _Room(r) for r in room_ids} + self.sent: list[dict] = [] + self.left: list[str] = [] + self._leave_error = leave_error + + def add_event_callback(self, cb, event_type): + pass + + async def room_send(self, room_id, message_type, content, **kw): + self.sent.append({"room_id": room_id, "content": content}) + + async def room_leave(self, room_id): + if self._leave_error is not None: + raise self._leave_error + self.left.append(room_id) + + +def _bot(tmp_path, client) -> ScribeBot: + bot = ScribeBot(homeserver="http://hs", user_id="@scribe-bot:simpson", + password="x", session_dir=str(tmp_path)) + bot._client = client + return bot + + +class TestScribeRetires: + """The one behaviour it has left.""" + + @pytest.mark.asyncio + async def test_it_says_goodbye_and_leaves_every_room(self, tmp_path): + client = _FakeClient("!kitchen:simpson", "!notes:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert client.left == ["!kitchen:simpson", "!notes:simpson"] + assert [s["room_id"] for s in client.sent] == [ + "!kitchen:simpson", "!notes:simpson", + ] + + @pytest.mark.asyncio + async def test_the_goodbye_explains_itself(self, tmp_path): + """A member that vanishes without a word is a mystery to debug. + It has to say what replaced it and that nothing is lost.""" + client = _FakeClient("!kitchen:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + body = client.sent[0]["content"]["body"].lower() + assert "voice" in body + assert "automatic" in body or "automatically" in body + # A notice, not a chat message: this is the machine talking. + assert client.sent[0]["content"]["msgtype"] == "m.notice" + + @pytest.mark.asyncio + async def test_it_leaves_a_room_it_is_freshly_invited_to(self, tmp_path): + """Someone following an older guide invites it. Same answer, so + the invite does not leave a silent member behind.""" + client = _FakeClient("!new:simpson") + bot = _bot(tmp_path, client) + + await bot.on_room_joined("!new:simpson") + + assert client.left == ["!new:simpson"] + assert len(client.sent) == 1 + + @pytest.mark.asyncio + async def test_a_room_it_cannot_leave_does_not_stop_the_others( + self, tmp_path, + ): + """The sweep runs on every launch, so a failure is retried next + boot. It must not strand the rooms behind it in the meantime.""" + client = _FakeClient("!stuck:simpson", "!fine:simpson", + leave_error=RuntimeError("homeserver said no")) + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert len(client.sent) == 2 + + @pytest.mark.asyncio + async def test_it_speaks_the_household_language(self, tmp_path, monkeypatch): + monkeypatch.setenv("LANGUAGE", "de") + client = _FakeClient("!kueche:simpson") + bot = _bot(tmp_path, client) + + await bot.retire_everywhere() + + assert "Sprachnachrichten" in client.sent[0]["content"]["body"] + + +class TestScribeAnswersNothing: + """It transcribes nothing and replies to nothing. The framework does + the transcribing now, and a second transcriber in the same process is + the exact thing this release removed.""" + + @pytest.mark.asyncio + async def test_it_registers_no_message_handlers(self, tmp_path): + # Async because `register_callbacks` schedules the retirement + # sweep, and the framework always calls it from inside `start()`'s + # running loop. + client = _FakeClient() + bot = _bot(tmp_path, client) + bot.register_callbacks(client) + assert bot._handlers == [] + + def test_it_builds_no_transcriber_of_its_own(self, tmp_path, monkeypatch): + """MicroBot gives every bot one for the transport decode; Scribe + must not reach for it. Nothing here should ever call whisper.""" + monkeypatch.setenv("WHISPER_URL", "http://localhost:42062/v1") + bot = _bot(tmp_path, _FakeClient()) + assert not hasattr(bot, "_scribe_transcriber") + import scribe + assert "Transcriber" not in scribe.__dict__