From ebe927f5ef42ae11649dd59ce5c95c87d836f596 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 13 Oct 2026 12:30:43 +0200 Subject: [PATCH 1/2] fix(wiki): keep rebuilding after a page the model does not answer A page whose model call timed out raised `LLMTimeoutError`, which the wiki did not catch: the run stopped there, and every page after it in the selection or in the nightly sweep stayed stale until the next run. Only an unreachable server was handled per page. Each page now handles every model error the same way: it says which page got no answer, leaves that page as it was, and the run goes on. --- stacklets/memory/bot/cli/wiki.py | 14 +-- .../stacklets/test_memory_wiki_slow_model.py | 94 +++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 tests/unit/stacklets/test_memory_wiki_slow_model.py diff --git a/stacklets/memory/bot/cli/wiki.py b/stacklets/memory/bot/cli/wiki.py index f73e7ba7..56c169ca 100644 --- a/stacklets/memory/bot/cli/wiki.py +++ b/stacklets/memory/bot/cli/wiki.py @@ -76,7 +76,7 @@ from stack.vault import correspondents_dir, slug, slugify_person # noqa: E402 from stack.people import Person, find_people, resolve # noqa: E402 -from stack.ai.client import LLM, LLMUnavailableError # noqa: E402 +from stack.ai.client import LLM, LLMError # noqa: E402 # The diary compiler owns the diary's path and its reader-facing words; # the home page only places the pointer it renders. Imported by bare @@ -455,8 +455,8 @@ async def _generate_home( try: raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) answer = home_page.read_answer(raw, ids=roster, recent=recent, today=today) - except LLMUnavailableError as e: - _err(f"LLM unavailable: {e}") + except LLMError as e: + _err(f"home: no answer from the model ({e}); the page stays as it was") return 1 except home_page.AnswerError as e: _err(f"home: the model's answer is unusable ({e}); the page stays as it was") @@ -558,8 +558,8 @@ async def _generate_member( try: raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) parts = person_page.read_answer(raw, lately=lately) - except LLMUnavailableError as e: - _err(f"LLM unavailable: {e}") + except LLMError as e: + _err(f"{slug}: no answer from the model ({e}); the page stays as it was") return 1 except person_page.AnswerError as e: _err(f"{slug}: the model's answer is unusable ({e}); the page stays as it was") @@ -644,8 +644,8 @@ async def _generate_topic( try: raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) answer = topic_page.read_answer(raw) - except LLMUnavailableError as e: - _err(f"LLM unavailable: {e}") + except LLMError as e: + _err(f"{page_dir}: no answer from the model ({e}); the page stays as it was") return 1 except topic_page.AnswerError as e: _err(f"{page_dir}: the model's answer is unusable ({e}); the page stays as it was") diff --git a/tests/unit/stacklets/test_memory_wiki_slow_model.py b/tests/unit/stacklets/test_memory_wiki_slow_model.py new file mode 100644 index 00000000..38dbdec7 --- /dev/null +++ b/tests/unit/stacklets/test_memory_wiki_slow_model.py @@ -0,0 +1,94 @@ +"""A slow or failing model costs one wiki page, never the whole rebuild. + +The wiki asks the model once per page, and the AI server is shared with +everything else the household runs. A call that waits past the client's +timeout must leave that page as +it was and let the rebuild go on to the next one: the nightly sweep +covers every page in one run, and a run that stops at the first slow +page leaves all the pages after it stale. + +The vault and the brain are real files in a temporary directory; only +the model is a stand-in, since it is the boundary under test. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_REPO_ROOT / "stacklets")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "memory" / "bot" / "cli")) + +import wiki # noqa: E402 +from stack.ai.client import LLMTimeoutError # noqa: E402 + +HOMER = "---\ntype: person\ntitle: Homer Simpson\naccount: homer\n---\n" +MARGE = "---\ntype: person\ntitle: Marge Simpson\naccount: marge\n---\n" + + +def _entry(person: str, title: str) -> dict: + return {"rel": f"{person}/notes/2026-10-01-{person}.md", "title": title, + "date": "2026-10-01", "persons": [person], "summary": title} + + +class _Model: + """Answers every page, except that it never answers in time for `stalls_on`.""" + + def __init__(self, stalls_on: str = "") -> None: + self.stalls_on = stalls_on + self.calls: list[dict] = [] + + async def complete(self, role, prompt, **kwargs) -> str: + self.calls.append(kwargs) + if self.stalls_on and f"This page is about: {self.stalls_on}." in prompt: + raise LLMTimeoutError("Qwen3.6-35B-A3B — model may still be loading, try again") + return json.dumps({"tagline": "Family.", "about": "Lives in Springfield.", "loves": []}) + + +@pytest.fixture +def simpsons(tmp_path, monkeypatch): + """Homer and Marge, each with one note of their own.""" + vault, brain = tmp_path / "vault", tmp_path / "brain" + for person in ("homer", "marge"): + (vault / person).mkdir(parents=True) + (brain / person).mkdir(parents=True) + people = vault / "family" / "people" + people.mkdir(parents=True) + (people / "homer.md").write_text(HOMER) + (people / "marge.md").write_text(MARGE) + monkeypatch.setenv("MEMORY_VAULT_DIR", str(vault)) + monkeypatch.setenv("BRAIN_REPO_DIR", str(brain)) + index = [_entry("homer", "Donut run"), _entry("marge", "Bowling night")] + monkeypatch.setattr(wiki, "_index_vault", lambda vault: index) + monkeypatch.setattr(wiki, "_brain_dir", lambda: brain) + return brain + + +class TestAPageTheModelDoesNotAnswerInTime: + def test_the_next_page_is_still_rebuilt(self, simpsons): + asyncio.run(wiki.run(_Model(stalls_on="Homer Simpson"), + ["--member", "homer", "--member", "marge"])) + + assert "Lives in Springfield." in (simpsons / "marge" / "about.md").read_text() + + def test_its_own_page_stays_as_it_was(self, simpsons): + before = "---\ntitle: Homer Simpson\ngenerated: true\n---\n\nHomer, as of last night.\n" + (simpsons / "homer" / "about.md").write_text(before) + + asyncio.run(wiki.run(_Model(stalls_on="Homer Simpson"), + ["--member", "homer", "--member", "marge"])) + + assert (simpsons / "homer" / "about.md").read_text() == before + + def test_the_rebuild_still_counts_as_done(self, simpsons): + """The curator only logs a failed run; the page waits for the next one.""" + rc = asyncio.run(wiki.run(_Model(stalls_on="Homer Simpson"), + ["--member", "homer", "--member", "marge"])) + + assert rc == 0 + From 0ee4b91d705ac47629f63abc16d203c416da03f1 Mon Sep 17 00:00:00 2001 From: Test Date: Tue, 13 Oct 2026 12:31:05 +0200 Subject: [PATCH 2/2] fix(wiki): cap the length of the model's answer for a page The wiki asked for each page without a token limit, so a model caught in a repetition loop kept writing until the client timeout of 15 minutes, and the SDK's one retry could double that, all while holding the AI server the rest of the household shares. Each page call now asks for at most 8,192 tokens, about four times the longest answer measured (the home page). A cut-off answer does not parse, so the page stays as it was and the run goes on. --- stacklets/memory/bot/cli/wiki.py | 17 ++++++++++++++--- .../stacklets/test_memory_wiki_slow_model.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/stacklets/memory/bot/cli/wiki.py b/stacklets/memory/bot/cli/wiki.py index 56c169ca..cfbbf4c8 100644 --- a/stacklets/memory/bot/cli/wiki.py +++ b/stacklets/memory/bot/cli/wiki.py @@ -113,6 +113,14 @@ # override exists for experiments and unusual models. _TEMPERATURE = float(os.environ.get("WIKI_TEMPERATURE", "0.0")) +# The longest answer a page needs, in tokens. The home page is the +# longest, measured at about 1,800 for a family of five; this leaves +# room for a family several times that. A model caught in a loop +# otherwise writes until the client timeout (15 minutes) and holds the +# AI server the whole time; capped, its answer is cut off, fails to +# parse, and the page stays as it was. +_ANSWER_TOKENS = 8192 + # Bracketed regenerate markers. Match the correspondent-page convention # verbatim so the same human contract holds across every generated page: # edits outside the brackets are preserved, inside is ours to rewrite. @@ -453,7 +461,8 @@ async def _generate_home( recent=recent, today=today, aliases={slug: list(p.aliases) for slug, p in people.items()}) try: - raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) + raw = await llm.complete("overview", prompt, json_mode=True, + temperature=_TEMPERATURE, max_tokens=_ANSWER_TOKENS) answer = home_page.read_answer(raw, ids=roster, recent=recent, today=today) except LLMError as e: _err(f"home: no answer from the model ({e}); the page stays as it was") @@ -556,7 +565,8 @@ async def _generate_member( lately=lately, lang=lang, previous=previous, new_evidence=new_evidence, ) try: - raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) + raw = await llm.complete("overview", prompt, json_mode=True, + temperature=_TEMPERATURE, max_tokens=_ANSWER_TOKENS) parts = person_page.read_answer(raw, lately=lately) except LLMError as e: _err(f"{slug}: no answer from the model ({e}); the page stays as it was") @@ -642,7 +652,8 @@ async def _generate_topic( sources, {n: e.get("summary") or "" for n, e in enumerate(entries + cross_refs, 1)}), ) try: - raw = await llm.complete("overview", prompt, json_mode=True, temperature=_TEMPERATURE) + raw = await llm.complete("overview", prompt, json_mode=True, + temperature=_TEMPERATURE, max_tokens=_ANSWER_TOKENS) answer = topic_page.read_answer(raw) except LLMError as e: _err(f"{page_dir}: no answer from the model ({e}); the page stays as it was") diff --git a/tests/unit/stacklets/test_memory_wiki_slow_model.py b/tests/unit/stacklets/test_memory_wiki_slow_model.py index 38dbdec7..827855b5 100644 --- a/tests/unit/stacklets/test_memory_wiki_slow_model.py +++ b/tests/unit/stacklets/test_memory_wiki_slow_model.py @@ -2,7 +2,7 @@ The wiki asks the model once per page, and the AI server is shared with everything else the household runs. A call that waits past the client's -timeout must leave that page as +timeout, or a model that will not stop writing, must leave that page as it was and let the rebuild go on to the next one: the nightly sweep covers every page in one run, and a run that stops at the first slow page leaves all the pages after it stale. @@ -92,3 +92,14 @@ def test_the_rebuild_still_counts_as_done(self, simpsons): assert rc == 0 + +class TestAModelThatWillNotStop: + def test_every_page_asks_for_an_answer_of_bounded_length(self, simpsons): + """Without a cap, a model caught in a loop writes until the client + timeout (15 minutes) and holds the shared AI server all that time.""" + model = _Model() + + asyncio.run(wiki.run(model, ["--member", "homer", "--member", "marge"])) + + assert model.calls + assert all(call.get("max_tokens") for call in model.calls)