fix(wiki): stop waiting when the curator fails to regenerate the pages

stacklets/memory/bot/curator.py

@@ -70,6 +70,7 @@
     SyncResult,
     _parse_frontmatter,
     authenticated_remote,
+    record_failed_rebuild,
     report_progress,
     brain_remote_url,
     is_auth_failure,
@@ -1064,6 +1065,7 @@ async def mirror_reconcile() -> bool:
             debounce.reset()
         else:
             debounce.retry_later(time.monotonic())
+            record_failed_rebuild(state_dir, retry_in=debounce.window())
             # Say the shape of the failure, not just that there was one. A
             # per-cycle line at the same interval reads as a heartbeat; a
             # widening gap with a count on it reads as an outage.

stacklets/memory/cli/_curator.py

@@ -19,11 +19,13 @@
 from __future__ import annotations
 
 import sys
+import time
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 from lib import (  # noqa: E402
     curator_state_dir_for,
+    failed_rebuild_since,
     progress_since,
     request_mirror,
     request_nightly,
@@ -118,8 +120,13 @@ def pages(vault: Vault, target: str, since: float) -> tuple[str | None, str]:
     """Wait for the touched pages, printing what the curator did since `since`."""
     echo = Echo(vault.state_dir, since)
     sha = wait_for_rebuilt(vault.state_dir, vault.memory, target,
-                           timeout=PAGES_WAIT_SECS, interval=POLL_INTERVAL, on_poll=echo)
+                           timeout=PAGES_WAIT_SECS, interval=POLL_INTERVAL, on_poll=echo,
+                           asked=since)
     echo()
+    if sha is None and (failure := failed_rebuild_since(vault.state_dir, since)):
+        minutes = max(1, round((failure["retry_at"] - time.time()) / 60))
+        return None, ("regenerating the wiki pages failed; the curator tries again in "
+                      f"{minutes} minutes. See the lines above, or ./stack logs memory")
     if sha is None:
         return None, (f"curator did not regenerate the wiki pages for {target[:10]} within "
                       f"{int(PAGES_WAIT_SECS)}s - is [memory] wiki_auto_rebuild off, or the AI down?")

stacklets/memory/lib.py

@@ -713,6 +713,9 @@ def _preserve_and_reset(git, local_head: str, remote_head: str) -> SyncResult:
 MIRROR_SHA_NAME = "last-mirrored-sha"
 # The memory commit the wiki pages were last regenerated up to.
 REBUILT_SHA_NAME = "last-rebuilt-sha"
+# When regenerating the touched pages last failed, and when the curator
+# tries again. A waiting command reads it to stop instead of timing out.
+REBUILD_FAILED_NAME = "last-rebuild-failed"
 # Asks the curator for its nightly sweep now, whatever the time.
 NIGHTLY_TRIGGER_NAME = "nightly-now"
 # When the last nightly sweep finished, and whether its rebuild succeeded.
@@ -774,29 +777,52 @@ def wait_for_mirror(state_dir: Path, memory: Path, target: str, *,
 
 def wait_for_rebuilt(state_dir: Path, memory: Path, target: str, *,
                      timeout: float, interval: float = MIRROR_POLL_INTERVAL,
-                     on_poll: Callable[[], None] | None = None) -> Optional[str]:
+                     on_poll: Callable[[], None] | None = None,
+                     asked: float | None = None) -> Optional[str]:
     """Poll `last-rebuilt-sha` until the wiki pages are regenerated up to `target`.
 
     The curator records it after regenerating the pages a change touched,
-    and also when a change touched none, so it always moves on.
+    and also when a change touched none, so it always moves on. With
+    `asked`, a failure the curator records after that moment ends the
+    wait early: None, and `failed_rebuild_since` says when it retries.
     """
-    return _wait_for_sha(Path(state_dir) / REBUILT_SHA_NAME, memory, target, timeout, interval, on_poll)
+    failed = (lambda: failed_rebuild_since(state_dir, asked) is not None) if asked else None
+    return _wait_for_sha(Path(state_dir) / REBUILT_SHA_NAME, memory, target, timeout,
+                         interval, on_poll, stop=failed)
 
 
 def _wait_for_sha(sha_file: Path, memory: Path, target: str, timeout: float,
-                  interval: float, on_poll: Callable[[], None] | None) -> Optional[str]:
+                  interval: float, on_poll: Callable[[], None] | None,
+                  stop: Callable[[], bool] | None = None) -> Optional[str]:
     deadline = time.monotonic() + timeout
     while True:
         if on_poll:
             on_poll()
         recorded = sha_file.read_text(encoding="utf-8").strip() if sha_file.exists() else ""
         if mirrored_contains(memory, target, recorded):
             return recorded
-        if time.monotonic() >= deadline:
+        if (stop and stop()) or time.monotonic() >= deadline:
             return None
         time.sleep(interval)
 
 
+def record_failed_rebuild(state_dir: Path, *, retry_in: float) -> None:
+    """Note that regenerating the touched pages failed, and the next attempt."""
+    state_dir.mkdir(parents=True, exist_ok=True)
+    now = time.time()
+    (Path(state_dir) / REBUILD_FAILED_NAME).write_text(
+        json.dumps({"at": now, "retry_at": now + retry_in}), encoding="utf-8")
+
+
+def failed_rebuild_since(state_dir: Path, since: float) -> Optional[dict]:
+    """The failure the curator recorded at or after `since`, or None."""
+    try:
+        record = json.loads((Path(state_dir) / REBUILD_FAILED_NAME).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    return record if record.get("at", 0) >= since else None
+
+
 # ── The nightly sweep, on request ────────────────────────────────────────
 # Tonight's sweep (diary, full source reconcile, every wiki page) can be
 # asked for now. The curator stays the one writer: this drops a trigger,

tests/unit/stacklets/test_memory_cycles.py

@@ -37,6 +37,7 @@
     REBUILT_SHA_NAME,
     curator_state_dir_for,
     progress_since,
+    record_failed_rebuild,
     report_progress,
     request_nightly,
     vault_path_for,
@@ -97,6 +98,44 @@ def test_pages_that_never_come_are_named_as_such(self, tmp_path, monkeypatch):
 
         assert "wiki pages" in result["error"]
 
+    def test_a_failed_rebuild_ends_the_wait_and_says_when_it_retries(self, tmp_path, monkeypatch):
+        # The usual cause is the AI refusing or being down. Waiting out the
+        # full timeout then says nothing a person can act on.
+        data_dir, head = _vault(tmp_path)
+        state = curator_state_dir_for(data_dir)
+        wiki = _command("wiki")
+        monkeypatch.setattr(sys.modules["_curator"], "PAGES_WAIT_SECS", 30)
+
+        def curator():
+            (state / MIRROR_SHA_NAME).write_text(head)
+            record_failed_rebuild(state, retry_in=360)
+
+        timer = _curator(state, curator)
+        started = time.monotonic()
+        result = wiki.run(["update"], {}, {"data_dir": str(data_dir)})
+        timer.join()
+
+        assert time.monotonic() - started < 5
+        assert "failed" in result["error"] and "6 minutes" in result["error"]
+
+    def test_a_failure_from_before_the_request_does_not_end_the_wait(self, tmp_path):
+        data_dir, head = _vault(tmp_path)
+        state = curator_state_dir_for(data_dir)
+        state.mkdir(parents=True)
+        record_failed_rebuild(state, retry_in=360)
+        time.sleep(0.01)
+
+        def curator():
+            (state / MIRROR_SHA_NAME).write_text(head)
+            time.sleep(0.1)
+            (state / REBUILT_SHA_NAME).write_text(head)
+
+        timer = _curator(state, curator)
+        result = _command("wiki").run(["update"], {}, {"data_dir": str(data_dir)})
+        timer.join()
+
+        assert result == {"mirrored": head, "rebuilt": head, "target": head}
+
     def test_sync_stops_at_the_mirror(self, tmp_path):
         data_dir, head = _vault(tmp_path)
         state = curator_state_dir_for(data_dir)