diff --git a/stacklets/memory/bot/curator.py b/stacklets/memory/bot/curator.py index 423009c3..c76e5ba8 100644 --- a/stacklets/memory/bot/curator.py +++ b/stacklets/memory/bot/curator.py @@ -70,6 +70,7 @@ SyncResult, _parse_frontmatter, authenticated_remote, + record_failed_rebuild, report_progress, brain_remote_url, is_auth_failure, @@ -1064,6 +1065,7 @@ async def mirror_reconcile() -> bool: debounce.reset() else: debounce.retry_later(time.monotonic()) + record_failed_rebuild(state_dir, retry_in=debounce.window()) # Say the shape of the failure, not just that there was one. A # per-cycle line at the same interval reads as a heartbeat; a # widening gap with a count on it reads as an outage. diff --git a/stacklets/memory/cli/_curator.py b/stacklets/memory/cli/_curator.py index dbeed64b..5d6618ac 100644 --- a/stacklets/memory/cli/_curator.py +++ b/stacklets/memory/cli/_curator.py @@ -19,11 +19,13 @@ from __future__ import annotations import sys +import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from lib import ( # noqa: E402 curator_state_dir_for, + failed_rebuild_since, progress_since, request_mirror, request_nightly, @@ -118,8 +120,13 @@ def pages(vault: Vault, target: str, since: float) -> tuple[str | None, str]: """Wait for the touched pages, printing what the curator did since `since`.""" echo = Echo(vault.state_dir, since) sha = wait_for_rebuilt(vault.state_dir, vault.memory, target, - timeout=PAGES_WAIT_SECS, interval=POLL_INTERVAL, on_poll=echo) + timeout=PAGES_WAIT_SECS, interval=POLL_INTERVAL, on_poll=echo, + asked=since) echo() + if sha is None and (failure := failed_rebuild_since(vault.state_dir, since)): + minutes = max(1, round((failure["retry_at"] - time.time()) / 60)) + return None, ("regenerating the wiki pages failed; the curator tries again in " + f"{minutes} minutes. See the lines above, or ./stack logs memory") if sha is None: return None, (f"curator did not regenerate the wiki pages for {target[:10]} within " f"{int(PAGES_WAIT_SECS)}s - is [memory] wiki_auto_rebuild off, or the AI down?") diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index a4a02398..02a31159 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -713,6 +713,9 @@ def _preserve_and_reset(git, local_head: str, remote_head: str) -> SyncResult: MIRROR_SHA_NAME = "last-mirrored-sha" # The memory commit the wiki pages were last regenerated up to. REBUILT_SHA_NAME = "last-rebuilt-sha" +# When regenerating the touched pages last failed, and when the curator +# tries again. A waiting command reads it to stop instead of timing out. +REBUILD_FAILED_NAME = "last-rebuild-failed" # Asks the curator for its nightly sweep now, whatever the time. NIGHTLY_TRIGGER_NAME = "nightly-now" # When the last nightly sweep finished, and whether its rebuild succeeded. @@ -774,17 +777,23 @@ def wait_for_mirror(state_dir: Path, memory: Path, target: str, *, def wait_for_rebuilt(state_dir: Path, memory: Path, target: str, *, timeout: float, interval: float = MIRROR_POLL_INTERVAL, - on_poll: Callable[[], None] | None = None) -> Optional[str]: + on_poll: Callable[[], None] | None = None, + asked: float | None = None) -> Optional[str]: """Poll `last-rebuilt-sha` until the wiki pages are regenerated up to `target`. The curator records it after regenerating the pages a change touched, - and also when a change touched none, so it always moves on. + and also when a change touched none, so it always moves on. With + `asked`, a failure the curator records after that moment ends the + wait early: None, and `failed_rebuild_since` says when it retries. """ - return _wait_for_sha(Path(state_dir) / REBUILT_SHA_NAME, memory, target, timeout, interval, on_poll) + failed = (lambda: failed_rebuild_since(state_dir, asked) is not None) if asked else None + return _wait_for_sha(Path(state_dir) / REBUILT_SHA_NAME, memory, target, timeout, + interval, on_poll, stop=failed) def _wait_for_sha(sha_file: Path, memory: Path, target: str, timeout: float, - interval: float, on_poll: Callable[[], None] | None) -> Optional[str]: + interval: float, on_poll: Callable[[], None] | None, + stop: Callable[[], bool] | None = None) -> Optional[str]: deadline = time.monotonic() + timeout while True: if on_poll: @@ -792,11 +801,28 @@ def _wait_for_sha(sha_file: Path, memory: Path, target: str, timeout: float, recorded = sha_file.read_text(encoding="utf-8").strip() if sha_file.exists() else "" if mirrored_contains(memory, target, recorded): return recorded - if time.monotonic() >= deadline: + if (stop and stop()) or time.monotonic() >= deadline: return None time.sleep(interval) +def record_failed_rebuild(state_dir: Path, *, retry_in: float) -> None: + """Note that regenerating the touched pages failed, and the next attempt.""" + state_dir.mkdir(parents=True, exist_ok=True) + now = time.time() + (Path(state_dir) / REBUILD_FAILED_NAME).write_text( + json.dumps({"at": now, "retry_at": now + retry_in}), encoding="utf-8") + + +def failed_rebuild_since(state_dir: Path, since: float) -> Optional[dict]: + """The failure the curator recorded at or after `since`, or None.""" + try: + record = json.loads((Path(state_dir) / REBUILD_FAILED_NAME).read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return record if record.get("at", 0) >= since else None + + # ── The nightly sweep, on request ──────────────────────────────────────── # Tonight's sweep (diary, full source reconcile, every wiki page) can be # asked for now. The curator stays the one writer: this drops a trigger, diff --git a/tests/unit/stacklets/test_memory_cycles.py b/tests/unit/stacklets/test_memory_cycles.py index a257e8f4..314cd9b5 100644 --- a/tests/unit/stacklets/test_memory_cycles.py +++ b/tests/unit/stacklets/test_memory_cycles.py @@ -37,6 +37,7 @@ REBUILT_SHA_NAME, curator_state_dir_for, progress_since, + record_failed_rebuild, report_progress, request_nightly, vault_path_for, @@ -97,6 +98,44 @@ def test_pages_that_never_come_are_named_as_such(self, tmp_path, monkeypatch): assert "wiki pages" in result["error"] + def test_a_failed_rebuild_ends_the_wait_and_says_when_it_retries(self, tmp_path, monkeypatch): + # The usual cause is the AI refusing or being down. Waiting out the + # full timeout then says nothing a person can act on. + data_dir, head = _vault(tmp_path) + state = curator_state_dir_for(data_dir) + wiki = _command("wiki") + monkeypatch.setattr(sys.modules["_curator"], "PAGES_WAIT_SECS", 30) + + def curator(): + (state / MIRROR_SHA_NAME).write_text(head) + record_failed_rebuild(state, retry_in=360) + + timer = _curator(state, curator) + started = time.monotonic() + result = wiki.run(["update"], {}, {"data_dir": str(data_dir)}) + timer.join() + + assert time.monotonic() - started < 5 + assert "failed" in result["error"] and "6 minutes" in result["error"] + + def test_a_failure_from_before_the_request_does_not_end_the_wait(self, tmp_path): + data_dir, head = _vault(tmp_path) + state = curator_state_dir_for(data_dir) + state.mkdir(parents=True) + record_failed_rebuild(state, retry_in=360) + time.sleep(0.01) + + def curator(): + (state / MIRROR_SHA_NAME).write_text(head) + time.sleep(0.1) + (state / REBUILT_SHA_NAME).write_text(head) + + timer = _curator(state, curator) + result = _command("wiki").run(["update"], {}, {"data_dir": str(data_dir)}) + timer.join() + + assert result == {"mirrored": head, "rebuilt": head, "target": head} + def test_sync_stops_at_the_mirror(self, tmp_path): data_dir, head = _vault(tmp_path) state = curator_state_dir_for(data_dir)