From d4def1b65fe884ef6c318e80b794c55317d918ce Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Thu, 29 Oct 2026 18:51:24 -0400 Subject: [PATCH 1/2] Launcher: say when a newer ChainForge is out, and update it from the menu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Mac launcher never updates ChainForge by itself, so a shared lab machine doesn't change under the people using it. Instead: - Once a day, on starting, it checks PyPI. If a newer ChainForge is out, a notification says so, once per version, and the menu item reads "Update to ChainForge 0.3.7.8…". The menu's first line shows the version running. - "Update ChainForge…" checks right away. If there's a newer version, it asks, then stops ChainForge, runs `uv tool upgrade chainforge`, and starts it again, saying to close tabs opened before (the new server doesn't know their session). If the update fails, the old version starts again. build_app.sh now records uv's path in the app, which updating needs; apps built before this have no Update item until rebuilt. Checks and updates run off the main thread, with the menu updated from the app's timer. The README now installs ChainForge from PyPI (0.3.7.7 has --idle-shutdown), instead of from a checkout with `npm install --legacy-peer-deps`. Co-Authored-By: Claude Opus 5.5 --- packager/README.md | 47 +++++-- packager/build_app.sh | 5 +- packager/chainforge_menubar.py | 139 +++++++++++++++++++- packager/launcher_core.py | 123 +++++++++++++++++- tests/test_packager_launcher.py | 75 +++++++++++ tests/test_packager_menubar.py | 217 ++++++++++++++++++++++++++++++++ 6 files changed, 583 insertions(+), 23 deletions(-) create mode 100644 tests/test_packager_menubar.py diff --git a/packager/README.md b/packager/README.md index 699d7f6ea..81848cb97 100644 --- a/packager/README.md +++ b/packager/README.md @@ -6,7 +6,8 @@ only run ChainForge while someone is using it. - **Double-click** the app (or its Desktop shortcut): ChainForge starts on this Mac only (`127.0.0.1`) and opens in your default browser, in a normal tab. - A **ChainForge icon in the menu bar** shows while it runs, with *Open - ChainForge*, *Show Log* and *Quit ChainForge*. Quitting stops the server. + ChainForge*, *Update ChainForge*, *Show Log* and *Quit ChainForge*. Quitting + stops the server. - If you forget, it **stops by itself**: each open ChainForge tab sends a heartbeat every 5 minutes, and the server stops once none has arrived for 20 minutes (a notification says so). This also covers the launcher being @@ -27,17 +28,14 @@ with `--idle-shutdown MINUTES`, as this app does, stops by itself. ## Install 1. Install [uv](https://docs.astral.sh/uv/getting-started/installation/). -2. Install ChainForge as a uv tool. It needs a version with `--idle-shutdown` - (check with `chainforge serve --help`). Until that is released, install - this repository, after building its front end - (`cd chainforge/react-server && npm install --legacy-peer-deps && npm run build`): +2. Install ChainForge as a uv tool (version 0.3.7.7 or later): ```bash - uv tool install --force "chainforge[rag] @ /path/to/ChainForge" + uv tool install "chainforge[rag]" ``` - Once released: `uv tool install "chainforge[rag]"`. -3. Build the app, with a Desktop shortcut: +3. Build the app, with a Desktop shortcut, from a copy of this repository + (`git clone https://github.com/ianarawjo/ChainForge.git`): ```bash packager/build_app.sh --desktop-shortcut @@ -52,6 +50,25 @@ with `--idle-shutdown MINUTES`, as this app does, stops by itself. The app is built on your machine, so macOS doesn't block it. Copying the built app to another Mac would need signing; build it there instead. +## Updates + +The app never updates ChainForge by itself, so a shared machine doesn't change +under the people using it. Instead: + +- Once a day, when it starts, it checks PyPI for a newer ChainForge. If there + is one, a notification says so (once per version), and the menu item reads + *Update to ChainForge 0.3.7.8…*. The menu's first line shows the version + running. +- *Update ChainForge…* checks right away. If there's a newer version, it asks + first, then stops ChainForge (and anything running in it), runs + `uv tool upgrade chainforge`, and starts it again. Flows are kept. Tabs + opened before the update need closing: the updated server doesn't recognise + them. If the update fails, the version you had starts again, and the log + says why. + +This needs ChainForge installed with uv, as above. Apps built before updates +were added have no *Update* item: rebuild them with `build_app.sh`. + ## When something goes wrong - **Logs:** *Show Log* in the menu, or `~/Library/Logs/ChainForge Launcher/` @@ -59,8 +76,10 @@ app to another Mac would need signing; build it there instead. - **"Something is already using port 8000":** another ChainForge (say, one started in a terminal) is running. Stop it, or build with a different `--port` -- but note the browser keeps ChainForge's saved state per port. -- **Upgrading ChainForge:** `uv tool upgrade chainforge`, then rebuild the app - if the `chainforge` command moved. +- **Updating from a terminal instead:** `uv tool upgrade chainforge`, then + rebuild the app if the `chainforge` command moved. +- **"Can't update ChainForge from here":** ChainForge wasn't installed with + uv (e.g. with pip). Update it the way it was installed. ## Uninstall @@ -73,10 +92,12 @@ ChainForge keeps them. - `launcher_core.py` -- the parts that need no GUI: settings, a single-instance lock, starting `chainforge serve --idle-shutdown` in its own process group, - waiting for it, stopping the whole group, and telling an idle stop from a - crash. Tested in `tests/test_packager_launcher.py`. + waiting for it, stopping the whole group, telling an idle stop from a + crash, and checking for and installing updates. Tested in + `tests/test_packager_launcher.py`. - `chainforge_menubar.py` -- the menu-bar app, using - [rumps](https://github.com/jaredks/rumps), run by uv. + [rumps](https://github.com/jaredks/rumps), run by uv. Its update flow is + tested, with rumps stood in for, in `tests/test_packager_menubar.py`. - `build_app.sh` -- writes the app bundle: settings, icons, and a start script that runs the launcher with uv. - The idle shutdown itself is in ChainForge, used only with `--idle-shutdown`: diff --git a/packager/build_app.sh b/packager/build_app.sh index a8bfa8f46..cd57522fc 100755 --- a/packager/build_app.sh +++ b/packager/build_app.sh @@ -79,7 +79,8 @@ cat > "$RES/launcher.json" < None: @@ -56,14 +62,28 @@ def __init__(self): self.state = "starting" self.started_at = time.monotonic() self.status = rumps.MenuItem("Starting ChainForge…") - self.menu = [ + # Results of work done off the main thread (checking for and + # installing updates), for tick() to act on: rumps menus and + # alerts only work on the main thread + self.events = queue.Queue() + self.installed = None # this ChainForge's version, once known + self.available = None # a newer version on PyPI, once known + self.checking = False + self.updated_to = None # the version just installed, to say so once it's up + self.update_item = rumps.MenuItem("Update ChainForge…", callback=self.update_chainforge) + items = [ self.status, None, rumps.MenuItem("Open ChainForge", callback=self.open_chainforge), + ] + if config.uv: # an app built before updates were added has no uv + items.append(self.update_item) + items += [ rumps.MenuItem("Show Log", callback=self.show_log), None, rumps.MenuItem("Quit ChainForge", callback=self.quit_chainforge), ] + self.menu = items self.timer = rumps.Timer(self.tick, 1) def begin(self): @@ -78,9 +98,110 @@ def begin(self): self.process = core.start_server(config, LOG_PATH) atexit.register(self.stop_server) self.timer.start() + if config.uv: + self.check_for_update(chosen=False) + + # --- updates --------------------------------------------------- + + def check_for_update(self, chosen: bool): + """Looks up the installed and newest versions, off the main thread. + Unless someone chose to, at most once a day.""" + if self.checking: + return + record = core.UpdateCheck(UPDATE_CHECK_PATH) + if not chosen and not record.due(): + # Still worth knowing the version, for the menu + threading.Thread(target=self._find_installed, daemon=True).start() + return + self.checking = True + + def work(): + installed = core.installed_version(config.uv) + latest = core.latest_version() + if latest is not None: + record.checked() + self.events.put(("checked", chosen, installed, latest)) + + threading.Thread(target=work, daemon=True).start() + + def _find_installed(self): + self.events.put(("checked", False, core.installed_version(config.uv), None)) + + def on_checked(self, chosen, installed, latest): + self.checking = False + self.installed = installed or self.installed + if self.state == "running" and self.installed: + self.status.title = f"ChainForge {self.installed} is running" + if core.is_newer(latest, self.installed): + self.available = latest + self.update_item.title = f"Update to ChainForge {latest}…" + record = core.UpdateCheck(UPDATE_CHECK_PATH) + if not chosen and record.should_announce(latest): + record.announced(latest) + notify(f"ChainForge {latest} is available. " + "Choose Update in the ChainForge menu-bar menu to install it.") + if not chosen: + return + if latest is None: + rumps.alert(title="Couldn't check for updates", + message="PyPI, where ChainForge is published, didn't answer. " + "Check the internet connection and try again.") + elif self.installed is None: + rumps.alert(title="Can't update ChainForge from here", + message="This ChainForge wasn't installed with uv, so update it the " + "way it was installed. The newest version is " + latest + ".") + elif not core.is_newer(latest, self.installed): + rumps.alert(title="ChainForge is up to date", + message=f"You have the newest version, {self.installed}.") + else: + answer = rumps.alert( + title=f"Update to ChainForge {latest}?", + message=f"You have {self.installed}. Updating restarts ChainForge, " + "which stops anything running in it. Your flows are kept.", + ok="Update", cancel="Not Now") + if answer == 1: + self.install_update(latest) + + def update_chainforge(self, _item): + if self.state == "running": + self.check_for_update(chosen=True) + + def install_update(self, version): + """Stops ChainForge, updates it with uv off the main thread, and + starts it again (the old version, if the update failed).""" + self.state = "updating" + self.status.title = f"Updating to ChainForge {version}…" + self.stop_server() + + def work(): + self.events.put(("updated", version, core.upgrade(config.uv, LOG_PATH))) + + threading.Thread(target=work, daemon=True).start() + + def on_updated(self, version, ok): + if ok: + self.installed, self.available = version, None + self.update_item.title = "Update ChainForge…" + else: + rumps.alert(title="ChainForge wasn't updated", + message="The update didn't work, so ChainForge will start as it was. " + "See the log for details: " + str(LOG_PATH)) + self.status.title = "Starting ChainForge…" + self.state = "starting" + self.started_at = time.monotonic() + self.updated_to = version if ok else None + self.process = core.start_server(config, LOG_PATH) + + # --- the server -------------------------------------------------- def tick(self, _timer): - if self.state == "stopping": + while not self.events.empty(): + event, *args = self.events.get() + if event == "checked": + self.on_checked(*args) + elif event == "updated": + self.on_updated(*args) + if self.state in ("stopping", "updating"): return exited = self.process.poll() is not None if self.state == "starting": @@ -92,8 +213,14 @@ def tick(self, _timer): self.finish() elif core.server_responds(config.url, timeout=0.5): self.state = "running" - self.status.title = "ChainForge is running" + self.status.title = (f"ChainForge {self.installed} is running" + if self.installed else "ChainForge is running") open_in_browser(config.url) + if self.updated_to: + # The restarted server doesn't know tabs opened before it + notify(f"ChainForge is updated to {self.updated_to}. " + "Close any ChainForge tabs opened before the update.") + self.updated_to = None elif time.monotonic() - self.started_at > 180: rumps.alert( title="ChainForge is taking too long to start", diff --git a/packager/launcher_core.py b/packager/launcher_core.py index 07eb1c911..7e411dda6 100644 --- a/packager/launcher_core.py +++ b/packager/launcher_core.py @@ -7,6 +7,7 @@ import fcntl import json import os +import re import signal import socket import subprocess @@ -15,7 +16,7 @@ import urllib.request from dataclasses import dataclass from pathlib import Path -from typing import IO, Callable, List, Optional +from typing import IO, Callable, List, Optional, Tuple SUPPORT_DIR = Path.home() / "Library" / "Application Support" / "ChainForge Launcher" LOG_DIR = Path.home() / "Library" / "Logs" / "ChainForge Launcher" @@ -35,6 +36,9 @@ class LauncherConfig: port: int = 8000 idle_shutdown_minutes: float = 20 flows_dir: Optional[str] = None + # The uv that installed ChainForge, to check for and install updates. + # Apps built before updates were added don't have it: no update menu then. + uv: Optional[str] = None @property def url(self) -> str: @@ -45,7 +49,7 @@ def load(cls, path: Path) -> "LauncherConfig": data = json.loads(Path(path).read_text(encoding="utf-8")) if not data.get("chainforge"): raise ValueError(f"{path} does not say where the chainforge command is") - known = {"chainforge", "host", "port", "idle_shutdown_minutes", "flows_dir"} + known = {"chainforge", "host", "port", "idle_shutdown_minutes", "flows_dir", "uv"} config = cls(**{k: v for k, v in data.items() if k in known}) if config.idle_shutdown_minutes <= 0: raise ValueError("idle_shutdown_minutes must be positive") @@ -188,3 +192,118 @@ def why_it_stopped(log_path: Path) -> str: return "exited" last_start = tail.rfind("--- ") return "idle" if IDLE_SHUTDOWN_MARKER in tail[last_start:] else "exited" + + +# ------------------------------- updates ------------------------------- +# The launcher never updates ChainForge by itself: a lab machine shouldn't +# change under the people using it. It says when a newer version is out (at +# most once a day, and once per version), and updates when someone chooses +# Update ChainForge in the menu. + +PYPI_URL = "https://pypi.org/pypi/chainforge/json" +CHECK_EVERY_SECONDS = 24 * 60 * 60 + + +def installed_version(uv: str, run: Callable = subprocess.run) -> Optional[str]: + """The ChainForge version uv installed, e.g. "0.3.7.7"; None if uv didn't + install it (e.g. pip did), or uv can't say.""" + try: + out = run([uv, "tool", "list"], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + # e.g. "chainforge v0.3.7.7", then a line per command it installed + match = re.search(r"^chainforge v(\S+)", out.stdout, re.MULTILINE) + return match.group(1) if match else None + + +def latest_version(timeout: float = 5.0, fetch: Optional[Callable] = None) -> Optional[str]: + """The newest ChainForge on PyPI; None if PyPI can't be reached.""" + try: + if fetch is not None: + data = fetch(PYPI_URL, timeout) + else: + with urllib.request.urlopen(PYPI_URL, timeout=timeout) as resp: + data = json.load(resp) + version = data["info"]["version"] + return version if isinstance(version, str) and version else None + except (OSError, ValueError, KeyError, TypeError, urllib.error.URLError): + return None + + +def version_key(version: str) -> Tuple[int, ...]: + """A version's numbers, for comparing: "0.3.7.10" -> (0, 3, 7, 10).""" + key = [] + for part in version.split("."): + digits = re.match(r"\d+", part) + if not digits: + break + key.append(int(digits.group())) + return tuple(key) + + +def is_newer(latest: Optional[str], installed: Optional[str]) -> bool: + """Whether `latest` is a newer version than `installed`.""" + if not latest or not installed: + return False + return version_key(latest) > version_key(installed) + + +def upgrade(uv: str, log_path: Path, run: Callable = subprocess.run) -> bool: + """Updates ChainForge with uv (keeping its extras, e.g. [rag]), writing + what uv says to the log. Whether it worked.""" + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + command = [uv, "tool", "upgrade", "chainforge"] + with open(log_path, "a", encoding="utf-8") as log: + log.write(f"\n--- {time.strftime('%Y-%m-%d %H:%M:%S')} updating: {' '.join(command)}\n") + log.flush() + try: + result = run(command, stdout=log, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, timeout=15 * 60) + except (OSError, subprocess.SubprocessError) as err: + log.write(f"update failed: {err}\n") + return False + return result.returncode == 0 + + +class UpdateCheck: + """When PyPI was last checked, and which new version was last announced, + so the launcher checks at most daily and says each version once.""" + + def __init__(self, path: Path, every_seconds: float = CHECK_EVERY_SECONDS): + self.path = Path(path) + self.every = every_seconds + + def _read(self) -> dict: + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + def _write(self, data: dict) -> None: + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(data), encoding="utf-8") + except OSError: + pass + + def due(self, now: Optional[float] = None) -> bool: + now = time.time() if now is None else now + checked = self._read().get("checked_at") + return not isinstance(checked, (int, float)) or now - checked >= self.every + + def checked(self, now: Optional[float] = None) -> None: + data = self._read() + data["checked_at"] = time.time() if now is None else now + self._write(data) + + def should_announce(self, version: str) -> bool: + return self._read().get("announced") != version + + def announced(self, version: str) -> None: + data = self._read() + data["announced"] = version + self._write(data) diff --git a/tests/test_packager_launcher.py b/tests/test_packager_launcher.py index 5ce22ebcd..f0f114c74 100644 --- a/tests/test_packager_launcher.py +++ b/tests/test_packager_launcher.py @@ -221,3 +221,78 @@ def test_an_earlier_idle_stop_does_not_explain_a_later_crash(self, core, tmp_pat def test_a_missing_log_means_it_simply_exited(self, core, tmp_path): assert core.why_it_stopped(tmp_path / "nope.log") == "exited" + + +class TestUpdates: + """Checking for and installing a newer ChainForge (never by itself).""" + + def test_the_app_knows_its_uv(self, core, tmp_path): + path = tmp_path / "launcher.json" + path.write_text(json.dumps({"chainforge": "/x/chainforge", "uv": "/x/uv"}), encoding="utf-8") + assert core.LauncherConfig.load(path).uv == "/x/uv" + # Built before updates were added: no uv, so no update menu + path.write_text(json.dumps({"chainforge": "/x/chainforge"}), encoding="utf-8") + assert core.LauncherConfig.load(path).uv is None + + def test_the_installed_version_from_uv(self, core): + def run(listed, code=0): + return lambda *a, **k: subprocess.CompletedProcess(a, code, stdout=listed, stderr="") + listed = "openhands v1.16.0\n- openhands\nchainforge v0.3.7.7\n- chainforge\n" + assert core.installed_version("uv", run(listed)) == "0.3.7.7" + # Installed some other way (e.g. pip), or uv failed + assert core.installed_version("uv", run("openhands v1.16.0\n- openhands\n")) is None + assert core.installed_version("uv", run(listed, code=2)) is None + + def missing(*a, **k): + raise FileNotFoundError("uv") + assert core.installed_version("uv", missing) is None + + def test_the_newest_version_on_pypi(self, core): + assert core.latest_version(fetch=lambda url, t: {"info": {"version": "0.3.7.8"}}) == "0.3.7.8" + assert core.latest_version(fetch=lambda url, t: {"info": {}}) is None + + def offline(url, timeout): + raise OSError("no network") + assert core.latest_version(fetch=offline) is None + + def test_comparing_versions(self, core): + assert core.is_newer("0.3.7.10", "0.3.7.9") # by number, not as text + assert core.is_newer("0.3.8", "0.3.7.7") + assert not core.is_newer("0.3.7.7", "0.3.7.7") + assert not core.is_newer("0.3.7.6", "0.3.7.7") + assert not core.is_newer(None, "0.3.7.7") + assert not core.is_newer("0.3.7.8", None) + + def test_updating_with_uv_logs_what_it_says(self, core, tmp_path): + log = tmp_path / "logs" / "chainforge.log" + for code, worked in ((0, True), (1, False)): + uv = tmp_path / f"uv{code}" + uv.write_text(f"#!/bin/sh\necho \"uv $*\"\nexit {code}\n", encoding="utf-8") + uv.chmod(0o755) + assert core.upgrade(str(uv), log) is worked + text = log.read_text(encoding="utf-8") + assert text.count("uv tool upgrade chainforge") == 2 # what it said, each time + + def test_updating_with_a_missing_uv_fails_cleanly(self, core, tmp_path): + log = tmp_path / "chainforge.log" + assert core.upgrade(str(tmp_path / "no-such-uv"), log) is False + assert "update failed" in log.read_text(encoding="utf-8") + + def test_checks_at_most_daily_and_announces_each_version_once(self, core, tmp_path): + record = core.UpdateCheck(tmp_path / "update-check.json", every_seconds=100) + assert record.due(now=1000) + record.checked(now=1000) + assert not record.due(now=1050) + assert record.due(now=1100) + assert record.should_announce("0.3.7.8") + record.announced("0.3.7.8") + assert not record.should_announce("0.3.7.8") + assert record.should_announce("0.3.7.9") + assert not record.due(now=1050) # announcing kept the check time + + def test_a_damaged_record_just_means_checking_again(self, core, tmp_path): + path = tmp_path / "update-check.json" + path.write_text("not json", encoding="utf-8") + record = core.UpdateCheck(path) + assert record.due() + assert record.should_announce("0.3.7.8") diff --git a/tests/test_packager_menubar.py b/tests/test_packager_menubar.py new file mode 100644 index 000000000..6f1534ac9 --- /dev/null +++ b/tests/test_packager_menubar.py @@ -0,0 +1,217 @@ +"""Tests for the menu-bar launcher's update flow (packager/chainforge_menubar.py). + +rumps and AppKit are replaced with stand-ins that record alerts and menu +titles, and the server and uv with fakes, so the app's logic runs anywhere: +checking for a newer ChainForge, saying so, and updating when chosen. +""" + +import importlib.util +import sys +import time +import types +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="the launcher is for macOS (POSIX process groups)" +) + +LAUNCHER_DIR = Path(__file__).resolve().parent.parent / "packager" + + +class FakeRumps(types.ModuleType): + def __init__(self): + super().__init__("rumps") + self.apps, self.alerts, self.answer = [], [], 1 + fake = self + + class MenuItem: + def __init__(self, title, callback=None): + self.title, self.callback = title, callback + + class App: + def __init__(self, name, icon=None, title=None, quit_button=None): + self.menu = [] + fake.apps.append(self) + + def run(self): + pass + + class Timer: + def __init__(self, callback, interval): + pass + + def start(self): + pass + + def stop(self): + pass + + self.MenuItem, self.App, self.Timer = MenuItem, App, Timer + + def alert(self, title, message, ok=None, cancel=None): + self.alerts.append(title) + return self.answer + + def quit_application(self): + pass + + +class FakeProcess: + def __init__(self): + self.pid, self.stopped = 12345, False + + def poll(self): + return 0 if self.stopped else None + + +@pytest.fixture +def launcher(monkeypatch, tmp_path): + rumps = FakeRumps() + appkit = types.ModuleType("AppKit") + appkit.NSApplicationActivationPolicyAccessory = 1 + appkit.NSApplication = types.SimpleNamespace( + sharedApplication=lambda: types.SimpleNamespace(setActivationPolicy_=lambda p: None)) + monkeypatch.setitem(sys.modules, "rumps", rumps) + monkeypatch.setitem(sys.modules, "AppKit", appkit) + + spec = importlib.util.spec_from_file_location("chainforge_menubar", LAUNCHER_DIR / "chainforge_menubar.py") + menubar = importlib.util.module_from_spec(spec) + spec.loader.exec_module(menubar) + core = menubar.core + + state = types.SimpleNamespace( + rumps=rumps, notes=[], opened=[], processes=[], upgrades=[], + installed="0.3.7.7", latest="0.3.7.8", upgrade_works=True) + + def start_server(config, log_path): + state.processes.append(FakeProcess()) + return state.processes[-1] + + def stop_server(process, grace_seconds=10.0): + process.stopped = True + + def upgrade(uv, log_path): + state.upgrades.append(uv) + if state.upgrade_works: + state.installed = state.latest + return state.upgrade_works + + monkeypatch.setattr(core, "start_server", start_server) + monkeypatch.setattr(core, "stop_server", stop_server) + monkeypatch.setattr(core, "port_in_use", lambda host, port: False) + monkeypatch.setattr(core, "server_responds", lambda url, timeout=1.0: True) + monkeypatch.setattr(core, "installed_version", lambda uv: state.installed) + monkeypatch.setattr(core, "latest_version", lambda: state.latest) + monkeypatch.setattr(core, "upgrade", upgrade) + monkeypatch.setattr(menubar, "UPDATE_CHECK_PATH", tmp_path / "update-check.json") + monkeypatch.setattr(menubar, "LOG_PATH", tmp_path / "chainforge.log") + monkeypatch.setattr(menubar, "notify", state.notes.append) + monkeypatch.setattr(menubar, "open_in_browser", state.opened.append) + monkeypatch.setattr(menubar.atexit, "register", lambda f: None) + # Not this test process's SIGTERM handler + monkeypatch.setattr(menubar.signal, "signal", lambda *args: None) + + def launch(uv="/x/uv"): + config = core.LauncherConfig(chainforge="/x/chainforge", uv=uv) + lock = types.SimpleNamespace(release=lambda: None) + menubar.run_menu_bar(config, lock) + return rumps.apps[-1] + + state.launch = launch + return state + + +def settle(app, until, seconds=5.0): + """Runs the app's timer ticks until `until()` holds (work happens on threads).""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + app.tick(None) + if until(): + return + time.sleep(0.01) + raise AssertionError("timed out waiting for the launcher") + + +def titles(app): + return [item.title for item in app.menu if item is not None] + + +def test_says_once_when_a_newer_version_is_out(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and launcher.notes) + assert launcher.notes == [ + "ChainForge 0.3.7.8 is available. Choose Update in the ChainForge menu-bar menu to install it."] + assert "Update to ChainForge 0.3.7.8…" in titles(app) + assert app.status.title == "ChainForge 0.3.7.7 is running" + assert launcher.upgrades == [] # never updates by itself + + # Started again the same day: no second notice, but the menu still knows + app.quit_chainforge(None) + launcher.notes.clear() + again = launcher.launch() + settle(again, lambda: again.state == "running" and again.installed) + assert launcher.notes == [] + + +def test_updates_when_chosen_and_restarts(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + launcher.rumps.answer = 1 # "Update" + app.update_chainforge(None) + settle(app, lambda: launcher.upgrades and app.state == "running") + assert launcher.rumps.alerts == ["Update to ChainForge 0.3.7.8?"] + assert launcher.upgrades == ["/x/uv"] + first, second = launcher.processes + assert first.stopped and not second.stopped # stopped, updated, started again + assert app.status.title == "ChainForge 0.3.7.8 is running" + assert "Update ChainForge…" in titles(app) + assert launcher.notes[-1] == ( + "ChainForge is updated to 0.3.7.8. Close any ChainForge tabs opened before the update.") + + +def test_not_now_leaves_it_running(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + launcher.rumps.answer = 0 # "Not Now" + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.upgrades == [] and len(launcher.processes) == 1 + + +def test_a_failed_update_starts_the_old_version_again(launcher): + launcher.upgrade_works = False + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + app.update_chainforge(None) + settle(app, lambda: launcher.upgrades and app.state == "running") + assert launcher.rumps.alerts[-1] == "ChainForge wasn't updated" + assert app.status.title == "ChainForge 0.3.7.7 is running" + assert len(launcher.processes) == 2 + + +def test_up_to_date(launcher): + launcher.latest = "0.3.7.7" + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.installed) + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.rumps.alerts == ["ChainForge is up to date"] + assert launcher.notes == [] + + +def test_offline(launcher): + launcher.latest = None + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.installed) + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.rumps.alerts == ["Couldn't check for updates"] + + +def test_an_app_built_before_updates_has_no_update_item(launcher): + app = launcher.launch(uv=None) + settle(app, lambda: app.state == "running") + assert not any("Update" in t for t in titles(app)) + assert app.status.title == "ChainForge is running" From 0e9e7fba1cfe384518f06190032a1036750f558c Mon Sep 17 00:00:00 2001 From: Ian Arawjo Date: Thu, 29 Oct 2026 18:57:38 -0400 Subject: [PATCH 2/2] Tests: no paid API calls unless asked The LanceDB and FAISS tests in test_retrievers.py called OpenAI (and Cohere) with whatever key was in the environment: free and skipped on CI, but real requests on a developer's machine, and failures when offline. They only checked the results' shape, which test_vector_retrievers.py already checks with the fake embedder, and the embedders themselves are tested with fake clients in test_embedders.py. So they're removed, along with the OpenAI and Cohere cases of the both-backends test. - FAISS's metrics (l2, cosine, dot) were only exercised by the OpenAI test: now by test_vector_retrievers.py, with the fake embedder. - test_live_apis.py keeps one live check per provider (OpenAI and Cohere embeddings), marked live_api, run only with CHAINFORGE_LIVE_TESTS=1, not merely because a key is set. It replaces the unused needs_api_key marker. Co-Authored-By: Claude Opus 5.5 --- pytest.ini | 2 +- tests/README.md | 15 ++- tests/test_live_apis.py | 55 ++++++++ tests/test_retrievers.py | 228 +------------------------------- tests/test_vector_retrievers.py | 9 ++ 5 files changed, 84 insertions(+), 225 deletions(-) create mode 100644 tests/test_live_apis.py diff --git a/pytest.ini b/pytest.ini index 0a605660f..d93b75fd8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ pythonpath = . markers = slow: loads a real ML model or hits the network; excluded by `-m "not slow"` - needs_api_key: requires a provider API key in the environment + live_api: calls a real provider's API with your key; runs only with CHAINFORGE_LIVE_TESTS=1 needs_faiss: requires the optional faiss-cpu package diff --git a/tests/README.md b/tests/README.md index 7f4344ba5..33edd2e8e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,11 +30,20 @@ similar vectors, so don't write tests that assume otherwise. | Marker | Meaning | | --- | --- | | `slow` | Loads a real ML model or hits the network | -| `needs_api_key` | Needs a provider API key in the environment | +| `live_api` | Calls a real provider's API with your key (`test_live_apis.py`) | | `needs_faiss` | Needs the optional `faiss-cpu` package | -Tests needing an API key (`OPENAI_API_KEY`, `COHERE_API_KEY`) skip themselves -when it is absent, so a plain `pytest` stays green and free. +No test calls a paid API unless asked. The few live checks of real providers' +APIs (`test_live_apis.py`, marked `live_api`) run only with +`CHAINFORGE_LIVE_TESTS=1`, and each skips itself without its key +(`OPENAI_API_KEY`, `COHERE_API_KEY`). So a plain `pytest` stays free, even with +keys in your environment. Everything else about those providers is tested with +fakes: the embedders in `test_embedders.py`, the vector stores in +`test_vector_retrievers.py`. + +```bash +CHAINFORGE_LIVE_TESTS=1 pytest -m live_api +``` ## FAISS tests run separately diff --git a/tests/test_live_apis.py b/tests/test_live_apis.py new file mode 100644 index 000000000..0f0f00bbc --- /dev/null +++ b/tests/test_live_apis.py @@ -0,0 +1,55 @@ +"""Live checks against real providers' APIs, made with your own API keys. + +Everything ChainForge does with these providers' replies is tested without +them: the embedders with fake clients (test_embedders.py), and the vector +stores with a fake embedder (test_vector_retrievers.py). These only check that +each provider's real API still answers as ChainForge expects. Each call costs +a tiny amount and needs the network, so they run only when asked, not merely +because a key happens to be in your environment: + + CHAINFORGE_LIVE_TESTS=1 pytest -m live_api + +A provider's check also skips itself when its key isn't set. +""" + +import math +import os + +import pytest + +from chainforge.rag.embeddings import EmbeddingMethodRegistry + +pytestmark = [ + pytest.mark.live_api, + pytest.mark.skipif(os.environ.get("CHAINFORGE_LIVE_TESTS") != "1", + reason="calls a paid API: set CHAINFORGE_LIVE_TESTS=1 to run"), +] + +TEXTS = ["Python is a programming language.", "The sky is blue."] + + +def needs_key(name): + return pytest.mark.skipif(not os.environ.get(name), reason=f"{name} not set") + + +def check_vectors(vectors, dim): + assert len(vectors) == len(TEXTS) + assert all(len(v) == dim for v in vectors) + assert all(math.isfinite(x) for v in vectors for x in v) + assert vectors[0] != vectors[1] + + +@needs_key("OPENAI_API_KEY") +def test_openai_embeddings(): + embed = EmbeddingMethodRegistry.get_embedder("openai") + vectors = embed(TEXTS, model_name="text-embedding-3-small", + api_keys={"OpenAI": os.environ["OPENAI_API_KEY"]}) + check_vectors(vectors, 1536) + + +@needs_key("COHERE_API_KEY") +def test_cohere_embeddings(): + embed = EmbeddingMethodRegistry.get_embedder("cohere") + vectors = embed(TEXTS, model_name="embed-english-v3.0", + api_keys={"Cohere": os.environ["COHERE_API_KEY"]}) + check_vectors(vectors, 1024) diff --git a/tests/test_retrievers.py b/tests/test_retrievers.py index 871de2a26..5c4e0101b 100644 --- a/tests/test_retrievers.py +++ b/tests/test_retrievers.py @@ -231,106 +231,6 @@ def test_retrieve_tfidf(self, mock_get_handler, client): class TestEmbeddingRetrievalWithLanceDB: """Test embedding-based retrieval with LanceDB backend across all providers.""" - @pytest.mark.parametrize("similarity_metric", ["cosine", "euclidean", "dot_product"]) - def test_lancedb_openai_embeddings(self, temp_db_dir, sample_chunks, sample_queries, similarity_metric): - """Test LanceDB retrieval with OpenAI embeddings and different similarity metrics.""" - # Skip if OpenAI API key not available - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - # Get embedder - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - # Generate embeddings - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - # Test retrieval - handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - settings = { - "top_k": 3, - "metric": similarity_metric, - } - - results = handler( - sample_chunks, - chunk_embeddings, - sample_queries, - query_embeddings, - settings, - temp_db_dir - ) - - # Assertions - assert len(results) == len(sample_queries) - for result in results: - assert "query_object" in result - assert "retrieved_chunks" in result - assert len(result["retrieved_chunks"]) <= 3 - - # Check structure of retrieved chunks - for chunk in result["retrieved_chunks"]: - assert "text" in chunk - assert "similarity" in chunk - assert "id" in chunk - assert isinstance(chunk["similarity"], (float, int)) - assert 0 <= chunk["similarity"] <= 1.1 # Allow slight numerical imprecision - - @pytest.mark.parametrize("top_k", [1, 3, 5]) - def test_lancedb_openai_different_top_k(self, temp_db_dir, sample_chunks, sample_queries, top_k): - """Test LanceDB retrieval with different top_k values.""" - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - settings = {"top_k": top_k, "metric": "cosine"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - for result in results: - assert len(result["retrieved_chunks"]) <= top_k - # Check that results are ordered by similarity (descending) - similarities = [chunk["similarity"] for chunk in result["retrieved_chunks"]] - assert similarities == sorted(similarities, reverse=True) - - # Commented out Cohere tests for now due to API availability issues - # def test_lancedb_cohere_embeddings(self, temp_db_dir, sample_chunks, sample_queries): - # """Test LanceDB retrieval with Cohere embeddings.""" - # if not os.environ.get("COHERE_API_KEY"): - # pytest.skip("COHERE_API_KEY not set") - - # embedder = EmbeddingMethodRegistry.get_embedder("cohere") - # api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - - # chunk_texts = [c["text"] for c in sample_chunks] - # chunk_embeddings = embedder(chunk_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - # query_texts = [q["text"] for q in sample_queries] - # query_embeddings = embedder(query_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - # handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - # settings = {"top_k": 3, "metric": "cosine"} - - # results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - # assert len(results) == len(sample_queries) - # for result in results: - # assert len(result["retrieved_chunks"]) <= 3 - # assert all("similarity" in chunk for chunk in result["retrieved_chunks"]) - def test_lancedb_sentence_transformers(self, temp_db_dir, sample_chunks, sample_queries): """Test LanceDB retrieval with Sentence Transformers embeddings.""" embedder = EmbeddingMethodRegistry.get_embedder("sentence-transformers") @@ -423,100 +323,6 @@ def test_lancedb_relevance_ordering(self, temp_db_dir, sample_chunks): class TestEmbeddingRetrievalWithFAISS: """Test embedding-based retrieval with FAISS backend across all providers.""" - @pytest.mark.parametrize("similarity_metric", ["l2", "cosine", "dot"]) - def test_faiss_openai_embeddings(self, temp_db_dir, sample_chunks, sample_queries, similarity_metric): - """Test FAISS retrieval with OpenAI embeddings and different similarity metrics.""" - # Skip if FAISS not installed - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": 3, "metric": similarity_metric} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - assert len(results) == len(sample_queries) - for result in results: - assert "query_object" in result - assert "retrieved_chunks" in result - assert len(result["retrieved_chunks"]) <= 3 - - for chunk in result["retrieved_chunks"]: - assert "text" in chunk - assert "similarity" in chunk - assert "id" in chunk - assert isinstance(chunk["similarity"], (float, int)) - - @pytest.mark.parametrize("top_k", [1, 3, 5]) - def test_faiss_different_top_k(self, temp_db_dir, sample_chunks, sample_queries, top_k): - """Test FAISS retrieval with different top_k values.""" - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": top_k, "metric": "l2"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - for result in results: - assert len(result["retrieved_chunks"]) <= top_k - - def test_faiss_cohere_embeddings(self, temp_db_dir, sample_chunks, sample_queries): - """Test FAISS retrieval with Cohere embeddings.""" - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("COHERE_API_KEY"): - pytest.skip("COHERE_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("cohere") - api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": 3, "metric": "cosine"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - assert len(results) == len(sample_queries) - for result in results: - assert len(result["retrieved_chunks"]) <= 3 - def test_faiss_sentence_transformers(self, temp_db_dir, sample_chunks, sample_queries): """Test FAISS retrieval with Sentence Transformers embeddings.""" try: @@ -662,13 +468,10 @@ def test_backends_return_similar_results(self, temp_db_dir, sample_chunks, sampl # Both should return the same top result (most relevant chunk) assert lance_results[0]["retrieved_chunks"][0]["text"] == faiss_results[0]["retrieved_chunks"][0]["text"] - @pytest.mark.parametrize("embedding_provider", [ - "sentence-transformers", - pytest.param("openai", marks=pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")), - pytest.param("cohere", marks=pytest.mark.skipif(not os.environ.get("COHERE_API_KEY"), reason="COHERE_API_KEY not set")) - ]) - def test_all_providers_with_both_backends(self, temp_db_dir, sample_chunks, sample_queries, embedding_provider): - """Test that all embedding providers work with both backends.""" + def test_a_local_model_with_both_backends(self, temp_db_dir, sample_chunks, sample_queries): + """A real (local) embedding model's vectors work with both backends. Real + providers' APIs are checked, on request only, in test_live_apis.py.""" + embedding_provider = "sentence-transformers" try: import faiss except ImportError: @@ -676,28 +479,11 @@ def test_all_providers_with_both_backends(self, temp_db_dir, sample_chunks, samp embedder = EmbeddingMethodRegistry.get_embedder(embedding_provider) - # Configure model and API keys based on provider - if embedding_provider == "openai": - model_name = "text-embedding-3-small" - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - elif embedding_provider == "cohere": - model_name = "embed-english-v3.0" - api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - else: - model_name = "all-MiniLM-L6-v2" - api_keys = None - + model_name = "all-MiniLM-L6-v2" chunk_texts = [c["text"] for c in sample_chunks] - if api_keys: - chunk_embeddings = embedder(chunk_texts, model_name=model_name, api_keys=api_keys) - else: - chunk_embeddings = embedder(chunk_texts, model_name=model_name) - + chunk_embeddings = embedder(chunk_texts, model_name=model_name) query_texts = [q["text"] for q in sample_queries[:1]] - if api_keys: - query_embeddings = embedder(query_texts, model_name=model_name, api_keys=api_keys) - else: - query_embeddings = embedder(query_texts, model_name=model_name) + query_embeddings = embedder(query_texts, model_name=model_name) # Test LanceDB lancedb_dir = os.path.join(temp_db_dir, f"lancedb_{embedding_provider}") diff --git a/tests/test_vector_retrievers.py b/tests/test_vector_retrievers.py index e4c0a08dd..ecd19f6f9 100644 --- a/tests/test_vector_retrievers.py +++ b/tests/test_vector_retrievers.py @@ -221,6 +221,15 @@ def test_top_k_is_respected(self, embedded, temp_db_dir): embedded["query_embeddings"], temp_db_dir, top_k=1) assert all(len(r["retrieved_chunks"]) <= 1 for r in results) + @pytest.mark.parametrize("metric", ["l2", "cosine", "dot"]) + def test_every_supported_metric_ranks_by_similarity(self, embedded, temp_db_dir, metric): + results = run("faiss_vector_store", embedded["chunks"], + embedded["chunk_embeddings"], embedded["queries"], + embedded["query_embeddings"], temp_db_dir, top_k=3, metric=metric) + for result in results: + similarities = [h["similarity"] for h in result["retrieved_chunks"]] + assert similarities and similarities == sorted(similarities, reverse=True) + def test_store_persists_across_calls(self, embedded, temp_db_dir): run("faiss_vector_store", embedded["chunks"], embedded["chunk_embeddings"], embedded["queries"], embedded["query_embeddings"], temp_db_dir, top_k=3)