diff --git a/packager/README.md b/packager/README.md index 699d7f6ea..81848cb97 100644 --- a/packager/README.md +++ b/packager/README.md @@ -6,7 +6,8 @@ only run ChainForge while someone is using it. - **Double-click** the app (or its Desktop shortcut): ChainForge starts on this Mac only (`127.0.0.1`) and opens in your default browser, in a normal tab. - A **ChainForge icon in the menu bar** shows while it runs, with *Open - ChainForge*, *Show Log* and *Quit ChainForge*. Quitting stops the server. + ChainForge*, *Update ChainForge*, *Show Log* and *Quit ChainForge*. Quitting + stops the server. - If you forget, it **stops by itself**: each open ChainForge tab sends a heartbeat every 5 minutes, and the server stops once none has arrived for 20 minutes (a notification says so). This also covers the launcher being @@ -27,17 +28,14 @@ with `--idle-shutdown MINUTES`, as this app does, stops by itself. ## Install 1. Install [uv](https://docs.astral.sh/uv/getting-started/installation/). -2. Install ChainForge as a uv tool. It needs a version with `--idle-shutdown` - (check with `chainforge serve --help`). Until that is released, install - this repository, after building its front end - (`cd chainforge/react-server && npm install --legacy-peer-deps && npm run build`): +2. Install ChainForge as a uv tool (version 0.3.7.7 or later): ```bash - uv tool install --force "chainforge[rag] @ /path/to/ChainForge" + uv tool install "chainforge[rag]" ``` - Once released: `uv tool install "chainforge[rag]"`. -3. Build the app, with a Desktop shortcut: +3. Build the app, with a Desktop shortcut, from a copy of this repository + (`git clone https://github.com/ianarawjo/ChainForge.git`): ```bash packager/build_app.sh --desktop-shortcut @@ -52,6 +50,25 @@ with `--idle-shutdown MINUTES`, as this app does, stops by itself. The app is built on your machine, so macOS doesn't block it. Copying the built app to another Mac would need signing; build it there instead. +## Updates + +The app never updates ChainForge by itself, so a shared machine doesn't change +under the people using it. Instead: + +- Once a day, when it starts, it checks PyPI for a newer ChainForge. If there + is one, a notification says so (once per version), and the menu item reads + *Update to ChainForge 0.3.7.8…*. The menu's first line shows the version + running. +- *Update ChainForge…* checks right away. If there's a newer version, it asks + first, then stops ChainForge (and anything running in it), runs + `uv tool upgrade chainforge`, and starts it again. Flows are kept. Tabs + opened before the update need closing: the updated server doesn't recognise + them. If the update fails, the version you had starts again, and the log + says why. + +This needs ChainForge installed with uv, as above. Apps built before updates +were added have no *Update* item: rebuild them with `build_app.sh`. + ## When something goes wrong - **Logs:** *Show Log* in the menu, or `~/Library/Logs/ChainForge Launcher/` @@ -59,8 +76,10 @@ app to another Mac would need signing; build it there instead. - **"Something is already using port 8000":** another ChainForge (say, one started in a terminal) is running. Stop it, or build with a different `--port` -- but note the browser keeps ChainForge's saved state per port. -- **Upgrading ChainForge:** `uv tool upgrade chainforge`, then rebuild the app - if the `chainforge` command moved. +- **Updating from a terminal instead:** `uv tool upgrade chainforge`, then + rebuild the app if the `chainforge` command moved. +- **"Can't update ChainForge from here":** ChainForge wasn't installed with + uv (e.g. with pip). Update it the way it was installed. ## Uninstall @@ -73,10 +92,12 @@ ChainForge keeps them. - `launcher_core.py` -- the parts that need no GUI: settings, a single-instance lock, starting `chainforge serve --idle-shutdown` in its own process group, - waiting for it, stopping the whole group, and telling an idle stop from a - crash. Tested in `tests/test_packager_launcher.py`. + waiting for it, stopping the whole group, telling an idle stop from a + crash, and checking for and installing updates. Tested in + `tests/test_packager_launcher.py`. - `chainforge_menubar.py` -- the menu-bar app, using - [rumps](https://github.com/jaredks/rumps), run by uv. + [rumps](https://github.com/jaredks/rumps), run by uv. Its update flow is + tested, with rumps stood in for, in `tests/test_packager_menubar.py`. - `build_app.sh` -- writes the app bundle: settings, icons, and a start script that runs the launcher with uv. - The idle shutdown itself is in ChainForge, used only with `--idle-shutdown`: diff --git a/packager/build_app.sh b/packager/build_app.sh index a8bfa8f46..cd57522fc 100755 --- a/packager/build_app.sh +++ b/packager/build_app.sh @@ -79,7 +79,8 @@ cat > "$RES/launcher.json" < None: @@ -56,14 +62,28 @@ def __init__(self): self.state = "starting" self.started_at = time.monotonic() self.status = rumps.MenuItem("Starting ChainForge…") - self.menu = [ + # Results of work done off the main thread (checking for and + # installing updates), for tick() to act on: rumps menus and + # alerts only work on the main thread + self.events = queue.Queue() + self.installed = None # this ChainForge's version, once known + self.available = None # a newer version on PyPI, once known + self.checking = False + self.updated_to = None # the version just installed, to say so once it's up + self.update_item = rumps.MenuItem("Update ChainForge…", callback=self.update_chainforge) + items = [ self.status, None, rumps.MenuItem("Open ChainForge", callback=self.open_chainforge), + ] + if config.uv: # an app built before updates were added has no uv + items.append(self.update_item) + items += [ rumps.MenuItem("Show Log", callback=self.show_log), None, rumps.MenuItem("Quit ChainForge", callback=self.quit_chainforge), ] + self.menu = items self.timer = rumps.Timer(self.tick, 1) def begin(self): @@ -78,9 +98,110 @@ def begin(self): self.process = core.start_server(config, LOG_PATH) atexit.register(self.stop_server) self.timer.start() + if config.uv: + self.check_for_update(chosen=False) + + # --- updates --------------------------------------------------- + + def check_for_update(self, chosen: bool): + """Looks up the installed and newest versions, off the main thread. + Unless someone chose to, at most once a day.""" + if self.checking: + return + record = core.UpdateCheck(UPDATE_CHECK_PATH) + if not chosen and not record.due(): + # Still worth knowing the version, for the menu + threading.Thread(target=self._find_installed, daemon=True).start() + return + self.checking = True + + def work(): + installed = core.installed_version(config.uv) + latest = core.latest_version() + if latest is not None: + record.checked() + self.events.put(("checked", chosen, installed, latest)) + + threading.Thread(target=work, daemon=True).start() + + def _find_installed(self): + self.events.put(("checked", False, core.installed_version(config.uv), None)) + + def on_checked(self, chosen, installed, latest): + self.checking = False + self.installed = installed or self.installed + if self.state == "running" and self.installed: + self.status.title = f"ChainForge {self.installed} is running" + if core.is_newer(latest, self.installed): + self.available = latest + self.update_item.title = f"Update to ChainForge {latest}…" + record = core.UpdateCheck(UPDATE_CHECK_PATH) + if not chosen and record.should_announce(latest): + record.announced(latest) + notify(f"ChainForge {latest} is available. " + "Choose Update in the ChainForge menu-bar menu to install it.") + if not chosen: + return + if latest is None: + rumps.alert(title="Couldn't check for updates", + message="PyPI, where ChainForge is published, didn't answer. " + "Check the internet connection and try again.") + elif self.installed is None: + rumps.alert(title="Can't update ChainForge from here", + message="This ChainForge wasn't installed with uv, so update it the " + "way it was installed. The newest version is " + latest + ".") + elif not core.is_newer(latest, self.installed): + rumps.alert(title="ChainForge is up to date", + message=f"You have the newest version, {self.installed}.") + else: + answer = rumps.alert( + title=f"Update to ChainForge {latest}?", + message=f"You have {self.installed}. Updating restarts ChainForge, " + "which stops anything running in it. Your flows are kept.", + ok="Update", cancel="Not Now") + if answer == 1: + self.install_update(latest) + + def update_chainforge(self, _item): + if self.state == "running": + self.check_for_update(chosen=True) + + def install_update(self, version): + """Stops ChainForge, updates it with uv off the main thread, and + starts it again (the old version, if the update failed).""" + self.state = "updating" + self.status.title = f"Updating to ChainForge {version}…" + self.stop_server() + + def work(): + self.events.put(("updated", version, core.upgrade(config.uv, LOG_PATH))) + + threading.Thread(target=work, daemon=True).start() + + def on_updated(self, version, ok): + if ok: + self.installed, self.available = version, None + self.update_item.title = "Update ChainForge…" + else: + rumps.alert(title="ChainForge wasn't updated", + message="The update didn't work, so ChainForge will start as it was. " + "See the log for details: " + str(LOG_PATH)) + self.status.title = "Starting ChainForge…" + self.state = "starting" + self.started_at = time.monotonic() + self.updated_to = version if ok else None + self.process = core.start_server(config, LOG_PATH) + + # --- the server -------------------------------------------------- def tick(self, _timer): - if self.state == "stopping": + while not self.events.empty(): + event, *args = self.events.get() + if event == "checked": + self.on_checked(*args) + elif event == "updated": + self.on_updated(*args) + if self.state in ("stopping", "updating"): return exited = self.process.poll() is not None if self.state == "starting": @@ -92,8 +213,14 @@ def tick(self, _timer): self.finish() elif core.server_responds(config.url, timeout=0.5): self.state = "running" - self.status.title = "ChainForge is running" + self.status.title = (f"ChainForge {self.installed} is running" + if self.installed else "ChainForge is running") open_in_browser(config.url) + if self.updated_to: + # The restarted server doesn't know tabs opened before it + notify(f"ChainForge is updated to {self.updated_to}. " + "Close any ChainForge tabs opened before the update.") + self.updated_to = None elif time.monotonic() - self.started_at > 180: rumps.alert( title="ChainForge is taking too long to start", diff --git a/packager/launcher_core.py b/packager/launcher_core.py index 07eb1c911..7e411dda6 100644 --- a/packager/launcher_core.py +++ b/packager/launcher_core.py @@ -7,6 +7,7 @@ import fcntl import json import os +import re import signal import socket import subprocess @@ -15,7 +16,7 @@ import urllib.request from dataclasses import dataclass from pathlib import Path -from typing import IO, Callable, List, Optional +from typing import IO, Callable, List, Optional, Tuple SUPPORT_DIR = Path.home() / "Library" / "Application Support" / "ChainForge Launcher" LOG_DIR = Path.home() / "Library" / "Logs" / "ChainForge Launcher" @@ -35,6 +36,9 @@ class LauncherConfig: port: int = 8000 idle_shutdown_minutes: float = 20 flows_dir: Optional[str] = None + # The uv that installed ChainForge, to check for and install updates. + # Apps built before updates were added don't have it: no update menu then. + uv: Optional[str] = None @property def url(self) -> str: @@ -45,7 +49,7 @@ def load(cls, path: Path) -> "LauncherConfig": data = json.loads(Path(path).read_text(encoding="utf-8")) if not data.get("chainforge"): raise ValueError(f"{path} does not say where the chainforge command is") - known = {"chainforge", "host", "port", "idle_shutdown_minutes", "flows_dir"} + known = {"chainforge", "host", "port", "idle_shutdown_minutes", "flows_dir", "uv"} config = cls(**{k: v for k, v in data.items() if k in known}) if config.idle_shutdown_minutes <= 0: raise ValueError("idle_shutdown_minutes must be positive") @@ -188,3 +192,118 @@ def why_it_stopped(log_path: Path) -> str: return "exited" last_start = tail.rfind("--- ") return "idle" if IDLE_SHUTDOWN_MARKER in tail[last_start:] else "exited" + + +# ------------------------------- updates ------------------------------- +# The launcher never updates ChainForge by itself: a lab machine shouldn't +# change under the people using it. It says when a newer version is out (at +# most once a day, and once per version), and updates when someone chooses +# Update ChainForge in the menu. + +PYPI_URL = "https://pypi.org/pypi/chainforge/json" +CHECK_EVERY_SECONDS = 24 * 60 * 60 + + +def installed_version(uv: str, run: Callable = subprocess.run) -> Optional[str]: + """The ChainForge version uv installed, e.g. "0.3.7.7"; None if uv didn't + install it (e.g. pip did), or uv can't say.""" + try: + out = run([uv, "tool", "list"], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + # e.g. "chainforge v0.3.7.7", then a line per command it installed + match = re.search(r"^chainforge v(\S+)", out.stdout, re.MULTILINE) + return match.group(1) if match else None + + +def latest_version(timeout: float = 5.0, fetch: Optional[Callable] = None) -> Optional[str]: + """The newest ChainForge on PyPI; None if PyPI can't be reached.""" + try: + if fetch is not None: + data = fetch(PYPI_URL, timeout) + else: + with urllib.request.urlopen(PYPI_URL, timeout=timeout) as resp: + data = json.load(resp) + version = data["info"]["version"] + return version if isinstance(version, str) and version else None + except (OSError, ValueError, KeyError, TypeError, urllib.error.URLError): + return None + + +def version_key(version: str) -> Tuple[int, ...]: + """A version's numbers, for comparing: "0.3.7.10" -> (0, 3, 7, 10).""" + key = [] + for part in version.split("."): + digits = re.match(r"\d+", part) + if not digits: + break + key.append(int(digits.group())) + return tuple(key) + + +def is_newer(latest: Optional[str], installed: Optional[str]) -> bool: + """Whether `latest` is a newer version than `installed`.""" + if not latest or not installed: + return False + return version_key(latest) > version_key(installed) + + +def upgrade(uv: str, log_path: Path, run: Callable = subprocess.run) -> bool: + """Updates ChainForge with uv (keeping its extras, e.g. [rag]), writing + what uv says to the log. Whether it worked.""" + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + command = [uv, "tool", "upgrade", "chainforge"] + with open(log_path, "a", encoding="utf-8") as log: + log.write(f"\n--- {time.strftime('%Y-%m-%d %H:%M:%S')} updating: {' '.join(command)}\n") + log.flush() + try: + result = run(command, stdout=log, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, timeout=15 * 60) + except (OSError, subprocess.SubprocessError) as err: + log.write(f"update failed: {err}\n") + return False + return result.returncode == 0 + + +class UpdateCheck: + """When PyPI was last checked, and which new version was last announced, + so the launcher checks at most daily and says each version once.""" + + def __init__(self, path: Path, every_seconds: float = CHECK_EVERY_SECONDS): + self.path = Path(path) + self.every = every_seconds + + def _read(self) -> dict: + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + def _write(self, data: dict) -> None: + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(json.dumps(data), encoding="utf-8") + except OSError: + pass + + def due(self, now: Optional[float] = None) -> bool: + now = time.time() if now is None else now + checked = self._read().get("checked_at") + return not isinstance(checked, (int, float)) or now - checked >= self.every + + def checked(self, now: Optional[float] = None) -> None: + data = self._read() + data["checked_at"] = time.time() if now is None else now + self._write(data) + + def should_announce(self, version: str) -> bool: + return self._read().get("announced") != version + + def announced(self, version: str) -> None: + data = self._read() + data["announced"] = version + self._write(data) diff --git a/pytest.ini b/pytest.ini index 0a605660f..d93b75fd8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,5 +3,5 @@ pythonpath = . markers = slow: loads a real ML model or hits the network; excluded by `-m "not slow"` - needs_api_key: requires a provider API key in the environment + live_api: calls a real provider's API with your key; runs only with CHAINFORGE_LIVE_TESTS=1 needs_faiss: requires the optional faiss-cpu package diff --git a/tests/README.md b/tests/README.md index 7f4344ba5..33edd2e8e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,11 +30,20 @@ similar vectors, so don't write tests that assume otherwise. | Marker | Meaning | | --- | --- | | `slow` | Loads a real ML model or hits the network | -| `needs_api_key` | Needs a provider API key in the environment | +| `live_api` | Calls a real provider's API with your key (`test_live_apis.py`) | | `needs_faiss` | Needs the optional `faiss-cpu` package | -Tests needing an API key (`OPENAI_API_KEY`, `COHERE_API_KEY`) skip themselves -when it is absent, so a plain `pytest` stays green and free. +No test calls a paid API unless asked. The few live checks of real providers' +APIs (`test_live_apis.py`, marked `live_api`) run only with +`CHAINFORGE_LIVE_TESTS=1`, and each skips itself without its key +(`OPENAI_API_KEY`, `COHERE_API_KEY`). So a plain `pytest` stays free, even with +keys in your environment. Everything else about those providers is tested with +fakes: the embedders in `test_embedders.py`, the vector stores in +`test_vector_retrievers.py`. + +```bash +CHAINFORGE_LIVE_TESTS=1 pytest -m live_api +``` ## FAISS tests run separately diff --git a/tests/test_live_apis.py b/tests/test_live_apis.py new file mode 100644 index 000000000..0f0f00bbc --- /dev/null +++ b/tests/test_live_apis.py @@ -0,0 +1,55 @@ +"""Live checks against real providers' APIs, made with your own API keys. + +Everything ChainForge does with these providers' replies is tested without +them: the embedders with fake clients (test_embedders.py), and the vector +stores with a fake embedder (test_vector_retrievers.py). These only check that +each provider's real API still answers as ChainForge expects. Each call costs +a tiny amount and needs the network, so they run only when asked, not merely +because a key happens to be in your environment: + + CHAINFORGE_LIVE_TESTS=1 pytest -m live_api + +A provider's check also skips itself when its key isn't set. +""" + +import math +import os + +import pytest + +from chainforge.rag.embeddings import EmbeddingMethodRegistry + +pytestmark = [ + pytest.mark.live_api, + pytest.mark.skipif(os.environ.get("CHAINFORGE_LIVE_TESTS") != "1", + reason="calls a paid API: set CHAINFORGE_LIVE_TESTS=1 to run"), +] + +TEXTS = ["Python is a programming language.", "The sky is blue."] + + +def needs_key(name): + return pytest.mark.skipif(not os.environ.get(name), reason=f"{name} not set") + + +def check_vectors(vectors, dim): + assert len(vectors) == len(TEXTS) + assert all(len(v) == dim for v in vectors) + assert all(math.isfinite(x) for v in vectors for x in v) + assert vectors[0] != vectors[1] + + +@needs_key("OPENAI_API_KEY") +def test_openai_embeddings(): + embed = EmbeddingMethodRegistry.get_embedder("openai") + vectors = embed(TEXTS, model_name="text-embedding-3-small", + api_keys={"OpenAI": os.environ["OPENAI_API_KEY"]}) + check_vectors(vectors, 1536) + + +@needs_key("COHERE_API_KEY") +def test_cohere_embeddings(): + embed = EmbeddingMethodRegistry.get_embedder("cohere") + vectors = embed(TEXTS, model_name="embed-english-v3.0", + api_keys={"Cohere": os.environ["COHERE_API_KEY"]}) + check_vectors(vectors, 1024) diff --git a/tests/test_packager_launcher.py b/tests/test_packager_launcher.py index 5ce22ebcd..f0f114c74 100644 --- a/tests/test_packager_launcher.py +++ b/tests/test_packager_launcher.py @@ -221,3 +221,78 @@ def test_an_earlier_idle_stop_does_not_explain_a_later_crash(self, core, tmp_pat def test_a_missing_log_means_it_simply_exited(self, core, tmp_path): assert core.why_it_stopped(tmp_path / "nope.log") == "exited" + + +class TestUpdates: + """Checking for and installing a newer ChainForge (never by itself).""" + + def test_the_app_knows_its_uv(self, core, tmp_path): + path = tmp_path / "launcher.json" + path.write_text(json.dumps({"chainforge": "/x/chainforge", "uv": "/x/uv"}), encoding="utf-8") + assert core.LauncherConfig.load(path).uv == "/x/uv" + # Built before updates were added: no uv, so no update menu + path.write_text(json.dumps({"chainforge": "/x/chainforge"}), encoding="utf-8") + assert core.LauncherConfig.load(path).uv is None + + def test_the_installed_version_from_uv(self, core): + def run(listed, code=0): + return lambda *a, **k: subprocess.CompletedProcess(a, code, stdout=listed, stderr="") + listed = "openhands v1.16.0\n- openhands\nchainforge v0.3.7.7\n- chainforge\n" + assert core.installed_version("uv", run(listed)) == "0.3.7.7" + # Installed some other way (e.g. pip), or uv failed + assert core.installed_version("uv", run("openhands v1.16.0\n- openhands\n")) is None + assert core.installed_version("uv", run(listed, code=2)) is None + + def missing(*a, **k): + raise FileNotFoundError("uv") + assert core.installed_version("uv", missing) is None + + def test_the_newest_version_on_pypi(self, core): + assert core.latest_version(fetch=lambda url, t: {"info": {"version": "0.3.7.8"}}) == "0.3.7.8" + assert core.latest_version(fetch=lambda url, t: {"info": {}}) is None + + def offline(url, timeout): + raise OSError("no network") + assert core.latest_version(fetch=offline) is None + + def test_comparing_versions(self, core): + assert core.is_newer("0.3.7.10", "0.3.7.9") # by number, not as text + assert core.is_newer("0.3.8", "0.3.7.7") + assert not core.is_newer("0.3.7.7", "0.3.7.7") + assert not core.is_newer("0.3.7.6", "0.3.7.7") + assert not core.is_newer(None, "0.3.7.7") + assert not core.is_newer("0.3.7.8", None) + + def test_updating_with_uv_logs_what_it_says(self, core, tmp_path): + log = tmp_path / "logs" / "chainforge.log" + for code, worked in ((0, True), (1, False)): + uv = tmp_path / f"uv{code}" + uv.write_text(f"#!/bin/sh\necho \"uv $*\"\nexit {code}\n", encoding="utf-8") + uv.chmod(0o755) + assert core.upgrade(str(uv), log) is worked + text = log.read_text(encoding="utf-8") + assert text.count("uv tool upgrade chainforge") == 2 # what it said, each time + + def test_updating_with_a_missing_uv_fails_cleanly(self, core, tmp_path): + log = tmp_path / "chainforge.log" + assert core.upgrade(str(tmp_path / "no-such-uv"), log) is False + assert "update failed" in log.read_text(encoding="utf-8") + + def test_checks_at_most_daily_and_announces_each_version_once(self, core, tmp_path): + record = core.UpdateCheck(tmp_path / "update-check.json", every_seconds=100) + assert record.due(now=1000) + record.checked(now=1000) + assert not record.due(now=1050) + assert record.due(now=1100) + assert record.should_announce("0.3.7.8") + record.announced("0.3.7.8") + assert not record.should_announce("0.3.7.8") + assert record.should_announce("0.3.7.9") + assert not record.due(now=1050) # announcing kept the check time + + def test_a_damaged_record_just_means_checking_again(self, core, tmp_path): + path = tmp_path / "update-check.json" + path.write_text("not json", encoding="utf-8") + record = core.UpdateCheck(path) + assert record.due() + assert record.should_announce("0.3.7.8") diff --git a/tests/test_packager_menubar.py b/tests/test_packager_menubar.py new file mode 100644 index 000000000..6f1534ac9 --- /dev/null +++ b/tests/test_packager_menubar.py @@ -0,0 +1,217 @@ +"""Tests for the menu-bar launcher's update flow (packager/chainforge_menubar.py). + +rumps and AppKit are replaced with stand-ins that record alerts and menu +titles, and the server and uv with fakes, so the app's logic runs anywhere: +checking for a newer ChainForge, saying so, and updating when chosen. +""" + +import importlib.util +import sys +import time +import types +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif( + sys.platform == "win32", reason="the launcher is for macOS (POSIX process groups)" +) + +LAUNCHER_DIR = Path(__file__).resolve().parent.parent / "packager" + + +class FakeRumps(types.ModuleType): + def __init__(self): + super().__init__("rumps") + self.apps, self.alerts, self.answer = [], [], 1 + fake = self + + class MenuItem: + def __init__(self, title, callback=None): + self.title, self.callback = title, callback + + class App: + def __init__(self, name, icon=None, title=None, quit_button=None): + self.menu = [] + fake.apps.append(self) + + def run(self): + pass + + class Timer: + def __init__(self, callback, interval): + pass + + def start(self): + pass + + def stop(self): + pass + + self.MenuItem, self.App, self.Timer = MenuItem, App, Timer + + def alert(self, title, message, ok=None, cancel=None): + self.alerts.append(title) + return self.answer + + def quit_application(self): + pass + + +class FakeProcess: + def __init__(self): + self.pid, self.stopped = 12345, False + + def poll(self): + return 0 if self.stopped else None + + +@pytest.fixture +def launcher(monkeypatch, tmp_path): + rumps = FakeRumps() + appkit = types.ModuleType("AppKit") + appkit.NSApplicationActivationPolicyAccessory = 1 + appkit.NSApplication = types.SimpleNamespace( + sharedApplication=lambda: types.SimpleNamespace(setActivationPolicy_=lambda p: None)) + monkeypatch.setitem(sys.modules, "rumps", rumps) + monkeypatch.setitem(sys.modules, "AppKit", appkit) + + spec = importlib.util.spec_from_file_location("chainforge_menubar", LAUNCHER_DIR / "chainforge_menubar.py") + menubar = importlib.util.module_from_spec(spec) + spec.loader.exec_module(menubar) + core = menubar.core + + state = types.SimpleNamespace( + rumps=rumps, notes=[], opened=[], processes=[], upgrades=[], + installed="0.3.7.7", latest="0.3.7.8", upgrade_works=True) + + def start_server(config, log_path): + state.processes.append(FakeProcess()) + return state.processes[-1] + + def stop_server(process, grace_seconds=10.0): + process.stopped = True + + def upgrade(uv, log_path): + state.upgrades.append(uv) + if state.upgrade_works: + state.installed = state.latest + return state.upgrade_works + + monkeypatch.setattr(core, "start_server", start_server) + monkeypatch.setattr(core, "stop_server", stop_server) + monkeypatch.setattr(core, "port_in_use", lambda host, port: False) + monkeypatch.setattr(core, "server_responds", lambda url, timeout=1.0: True) + monkeypatch.setattr(core, "installed_version", lambda uv: state.installed) + monkeypatch.setattr(core, "latest_version", lambda: state.latest) + monkeypatch.setattr(core, "upgrade", upgrade) + monkeypatch.setattr(menubar, "UPDATE_CHECK_PATH", tmp_path / "update-check.json") + monkeypatch.setattr(menubar, "LOG_PATH", tmp_path / "chainforge.log") + monkeypatch.setattr(menubar, "notify", state.notes.append) + monkeypatch.setattr(menubar, "open_in_browser", state.opened.append) + monkeypatch.setattr(menubar.atexit, "register", lambda f: None) + # Not this test process's SIGTERM handler + monkeypatch.setattr(menubar.signal, "signal", lambda *args: None) + + def launch(uv="/x/uv"): + config = core.LauncherConfig(chainforge="/x/chainforge", uv=uv) + lock = types.SimpleNamespace(release=lambda: None) + menubar.run_menu_bar(config, lock) + return rumps.apps[-1] + + state.launch = launch + return state + + +def settle(app, until, seconds=5.0): + """Runs the app's timer ticks until `until()` holds (work happens on threads).""" + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + app.tick(None) + if until(): + return + time.sleep(0.01) + raise AssertionError("timed out waiting for the launcher") + + +def titles(app): + return [item.title for item in app.menu if item is not None] + + +def test_says_once_when_a_newer_version_is_out(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and launcher.notes) + assert launcher.notes == [ + "ChainForge 0.3.7.8 is available. Choose Update in the ChainForge menu-bar menu to install it."] + assert "Update to ChainForge 0.3.7.8…" in titles(app) + assert app.status.title == "ChainForge 0.3.7.7 is running" + assert launcher.upgrades == [] # never updates by itself + + # Started again the same day: no second notice, but the menu still knows + app.quit_chainforge(None) + launcher.notes.clear() + again = launcher.launch() + settle(again, lambda: again.state == "running" and again.installed) + assert launcher.notes == [] + + +def test_updates_when_chosen_and_restarts(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + launcher.rumps.answer = 1 # "Update" + app.update_chainforge(None) + settle(app, lambda: launcher.upgrades and app.state == "running") + assert launcher.rumps.alerts == ["Update to ChainForge 0.3.7.8?"] + assert launcher.upgrades == ["/x/uv"] + first, second = launcher.processes + assert first.stopped and not second.stopped # stopped, updated, started again + assert app.status.title == "ChainForge 0.3.7.8 is running" + assert "Update ChainForge…" in titles(app) + assert launcher.notes[-1] == ( + "ChainForge is updated to 0.3.7.8. Close any ChainForge tabs opened before the update.") + + +def test_not_now_leaves_it_running(launcher): + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + launcher.rumps.answer = 0 # "Not Now" + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.upgrades == [] and len(launcher.processes) == 1 + + +def test_a_failed_update_starts_the_old_version_again(launcher): + launcher.upgrade_works = False + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.available) + app.update_chainforge(None) + settle(app, lambda: launcher.upgrades and app.state == "running") + assert launcher.rumps.alerts[-1] == "ChainForge wasn't updated" + assert app.status.title == "ChainForge 0.3.7.7 is running" + assert len(launcher.processes) == 2 + + +def test_up_to_date(launcher): + launcher.latest = "0.3.7.7" + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.installed) + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.rumps.alerts == ["ChainForge is up to date"] + assert launcher.notes == [] + + +def test_offline(launcher): + launcher.latest = None + app = launcher.launch() + settle(app, lambda: app.state == "running" and app.installed) + app.update_chainforge(None) + settle(app, lambda: launcher.rumps.alerts) + assert launcher.rumps.alerts == ["Couldn't check for updates"] + + +def test_an_app_built_before_updates_has_no_update_item(launcher): + app = launcher.launch(uv=None) + settle(app, lambda: app.state == "running") + assert not any("Update" in t for t in titles(app)) + assert app.status.title == "ChainForge is running" diff --git a/tests/test_retrievers.py b/tests/test_retrievers.py index 871de2a26..5c4e0101b 100644 --- a/tests/test_retrievers.py +++ b/tests/test_retrievers.py @@ -231,106 +231,6 @@ def test_retrieve_tfidf(self, mock_get_handler, client): class TestEmbeddingRetrievalWithLanceDB: """Test embedding-based retrieval with LanceDB backend across all providers.""" - @pytest.mark.parametrize("similarity_metric", ["cosine", "euclidean", "dot_product"]) - def test_lancedb_openai_embeddings(self, temp_db_dir, sample_chunks, sample_queries, similarity_metric): - """Test LanceDB retrieval with OpenAI embeddings and different similarity metrics.""" - # Skip if OpenAI API key not available - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - # Get embedder - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - # Generate embeddings - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - # Test retrieval - handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - settings = { - "top_k": 3, - "metric": similarity_metric, - } - - results = handler( - sample_chunks, - chunk_embeddings, - sample_queries, - query_embeddings, - settings, - temp_db_dir - ) - - # Assertions - assert len(results) == len(sample_queries) - for result in results: - assert "query_object" in result - assert "retrieved_chunks" in result - assert len(result["retrieved_chunks"]) <= 3 - - # Check structure of retrieved chunks - for chunk in result["retrieved_chunks"]: - assert "text" in chunk - assert "similarity" in chunk - assert "id" in chunk - assert isinstance(chunk["similarity"], (float, int)) - assert 0 <= chunk["similarity"] <= 1.1 # Allow slight numerical imprecision - - @pytest.mark.parametrize("top_k", [1, 3, 5]) - def test_lancedb_openai_different_top_k(self, temp_db_dir, sample_chunks, sample_queries, top_k): - """Test LanceDB retrieval with different top_k values.""" - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - settings = {"top_k": top_k, "metric": "cosine"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - for result in results: - assert len(result["retrieved_chunks"]) <= top_k - # Check that results are ordered by similarity (descending) - similarities = [chunk["similarity"] for chunk in result["retrieved_chunks"]] - assert similarities == sorted(similarities, reverse=True) - - # Commented out Cohere tests for now due to API availability issues - # def test_lancedb_cohere_embeddings(self, temp_db_dir, sample_chunks, sample_queries): - # """Test LanceDB retrieval with Cohere embeddings.""" - # if not os.environ.get("COHERE_API_KEY"): - # pytest.skip("COHERE_API_KEY not set") - - # embedder = EmbeddingMethodRegistry.get_embedder("cohere") - # api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - - # chunk_texts = [c["text"] for c in sample_chunks] - # chunk_embeddings = embedder(chunk_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - # query_texts = [q["text"] for q in sample_queries] - # query_embeddings = embedder(query_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - # handler = RetrievalMethodRegistry.get_handler("lancedb_vector_store") - # settings = {"top_k": 3, "metric": "cosine"} - - # results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - # assert len(results) == len(sample_queries) - # for result in results: - # assert len(result["retrieved_chunks"]) <= 3 - # assert all("similarity" in chunk for chunk in result["retrieved_chunks"]) - def test_lancedb_sentence_transformers(self, temp_db_dir, sample_chunks, sample_queries): """Test LanceDB retrieval with Sentence Transformers embeddings.""" embedder = EmbeddingMethodRegistry.get_embedder("sentence-transformers") @@ -423,100 +323,6 @@ def test_lancedb_relevance_ordering(self, temp_db_dir, sample_chunks): class TestEmbeddingRetrievalWithFAISS: """Test embedding-based retrieval with FAISS backend across all providers.""" - @pytest.mark.parametrize("similarity_metric", ["l2", "cosine", "dot"]) - def test_faiss_openai_embeddings(self, temp_db_dir, sample_chunks, sample_queries, similarity_metric): - """Test FAISS retrieval with OpenAI embeddings and different similarity metrics.""" - # Skip if FAISS not installed - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": 3, "metric": similarity_metric} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - assert len(results) == len(sample_queries) - for result in results: - assert "query_object" in result - assert "retrieved_chunks" in result - assert len(result["retrieved_chunks"]) <= 3 - - for chunk in result["retrieved_chunks"]: - assert "text" in chunk - assert "similarity" in chunk - assert "id" in chunk - assert isinstance(chunk["similarity"], (float, int)) - - @pytest.mark.parametrize("top_k", [1, 3, 5]) - def test_faiss_different_top_k(self, temp_db_dir, sample_chunks, sample_queries, top_k): - """Test FAISS retrieval with different top_k values.""" - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("openai") - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="text-embedding-3-small", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": top_k, "metric": "l2"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - for result in results: - assert len(result["retrieved_chunks"]) <= top_k - - def test_faiss_cohere_embeddings(self, temp_db_dir, sample_chunks, sample_queries): - """Test FAISS retrieval with Cohere embeddings.""" - try: - import faiss - except ImportError: - pytest.skip("FAISS not installed") - - if not os.environ.get("COHERE_API_KEY"): - pytest.skip("COHERE_API_KEY not set") - - embedder = EmbeddingMethodRegistry.get_embedder("cohere") - api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - - chunk_texts = [c["text"] for c in sample_chunks] - chunk_embeddings = embedder(chunk_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - query_texts = [q["text"] for q in sample_queries] - query_embeddings = embedder(query_texts, model_name="embed-english-v3.0", api_keys=api_keys) - - handler = RetrievalMethodRegistry.get_handler("faiss_vector_store") - settings = {"top_k": 3, "metric": "cosine"} - - results = handler(sample_chunks, chunk_embeddings, sample_queries, query_embeddings, settings, temp_db_dir) - - assert len(results) == len(sample_queries) - for result in results: - assert len(result["retrieved_chunks"]) <= 3 - def test_faiss_sentence_transformers(self, temp_db_dir, sample_chunks, sample_queries): """Test FAISS retrieval with Sentence Transformers embeddings.""" try: @@ -662,13 +468,10 @@ def test_backends_return_similar_results(self, temp_db_dir, sample_chunks, sampl # Both should return the same top result (most relevant chunk) assert lance_results[0]["retrieved_chunks"][0]["text"] == faiss_results[0]["retrieved_chunks"][0]["text"] - @pytest.mark.parametrize("embedding_provider", [ - "sentence-transformers", - pytest.param("openai", marks=pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set")), - pytest.param("cohere", marks=pytest.mark.skipif(not os.environ.get("COHERE_API_KEY"), reason="COHERE_API_KEY not set")) - ]) - def test_all_providers_with_both_backends(self, temp_db_dir, sample_chunks, sample_queries, embedding_provider): - """Test that all embedding providers work with both backends.""" + def test_a_local_model_with_both_backends(self, temp_db_dir, sample_chunks, sample_queries): + """A real (local) embedding model's vectors work with both backends. Real + providers' APIs are checked, on request only, in test_live_apis.py.""" + embedding_provider = "sentence-transformers" try: import faiss except ImportError: @@ -676,28 +479,11 @@ def test_all_providers_with_both_backends(self, temp_db_dir, sample_chunks, samp embedder = EmbeddingMethodRegistry.get_embedder(embedding_provider) - # Configure model and API keys based on provider - if embedding_provider == "openai": - model_name = "text-embedding-3-small" - api_keys = {"OpenAI": os.environ.get("OPENAI_API_KEY")} - elif embedding_provider == "cohere": - model_name = "embed-english-v3.0" - api_keys = {"Cohere": os.environ.get("COHERE_API_KEY")} - else: - model_name = "all-MiniLM-L6-v2" - api_keys = None - + model_name = "all-MiniLM-L6-v2" chunk_texts = [c["text"] for c in sample_chunks] - if api_keys: - chunk_embeddings = embedder(chunk_texts, model_name=model_name, api_keys=api_keys) - else: - chunk_embeddings = embedder(chunk_texts, model_name=model_name) - + chunk_embeddings = embedder(chunk_texts, model_name=model_name) query_texts = [q["text"] for q in sample_queries[:1]] - if api_keys: - query_embeddings = embedder(query_texts, model_name=model_name, api_keys=api_keys) - else: - query_embeddings = embedder(query_texts, model_name=model_name) + query_embeddings = embedder(query_texts, model_name=model_name) # Test LanceDB lancedb_dir = os.path.join(temp_db_dir, f"lancedb_{embedding_provider}") diff --git a/tests/test_vector_retrievers.py b/tests/test_vector_retrievers.py index e4c0a08dd..ecd19f6f9 100644 --- a/tests/test_vector_retrievers.py +++ b/tests/test_vector_retrievers.py @@ -221,6 +221,15 @@ def test_top_k_is_respected(self, embedded, temp_db_dir): embedded["query_embeddings"], temp_db_dir, top_k=1) assert all(len(r["retrieved_chunks"]) <= 1 for r in results) + @pytest.mark.parametrize("metric", ["l2", "cosine", "dot"]) + def test_every_supported_metric_ranks_by_similarity(self, embedded, temp_db_dir, metric): + results = run("faiss_vector_store", embedded["chunks"], + embedded["chunk_embeddings"], embedded["queries"], + embedded["query_embeddings"], temp_db_dir, top_k=3, metric=metric) + for result in results: + similarities = [h["similarity"] for h in result["retrieved_chunks"]] + assert similarities and similarities == sorted(similarities, reverse=True) + def test_store_persists_across_calls(self, embedded, temp_db_dir): run("faiss_vector_store", embedded["chunks"], embedded["chunk_embeddings"], embedded["queries"], embedded["query_embeddings"], temp_db_dir, top_k=3)