Keep the userguide fetch honest: production source, and a guard against fetching a shell

src/data_generation/userguide/fetch.py

@@ -1,3 +1,4 @@
+import re
 import time
 from collections.abc import Sequence
 from pathlib import Path
@@ -6,6 +7,29 @@
 import requests
 
 USER_AGENT = "ReactomeChatbot/1.0 (+https://github.com/reactome/reactome_chatbot)"
+
+# The thinnest real guide page is review-status at about 400 words; the shell that
+# prompted this floor is ~1,240 words of CSS, so a raw character count cannot
+# separate them. Stripping tags first is what makes it work: the shell's bulk sits
+# inside <style>, so its *visible* text is small.
+MIN_PAGE_WORDS = 250
+
+
+def _visible_word_count(html: str) -> int:
+    """Words a reader would see: script and style *contents* removed, not just tags.
+
+    Stripping tags alone is not enough and that is the whole difficulty. The shell
+    this guard exists for carries its bulk inside <style>, so tag-stripping leaves
+    about 1,240 words of CSS -- more than the thinnest real page has of prose. A
+    floor set against that number cannot separate them; removing the blocks first
+    takes the shell to near zero and leaves real pages untouched.
+    """
+    without_blocks = re.sub(
+        r"<(script|style)\b[^>]*>.*?</\1>", " ", html, flags=re.S | re.I
+    )
+    return len(re.sub(r"<[^>]*>", " ", without_blocks).split())
+
+
 REQUEST_DELAY_SECONDS = 0.5
 
 
@@ -44,7 +68,33 @@ def fetch_userguide_pages(
             continue
 
         response = session.get(url, timeout=60)
+        if response.status_code == 403:
+            # Non-production hosts serve block-all-automation.conf, which blocks
+            # anything self-identifying as automation -- which this deliberately
+            # does. A bare 403 reads as a missing page; it is not.
+            raise RuntimeError(
+                f"403 fetching {url} as User-Agent {USER_AGENT!r}.\n"
+                "That host blocks self-identified automation. Note that beta and "
+                "the internal Angular app serve the guide as a client-rendered "
+                "shell anyway, so fetching them yields stylesheets rather than "
+                "documentation -- see urls.py. Production is the only source that "
+                "renders it as HTML."
+            )
         response.raise_for_status()
+        # A 200 is not evidence that a page has content. beta and the internal
+        # Angular app both answer 200 for /userguide and return a client-rendered
+        # shell whose visible text is inlined CSS -- about 1,240 words of font
+        # declarations against production's 2,372 of documentation. A bundle built
+        # from that passes every structural check and answers nothing.
+        words = _visible_word_count(response.text)
+        if words < MIN_PAGE_WORDS:
+            raise RuntimeError(
+                f"{url} returned {response.status_code} but only {words} words "
+                f"of visible text (floor {MIN_PAGE_WORDS}). That is what a "
+                "single-page-app shell looks like: it renders in the browser and "
+                "a plain fetch gets stylesheets. See urls.py for which hosts "
+                "server-render the guide."
+            )
         cache_path.write_text(response.text, encoding=response.encoding or "utf-8")
         html_paths[url] = cache_path
 

src/data_generation/userguide/urls.py

@@ -1,6 +1,38 @@
-"""Canonical Reactome user guide URLs for ingestion."""
+"""Canonical Reactome user guide URLs for ingestion.
 
-REACTOME_BASE = "https://reactome.org"
+This is the one place in this repo that *fetches* from reactome.org rather than
+linking to it, so it is the one worth thinking about. Ten pages, and only when the
+userguide bundle is regenerated -- not per request and not per push.
+
+**Production, and only because nowhere else serves the guide as HTML.** The
+instruction on 2026-09-17 was that this repo should stop making requests to
+reactome.org, and the MCP moved to beta that day. This fetch was moved too, and moved
+back after measuring what the alternatives actually return:
+
+| source | visible text | what it is |
+|---|---|---|
+| `reactome.org/userguide` | **2,372 words** | server-rendered Joomla -- the guide |
+| `beta.reactome.org/userguide` | 1,245 words | Angular shell; the "text" is inlined CSS |
+| `127.0.0.1:4200/userguide` (internal) | 1,237 words | the same Angular app, below the edge |
+
+beta and the internal route are the same application, and it renders the guide in the
+browser. A plain HTTP fetch of either returns font declarations, not documentation.
+The installed Release95 bundle was built from the Joomla page and carries the same
+2,372 words, which is what makes the comparison meaningful.
+
+So a bundle built from beta would contain stylesheets instead of the user guide, and
+would pass every structural check on the way -- ten files, right dimensions, non-zero
+document count -- while being useless to answer with. That is worse than the load it
+would save, which is ten requests a few times a year.
+
+This stops being true the moment the guide is server-rendered somewhere other than
+production, or the Angular app exposes the content over an API. Re-measure before
+assuming it still holds.
+"""
+
+import os
+
+REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://reactome.org")
 
 USER_GUIDE_URLS: tuple[str, ...] = (
     f"{REACTOME_BASE}/userguide",

tests/data_generation/test_userguide_content_floor.py

@@ -0,0 +1,90 @@
+"""A 200 is not evidence that a page has content.
+
+beta and the internal Angular app both answer 200 for /userguide and return a
+client-rendered shell: the guide is assembled in the browser, so a plain fetch
+gets inlined CSS and font declarations. Measured 2026-09-17 -- about 1,240 words
+of visible text against production's 2,372 of documentation.
+
+A bundle built from that would pass every check we have (ten files, correct
+embedding dimensions, non-zero document count) and answer nothing. Two sessions
+checked status codes and paths and both concluded it was fine, which is exactly
+why the check is on content rather than on the response.
+"""
+
+from pathlib import Path
+
+import pytest
+import requests
+
+from data_generation.userguide.fetch import MIN_PAGE_WORDS, fetch_userguide_pages
+
+
+class _Response:
+    def __init__(self, text: str) -> None:
+        self.text = text
+        self.status_code = 200
+        self.encoding = "utf-8"
+
+    def raise_for_status(self) -> None:
+        pass
+
+
+class _Session:
+    def __init__(self, text: str) -> None:
+        self.text = text
+        self.headers: dict[str, str] = {}
+
+    def get(self, _url: str, timeout: int = 0) -> _Response:
+        return _Response(self.text)
+
+
+SHELL = (
+    "<html><head><style>"
+    + "@font-face{font-family:'Roboto';src:url(https://fonts.gstatic.com/x.woff2);} "
+    * 400
+    + "</style></head><body><app-root></app-root></body></html>"
+)
+REAL = (
+    "<html><body><h1>The Pathway Browser</h1><p>"
+    + ("word " * 600)
+    + "</p></body></html>"
+)
+
+
+def _fetch(
+    tmp_path: Path, html: str, monkeypatch: pytest.MonkeyPatch
+) -> dict[str, Path]:
+    monkeypatch.setattr(requests, "Session", lambda: _Session(html))
+    return fetch_userguide_pages(
+        ("https://example.invalid/userguide",), cache_dir=tmp_path, force=True
+    )
+
+
+def test_a_single_page_app_shell_is_rejected(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    with pytest.raises(RuntimeError, match="words of visible text"):
+        _fetch(tmp_path, SHELL, monkeypatch)
+
+
+def test_the_shell_would_otherwise_have_looked_fine(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """It is large, and it is a 200. Only the *visible* text gives it away."""
+    assert len(SHELL) > 20_000
+    assert len(SHELL.split()) > MIN_PAGE_WORDS
+
+
+def test_a_real_page_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    paths = _fetch(tmp_path, REAL, monkeypatch)
+    assert len(paths) == 1
+    assert next(iter(paths.values())).exists()
+
+
+def test_nothing_is_cached_when_the_content_is_rejected(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Caching a shell would make the next run succeed against bad content."""
+    with pytest.raises(RuntimeError):
+        _fetch(tmp_path, SHELL, monkeypatch)
+    assert list(tmp_path.glob("*.html")) == []