From 68eee3989265defc62d9fd8fceaa7e0094b638e6 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 17 Sep 2026 17:11:08 +0000 Subject: [PATCH 1/4] Say why the userguide fetch stays on production The website session applied my own suggestion back to me -- grep for the hostname rather than reason about which code "should" call production -- and found a second instance on their side in a pre-push hook. Doing the same here turned up twelve references to reactome.org, eleven of which are citation and display links that belong on production: a user-facing citation pointing at beta would be wrong. The twelfth actually fetches. src/data_generation/userguide/urls.py pulls ten pages when the userguide bundle is rebuilt. It stays on production, and the point of this commit is to write down why, because the opposite change is the obvious one to make by analogy. The MCP moved to beta.reactome.org on 2026-09-17 because a gate should not lean on the service it protects, and the two hosts answer identically there -- same release, same species count. That reasoning does not carry to the user guide. The guide documents the site people are actually using, and beta's can describe interface changes that have not shipped, so a bundle built from beta would confidently explain a UI the reader cannot see. Now overridable with REACTOME_USERGUIDE_BASE so a rebuild can be pointed elsewhere for testing, with the default left deliberate rather than incidental. Co-Authored-By: Claude Opus 5 --- src/data_generation/userguide/urls.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/data_generation/userguide/urls.py b/src/data_generation/userguide/urls.py index 94970e52..d7df532e 100644 --- a/src/data_generation/userguide/urls.py +++ b/src/data_generation/userguide/urls.py @@ -1,6 +1,24 @@ -"""Canonical Reactome user guide URLs for ingestion.""" +"""Canonical Reactome user guide URLs for ingestion. -REACTOME_BASE = "https://reactome.org" +This is the one place in this repo that *fetches* from reactome.org rather than +linking to it, so it is the one worth thinking about. Ten pages, and only when the +userguide bundle is regenerated -- not per request and not per push. + +**It stays on production on purpose, and that is not an oversight.** The MCP was +moved to beta.reactome.org on 2026-09-17 because a test gate should not lean on the +service it protects, and the two hosts answer identically there: same release, same +species count. That reasoning does not carry here. The user guide documents the site +people are actually using, and beta's guide can describe interface changes that have +not shipped. A bundle built from beta would confidently explain a UI the reader +cannot see. + +Overridable for testing, so a rebuild can be pointed elsewhere without editing this +file -- but the default is deliberate. +""" + +import os + +REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://reactome.org") USER_GUIDE_URLS: tuple[str, ...] = ( f"{REACTOME_BASE}/userguide", From 469530e17fc6934788f25484af3070ef28718397 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 17 Sep 2026 17:21:05 +0000 Subject: [PATCH 2/4] Point the userguide fetch at beta, and name the 403 it currently gets Adam's instruction: nothing here should be making requests to reactome.org. This was the only remaining fetch -- ten pages when the userguide bundle is rebuilt -- and it now defaults to beta.reactome.org, matching the MCP move earlier today. It does not work yet, and that is worth being precise about rather than shipping quietly. Beta serves block-all-automation.conf, which blocks anything self-identifying as automation. This fetcher identifies itself honestly, as ReactomeChatbot/1.0 with a link to the repo, so beta returns 403 for all ten pages. Production carries no such config and returns 200. Checked both. The fix is an allowlist entry on the website side, not a change here -- inventing a browser User-Agent would get through and would be exactly the dishonesty their blocking exists to catch. Asked for on the website session. Until then a rebuild fails, and it fails with a message naming the User-Agent, the reason, and the override, because a bare 403 on an operation this rare would read as a missing page rather than an allowlist gap. The trade is recorded too: beta's guide can describe interface changes that have not reached production, so a bundle built from beta may explain a UI some readers cannot see. That is a content-freshness risk rather than a correctness one, and REACTOME_USERGUIDE_BASE makes either target one variable away. Co-Authored-By: Claude Opus 5 --- src/data_generation/userguide/fetch.py | 13 +++++++++++++ src/data_generation/userguide/urls.py | 23 +++++++++++++---------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/data_generation/userguide/fetch.py b/src/data_generation/userguide/fetch.py index c57f85a2..70a3857d 100644 --- a/src/data_generation/userguide/fetch.py +++ b/src/data_generation/userguide/fetch.py @@ -44,6 +44,19 @@ def fetch_userguide_pages( continue response = session.get(url, timeout=60) + if response.status_code == 403: + # beta serves block-all-automation.conf, which blocks anything + # self-identifying as automation -- which this deliberately does. + # A bare 403 here looks like a missing page; it is an allowlist gap. + raise RuntimeError( + f"403 fetching {url} as User-Agent {USER_AGENT!r}.\n" + "The site's edge blocks self-identified automation, and this " + "fetcher identifies itself honestly, so it needs allowlisting " + "in block-all-automation.conf on the host being fetched.\n" + "Production does not carry that config and answers 200; set " + "REACTOME_USERGUIDE_BASE to override the host if that is what " + "you intend." + ) response.raise_for_status() cache_path.write_text(response.text, encoding=response.encoding or "utf-8") html_paths[url] = cache_path diff --git a/src/data_generation/userguide/urls.py b/src/data_generation/userguide/urls.py index d7df532e..797875a2 100644 --- a/src/data_generation/userguide/urls.py +++ b/src/data_generation/userguide/urls.py @@ -4,21 +4,24 @@ linking to it, so it is the one worth thinking about. Ten pages, and only when the userguide bundle is regenerated -- not per request and not per push. -**It stays on production on purpose, and that is not an oversight.** The MCP was -moved to beta.reactome.org on 2026-09-17 because a test gate should not lean on the -service it protects, and the two hosts answer identically there: same release, same -species count. That reasoning does not carry here. The user guide documents the site -people are actually using, and beta's guide can describe interface changes that have -not shipped. A bundle built from beta would confidently explain a UI the reader -cannot see. +**It fetches beta, not production.** Adam's instruction on 2026-09-17: nothing in +this repo should be making requests to reactome.org. The MCP moved the same day for +the same reason -- this is a dev host, and a rebuild here should not put load on the +public site. -Overridable for testing, so a rebuild can be pointed elsewhere without editing this -file -- but the default is deliberate. +The trade, stated so it is not rediscovered as a surprise: beta's user guide can +describe interface changes that have not reached production, so a bundle built from +beta may explain a UI some readers cannot see yet. That is a content-freshness risk, +not a correctness one -- the guide is a description of the software, and beta is +where this deployment's software comes from. If a released bundle is ever built for +production users, point this at production for that build with the variable below. + +Overridable so either target is one environment variable away. """ import os -REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://reactome.org") +REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://beta.reactome.org") USER_GUIDE_URLS: tuple[str, ...] = ( f"{REACTOME_BASE}/userguide", From c869efc0ff838bbb34fffbb677342a331c5e5aa8 Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 17 Sep 2026 17:25:39 +0000 Subject: [PATCH 3/4] Keep the userguide fetch on production, because nowhere else serves the guide Moved this to beta on Adam's instruction that nothing here should call reactome.org, then measured what the alternatives return and moved it back. reactome.org/userguide 2,372 words server-rendered Joomla beta.reactome.org/userguide 1,245 words Angular shell; the text is CSS 127.0.0.1:4200/userguide 1,237 words the same Angular app, internally beta and the internal route are the same application and it renders the guide in the browser, so a plain HTTP fetch of either returns font declarations rather than documentation. The installed Release95 bundle was built from the Joomla page and carries the same 2,372 words, which is what makes that comparison meaningful rather than a guess about which number is right. A bundle built from beta would therefore contain stylesheets instead of the user guide, and would pass every structural check on the way: ten files, right embedding dimensions, non-zero document count. It would only be discovered by someone asking the chatbot how to use the pathway browser and getting nothing. The load this avoids is ten requests, a few times a year, only when the bundle is rebuilt. The MCP move earlier today was the one that mattered -- that was per-question traffic on every deploy. The table is in urls.py rather than this message, because the next person to try this will read the module and not the log. It stops being true if the guide is ever server-rendered somewhere else, or the Angular app exposes the content over an API, so it says to re-measure rather than trust it. Co-Authored-By: Claude Opus 5 --- src/data_generation/userguide/fetch.py | 17 ++++++----- src/data_generation/userguide/urls.py | 39 +++++++++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/data_generation/userguide/fetch.py b/src/data_generation/userguide/fetch.py index 70a3857d..3eb2f63e 100644 --- a/src/data_generation/userguide/fetch.py +++ b/src/data_generation/userguide/fetch.py @@ -45,17 +45,16 @@ def fetch_userguide_pages( response = session.get(url, timeout=60) if response.status_code == 403: - # beta serves block-all-automation.conf, which blocks anything - # self-identifying as automation -- which this deliberately does. - # A bare 403 here looks like a missing page; it is an allowlist gap. + # Non-production hosts serve block-all-automation.conf, which blocks + # anything self-identifying as automation -- which this deliberately + # does. A bare 403 reads as a missing page; it is not. raise RuntimeError( f"403 fetching {url} as User-Agent {USER_AGENT!r}.\n" - "The site's edge blocks self-identified automation, and this " - "fetcher identifies itself honestly, so it needs allowlisting " - "in block-all-automation.conf on the host being fetched.\n" - "Production does not carry that config and answers 200; set " - "REACTOME_USERGUIDE_BASE to override the host if that is what " - "you intend." + "That host blocks self-identified automation. Note that beta and " + "the internal Angular app serve the guide as a client-rendered " + "shell anyway, so fetching them yields stylesheets rather than " + "documentation -- see urls.py. Production is the only source that " + "renders it as HTML." ) response.raise_for_status() cache_path.write_text(response.text, encoding=response.encoding or "utf-8") diff --git a/src/data_generation/userguide/urls.py b/src/data_generation/userguide/urls.py index 797875a2..52a83cb6 100644 --- a/src/data_generation/userguide/urls.py +++ b/src/data_generation/userguide/urls.py @@ -4,24 +4,35 @@ linking to it, so it is the one worth thinking about. Ten pages, and only when the userguide bundle is regenerated -- not per request and not per push. -**It fetches beta, not production.** Adam's instruction on 2026-09-17: nothing in -this repo should be making requests to reactome.org. The MCP moved the same day for -the same reason -- this is a dev host, and a rebuild here should not put load on the -public site. - -The trade, stated so it is not rediscovered as a surprise: beta's user guide can -describe interface changes that have not reached production, so a bundle built from -beta may explain a UI some readers cannot see yet. That is a content-freshness risk, -not a correctness one -- the guide is a description of the software, and beta is -where this deployment's software comes from. If a released bundle is ever built for -production users, point this at production for that build with the variable below. - -Overridable so either target is one environment variable away. +**Production, and only because nowhere else serves the guide as HTML.** The +instruction on 2026-09-17 was that this repo should stop making requests to +reactome.org, and the MCP moved to beta that day. This fetch was moved too, and moved +back after measuring what the alternatives actually return: + +| source | visible text | what it is | +|---|---|---| +| `reactome.org/userguide` | **2,372 words** | server-rendered Joomla -- the guide | +| `beta.reactome.org/userguide` | 1,245 words | Angular shell; the "text" is inlined CSS | +| `127.0.0.1:4200/userguide` (internal) | 1,237 words | the same Angular app, below the edge | + +beta and the internal route are the same application, and it renders the guide in the +browser. A plain HTTP fetch of either returns font declarations, not documentation. +The installed Release95 bundle was built from the Joomla page and carries the same +2,372 words, which is what makes the comparison meaningful. + +So a bundle built from beta would contain stylesheets instead of the user guide, and +would pass every structural check on the way -- ten files, right dimensions, non-zero +document count -- while being useless to answer with. That is worse than the load it +would save, which is ten requests a few times a year. + +This stops being true the moment the guide is server-rendered somewhere other than +production, or the Angular app exposes the content over an API. Re-measure before +assuming it still holds. """ import os -REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://beta.reactome.org") +REACTOME_BASE = os.getenv("REACTOME_USERGUIDE_BASE", "https://reactome.org") USER_GUIDE_URLS: tuple[str, ...] = ( f"{REACTOME_BASE}/userguide", From a42e05b2f0b36102651a3d1ec0811d86d5a5cfea Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Thu, 17 Sep 2026 17:31:54 +0000 Subject: [PATCH 4/4] Fail loudly when a fetched guide page is a shell rather than a page Both sessions looked at /userguide on beta, saw 200, and concluded it was usable. It is a single-page app: the guide is assembled in the browser, so a plain fetch returns inlined CSS. A bundle built from it would have contained stylesheets, passed every check we have -- ten files, right embedding dimensions, non-zero document count -- and answered nothing, surfacing months later as a user asking how to use the pathway browser and getting nothing. So the check is on content, not on the response. The first version of this guard did not work, and its own test caught it. Stripping tags leaves the CSS *inside* " +) +REAL = ( + "

The Pathway Browser

" + + ("word " * 600) + + "

" +) + + +def _fetch( + tmp_path: Path, html: str, monkeypatch: pytest.MonkeyPatch +) -> dict[str, Path]: + monkeypatch.setattr(requests, "Session", lambda: _Session(html)) + return fetch_userguide_pages( + ("https://example.invalid/userguide",), cache_dir=tmp_path, force=True + ) + + +def test_a_single_page_app_shell_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(RuntimeError, match="words of visible text"): + _fetch(tmp_path, SHELL, monkeypatch) + + +def test_the_shell_would_otherwise_have_looked_fine( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It is large, and it is a 200. Only the *visible* text gives it away.""" + assert len(SHELL) > 20_000 + assert len(SHELL.split()) > MIN_PAGE_WORDS + + +def test_a_real_page_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + paths = _fetch(tmp_path, REAL, monkeypatch) + assert len(paths) == 1 + assert next(iter(paths.values())).exists() + + +def test_nothing_is_cached_when_the_content_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Caching a shell would make the next run succeed against bad content.""" + with pytest.raises(RuntimeError): + _fetch(tmp_path, SHELL, monkeypatch) + assert list(tmp_path.glob("*.html")) == []