diff --git a/src/data_generation/userguide/fetch.py b/src/data_generation/userguide/fetch.py index c57f85a..e740aff 100644 --- a/src/data_generation/userguide/fetch.py +++ b/src/data_generation/userguide/fetch.py @@ -1,3 +1,4 @@ +import re import time from collections.abc import Sequence from pathlib import Path @@ -6,6 +7,29 @@ import requests USER_AGENT = "ReactomeChatbot/1.0 (+https://github.com/reactome/reactome_chatbot)" + +# The thinnest real guide page is review-status at about 400 words; the shell that +# prompted this floor is ~1,240 words of CSS, so a raw character count cannot +# separate them. Stripping tags first is what makes it work: the shell's bulk sits +# inside " +) +REAL = ( + "

The Pathway Browser

" + + ("word " * 600) + + "

" +) + + +def _fetch( + tmp_path: Path, html: str, monkeypatch: pytest.MonkeyPatch +) -> dict[str, Path]: + monkeypatch.setattr(requests, "Session", lambda: _Session(html)) + return fetch_userguide_pages( + ("https://example.invalid/userguide",), cache_dir=tmp_path, force=True + ) + + +def test_a_single_page_app_shell_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(RuntimeError, match="words of visible text"): + _fetch(tmp_path, SHELL, monkeypatch) + + +def test_the_shell_would_otherwise_have_looked_fine( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It is large, and it is a 200. Only the *visible* text gives it away.""" + assert len(SHELL) > 20_000 + assert len(SHELL.split()) > MIN_PAGE_WORDS + + +def test_a_real_page_passes(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + paths = _fetch(tmp_path, REAL, monkeypatch) + assert len(paths) == 1 + assert next(iter(paths.values())).exists() + + +def test_nothing_is_cached_when_the_content_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Caching a shell would make the next run succeed against bad content.""" + with pytest.raises(RuntimeError): + _fetch(tmp_path, SHELL, monkeypatch) + assert list(tmp_path.glob("*.html")) == []