diff --git a/src/data_generation/userguide/fetch.py b/src/data_generation/userguide/fetch.py index c57f85a..e740aff 100644 --- a/src/data_generation/userguide/fetch.py +++ b/src/data_generation/userguide/fetch.py @@ -1,3 +1,4 @@ +import re import time from collections.abc import Sequence from pathlib import Path @@ -6,6 +7,29 @@ import requests USER_AGENT = "ReactomeChatbot/1.0 (+https://github.com/reactome/reactome_chatbot)" + +# The thinnest real guide page is review-status at about 400 words; the shell that +# prompted this floor is ~1,240 words of CSS, so a raw character count cannot +# separate them. Stripping tags first is what makes it work: the shell's bulk sits +# inside