From 828fb6ebf6a02c3a8ad003ebe04494878e8cbeff Mon Sep 17 00:00:00 2001 From: Alex Lee <13192723+leeaky@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:54:15 +1100 Subject: [PATCH 1/3] arXiv: one plain line per skipped paper, hide PDF reader notes Co-Authored-By: Claude Opus 5.5 --- docs/build-map.md | 1 + src/readurlist/ingest/arxiv.py | 23 +++++++++++++++++++---- tests/test_arxiv.py | 26 ++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/build-map.md b/docs/build-map.md index 33284a9..2149cb7 100644 --- a/docs/build-map.md +++ b/docs/build-map.md @@ -321,6 +321,7 @@ Before any feature work, dev and live are separated: the live worker, bot, datab - **Terms:** Effort label - **Status:** done - **Changed:** 2026-11-01 — Stage added at the owner's request after the build finished. Owner chose to fetch the full paper ("Fetch the full paper", over treating all arXiv as long or tagging only) and to keep the summary from the abstract ("Abstract, length from paper"): the safety scan and the model's input stay as they are. + 2026-11-01 — After the live one-off fix (owner: "ok to the tidy up"): a paper whose PDF is skipped now prints one line naming it, its abstract link and the reason (not found, over 40 MB, can't be downloaded or read, no text), and the PDF reader's font and figure notes are hidden. Tests: `test_arxiv.py`. Local run: "arXiv 2608.00858 (https://arxiv.org/abs/2608.00858): PDF not found (HTTP 404); kept the abstract count", then the totals; nothing else printed. - **Tasks:** - [x] (a) A new arXiv link (abs, pdf or html form) is counted from the full paper's PDF; the title and summary still come from the abstract page. Detail: if the PDF can't be fetched or read, the abstract's count is kept, as today. Built: `ingest/arxiv.py` (`arxiv_id`, `paper_word_count`: PDF up to 40 MB, read with `pypdf`, words counted, text not kept); `ingest_url` reads any arXiv form from its abstract page. New dependency `pypdf`. Tests: `test_arxiv.py`. diff --git a/src/readurlist/ingest/arxiv.py b/src/readurlist/ingest/arxiv.py index 73edd05..1349430 100644 --- a/src/readurlist/ingest/arxiv.py +++ b/src/readurlist/ingest/arxiv.py @@ -17,6 +17,8 @@ from readurlist.effort import word_count logger = logging.getLogger(__name__) +# pypdf reports every odd font or figure it meets; the count doesn't need them. +logging.getLogger("pypdf").setLevel(logging.CRITICAL) # New ids (2410.12345, optional version) and old ones (hep-th/9901001). _ARXIV = re.compile( @@ -40,22 +42,35 @@ def abs_url(paper_id: str) -> str: def paper_word_count(paper_id: str) -> int | None: """Words in the paper's PDF, or None if it can't be fetched or read.""" url = f"https://arxiv.org/pdf/{paper_id}" + + def skip(reason: str) -> None: + logger.warning("arXiv %s (%s): %s; kept the abstract count", paper_id, abs_url(paper_id), reason) + try: with httpx.Client(follow_redirects=True, timeout=_TIMEOUT, transport=_TRANSPORT) as client: with client.stream("GET", url) as response: + if response.status_code == 404: + skip("PDF not found (HTTP 404)") + return None response.raise_for_status() body = bytearray() for chunk in response.iter_bytes(): body += chunk if len(body) > _MAX_PDF_BYTES: - logger.warning("arXiv PDF %s is over %s bytes; skipped", paper_id, _MAX_PDF_BYTES) + skip(f"PDF over {_MAX_PDF_BYTES // (1024 * 1024)} MB") return None reader = PdfReader(io.BytesIO(bytes(body))) words = sum(word_count(page.extract_text() or "") for page in reader.pages) - except Exception: - logger.warning("Could not count words in arXiv PDF %s", paper_id, exc_info=True) + except httpx.HTTPError as exc: + skip(f"PDF could not be downloaded ({exc.__class__.__name__})") + return None + except Exception as exc: + skip(f"PDF could not be read ({exc.__class__.__name__})") + return None + if not words: + skip("no text in the PDF") return None - return words or None + return words def fix_saved_papers(*, sleep=time.sleep) -> tuple[int, int]: diff --git a/tests/test_arxiv.py b/tests/test_arxiv.py index 64b756b..8343d9a 100644 --- a/tests/test_arxiv.py +++ b/tests/test_arxiv.py @@ -179,3 +179,29 @@ def fake_count(paper_id): assert words["https://arxiv.org/abs/2410.00001"] == 8000 assert words["https://arxiv.org/abs/2410.00002"] == 120 assert words["https://example.com/a"] == 120 + + +def test_a_missing_pdf_is_one_plain_line(monkeypatch, caplog): + monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(None, [])) + with caplog.at_level("WARNING"): + assert paper_word_count("2608.00858") is None + lines = [r for r in caplog.records if r.name.startswith("readurlist")] + assert [r.getMessage() for r in lines] == [ + "arXiv 2608.00858 (https://arxiv.org/abs/2608.00858): PDF not found (HTTP 404); kept the abstract count" + ] + assert all(r.exc_info is None for r in lines) # no traceback + + +def test_a_pdf_without_text_says_so(monkeypatch, caplog): + monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(_pdf(""), [])) + with caplog.at_level("WARNING"): + assert paper_word_count("2410.12345") is None + assert [r.getMessage() for r in caplog.records if r.name.startswith("readurlist")] == [ + "arXiv 2410.12345 (https://arxiv.org/abs/2410.12345): no text in the PDF; kept the abstract count" + ] + + +def test_pdf_reader_notes_are_hidden(): + import logging + + assert logging.getLogger("pypdf").getEffectiveLevel() > logging.ERROR From cce0192c1639d8c90b924924b76eabde39d857ad Mon Sep 17 00:00:00 2001 From: Alex Lee <13192723+leeaky@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:34:01 +1100 Subject: [PATCH 2/3] Stage 12: papers from other sites get their length from their PDF A page naming its PDF in the citation_pdf_url tag is counted from that PDF, with stage 11's rules. One-off readurlist-paper-lengths rechecks saved non-arXiv items shown as Short. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++ docs/build-map.md | 19 ++++ pyproject.toml | 1 + src/readurlist/ingest/__init__.py | 15 ++- src/readurlist/ingest/arxiv.py | 73 ++++++++++++++- src/readurlist/ingest/extract.py | 30 +++++- tests/test_arxiv.py | 21 ++++- tests/test_effort.py | 2 +- tests/test_ingest_dedup.py | 2 +- tests/test_ingest_vocab.py | 2 +- tests/test_paper_pdf.py | 151 ++++++++++++++++++++++++++++++ tests/test_tripwire.py | 2 +- 12 files changed, 310 insertions(+), 16 deletions(-) create mode 100644 tests/test_paper_pdf.py diff --git a/README.md b/README.md index 18072f6..1b22aa4 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,14 @@ readurlist-arxiv-lengths It pauses 3 seconds between papers, as arXiv asks. +**Papers from other sites:** any page that names its PDF in the standard `citation_pdf_url` tag (journals, ACL Anthology, bioRxiv and most paper sites) gets its word count from that PDF the same way. Sites that block downloads keep the page's count. Items saved before this change and shown as Short are rechecked once with: + +```bash +readurlist-paper-lengths +``` + +It re-reads each Short item's page, 2 seconds apart, and recounts only those that name a PDF. + ### Migrate an old local SQLite file ```bash diff --git a/docs/build-map.md b/docs/build-map.md index 2149cb7..1f15c61 100644 --- a/docs/build-map.md +++ b/docs/build-map.md @@ -334,6 +334,25 @@ Before any feature work, dev and live are separated: the live worker, bot, datab - **Verified:** 2026-11-01 — `uv run pytest -q`: 136 passed. Local database, real link `https://arxiv.org/abs/1706.03762`: saved ready, title "Attention Is All You Need", abstract 346 words, stored 6,111 words → long, 27 min. Reset to 346 and ran `readurlist-arxiv-lengths`: "counted from their PDF: 2; kept the abstract count: 0"; item back to 6,111. 2026-11-01 — Deploy: no database change; owner merged #19 (`da90cf4`) and ran `scripts/live-worker-update.ps1`; one live `readurlist.exe`, PID 7204, started 16:26, task result 0. Owner ran `readurlist-arxiv-lengths` in the live folder: "arXiv papers counted from their PDF: 82; kept the abstract count: 2". Both had no PDF (HTTP 404): 2608.00858, withdrawn by its author, shows "Short · 1 min" on All; and 2603.28371, also withdrawn (owner checked). The run also printed pypdf font and figure warnings; counts unaffected. +### Stage 12 — Papers from other sites get their real length +- **What gets built:** A saved page that names its paper's PDF in the standard `citation_pdf_url` tag (journals, bioRxiv, SSRN, OpenReview and most paper sites) gets its reading time from that PDF, as arXiv papers do since stage 11. +- **Why:** Owner, asked whether they save papers from sites other than arXiv: "Yes, some"; chose "Any page with a PDF tag" over a fixed list of sites (Chat 2026-11-01). +- **Depends on:** Stage 11 +- **Produces:** correct Short / Long labels and sections for papers saved from paper sites. +- **Terms:** Effort label +- **Status:** in progress +- **Changed:** 2026-11-01 — Stage added at the owner's request after stage 11. For papers already saved, the owner chose to recheck only items shown as Short ("Only items shown as Short"). +- **Tasks:** + - [x] (a) A new link whose page names its PDF in the standard citation tag is counted from that PDF; title and summary still come from the page. + Detail: the tag is `citation_pdf_url`. Same rules as stage 11: PDF up to 40 MB, its text is counted and not kept, never shown to the model; if the PDF can't be fetched or read (some sites block downloads), the page's count is kept and one line says why. Detail: built: `extract.pdf_link` reads the tag (relative links resolved, http(s) only); `extract_article` now also returns it; `arxiv.pdf_word_count` counts any PDF with stage 11's rules; an arXiv link never falls through to its abstract page's tag. Tests: `test_paper_pdf.py`, `test_arxiv.py`. + - [x] (b) Saved non-arXiv items shown as Short get their length fixed once. + Detail: a one-off command the owner runs on the live folder after the deploy; it re-reads each Short item's page for the tag, pausing between pages, and recounts only those that have one. Detail: built: `readurlist-paper-lengths` (ready, non-arXiv, Short only; 2 seconds between pages; prints counted / no PDF tag / kept). Test: `test_paper_pdf.py::test_one_off_fix_rechecks_short_paper_pages`. + - [ ] (c) Deploy (you): merge to main, update and restart the live worker, run the one-off fix, then check a paper from another site on All shows its real minutes. + Detail: no database change; merge → `scripts/live-worker-update.ps1` → one-off fix in the live folder. +- **How we know:** on the local database, saving a paper page with a `citation_pdf_url` tag stores the PDF's word count while the summary matches the page; the one-off fix turns an old Short paper item Long and leaves a plain Short article as it was. +- **Issues:** none +- **Verified:** 2026-11-01 — `uv run pytest -q`: 154 passed. Local database, real link `https://aclanthology.org/N19-1423/`: saved ready, title "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", page 1,062 words, stored 10,160 words → long, 44 min. Reset to the page count and ran the one-off fix: counted 1, no PDF tag 1, kept 13 (local test items on example.com, each one line "page could not be fetched"); item back to 10,160. + ## Blocks ### B1 — Stage 0: no room for a dev Supabase project (2026-11-01) diff --git a/pyproject.toml b/pyproject.toml index f23c9ec..4dce3d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ readurlist = "readurlist.main:main" readurlist-digest = "readurlist.rank.run:main" readurlist-effort-backfill = "readurlist.effort:main" readurlist-arxiv-lengths = "readurlist.ingest.arxiv:main" +readurlist-paper-lengths = "readurlist.ingest.arxiv:paper_pages_main" [build-system] requires = ["setuptools>=68"] diff --git a/src/readurlist/ingest/__init__.py b/src/readurlist/ingest/__init__.py index 6e0be37..af1a84f 100644 --- a/src/readurlist/ingest/__init__.py +++ b/src/readurlist/ingest/__init__.py @@ -3,7 +3,7 @@ from readurlist.config import Settings from readurlist.db import Item, get_session from readurlist.effort import word_count -from readurlist.ingest.arxiv import abs_url, arxiv_id, paper_word_count +from readurlist.ingest.arxiv import abs_url, arxiv_id, paper_word_count, pdf_word_count from readurlist.ingest.extract import ExtractError, FetchError, extract_article from readurlist.ingest.summarize import summarize_article from readurlist.ingest.untrusted import assess @@ -62,6 +62,15 @@ def _save_stub( session.close() +def _paper_words(url: str, paper_id: str | None, pdf_url: str | None) -> int | None: + """Words in the full paper (stages 11 and 12), or None to keep the page's count.""" + if paper_id: + return paper_word_count(paper_id) + if pdf_url: + return pdf_word_count(pdf_url, page_url=url) + return None + + def ingest_url( url: str, llm: LLMProvider, @@ -90,7 +99,7 @@ def ingest_url( paper_id = arxiv_id(url) try: # An arXiv link of any form is read from its abstract page. - title_hint, text = extract_article(abs_url(paper_id) if paper_id else url) + title_hint, text, pdf_url = extract_article(abs_url(paper_id) if paper_id else url) except FetchError as exc: return _save_stub(url=url, title=url, note=str(exc), note_clean=note_clean) except ExtractError as exc: @@ -137,7 +146,7 @@ def ingest_url( topics=list(result.topics), keywords=list(result.keywords), extracted_text=text, - word_count=(paper_id and paper_word_count(paper_id)) or word_count(text), + word_count=_paper_words(url, paper_id, pdf_url) or word_count(text), priority=result.priority, note=note_clean, ingest_status=INGEST_READY, diff --git a/src/readurlist/ingest/arxiv.py b/src/readurlist/ingest/arxiv.py index 1349430..4849bec 100644 --- a/src/readurlist/ingest/arxiv.py +++ b/src/readurlist/ingest/arxiv.py @@ -3,6 +3,8 @@ The abstract page gives the title and the text the summary is written from, as for any page. The PDF is downloaded only to count its words; its text is never stored or shown to the model (owner, 2026-11-01: "Abstract, length from paper"). + +Stage 12 does the same for any page naming its PDF in the citation_pdf_url tag. """ from __future__ import annotations @@ -40,12 +42,24 @@ def abs_url(paper_id: str) -> str: def paper_word_count(paper_id: str) -> int | None: - """Words in the paper's PDF, or None if it can't be fetched or read.""" - url = f"https://arxiv.org/pdf/{paper_id}" + """Words in the arXiv paper's PDF, or None if it can't be fetched or read.""" def skip(reason: str) -> None: logger.warning("arXiv %s (%s): %s; kept the abstract count", paper_id, abs_url(paper_id), reason) + return _pdf_words(f"https://arxiv.org/pdf/{paper_id}", skip) + + +def pdf_word_count(pdf_url: str, *, page_url: str) -> int | None: + """Words in the PDF a paper page names (build map stage 12), or None.""" + + def skip(reason: str) -> None: + logger.warning("Paper %s (PDF %s): %s; kept the page count", page_url, pdf_url, reason) + + return _pdf_words(pdf_url, skip) + + +def _pdf_words(url: str, skip) -> int | None: try: with httpx.Client(follow_redirects=True, timeout=_TIMEOUT, transport=_TRANSPORT) as client: with client.stream("GET", url) as response: @@ -103,6 +117,51 @@ def fix_saved_papers(*, sleep=time.sleep) -> tuple[int, int]: session.close() +def fix_saved_paper_pages(*, sleep=time.sleep) -> tuple[int, int, int]: + """One-off (stage 12): recheck saved non-arXiv items shown as Short for a PDF tag. + + Returns (counted from their PDF, no PDF tag, kept the page count).""" + from readurlist.db import Item, get_session + from readurlist.effort import effort + from readurlist.ingest import extract + + session = get_session() + try: + rows = [ + row + for row in session.query(Item).filter(Item.ingest_status == "ready").order_by(Item.id) + if row.word_count is not None and effort(row.word_count)[0] == "short" and not arxiv_id(row.url) + ] + fixed = no_tag = failed = 0 + for n, row in enumerate(rows): + if n: + sleep(2.0) # a polite pause between pages + try: + html = extract._fetch_html(row.url, timeout=30.0) + except extract.FetchError as exc: + why = f"HTTP {exc.status_code}" if exc.status_code else "no answer" + logger.warning("Paper %s: page could not be fetched (%s); kept the page count", row.url, why) + failed += 1 + continue + pdf_url = extract.pdf_link(html, row.url) + if not pdf_url: + no_tag += 1 + continue + words = pdf_word_count(pdf_url, page_url=row.url) + if words is None: + failed += 1 + continue + row.word_count = words + session.commit() + fixed += 1 + return fixed, no_tag, failed + except Exception: + session.rollback() + raise + finally: + session.close() + + def main() -> None: """CLI: `readurlist-arxiv-lengths` — fix the length of arXiv papers saved before stage 11.""" from readurlist.config import get_settings @@ -111,3 +170,13 @@ def main() -> None: init_db(get_settings().database_url) fixed, failed = fix_saved_papers() print(f"arXiv papers counted from their PDF: {fixed}; kept the abstract count: {failed}") + + +def paper_pages_main() -> None: + """CLI: `readurlist-paper-lengths` — fix the length of Short paper pages saved before stage 12.""" + from readurlist.config import get_settings + from readurlist.db import init_db + + init_db(get_settings().database_url) + fixed, no_tag, failed = fix_saved_paper_pages() + print(f"Short items counted from their PDF: {fixed}; no PDF tag: {no_tag}; kept the page count: {failed}") diff --git a/src/readurlist/ingest/extract.py b/src/readurlist/ingest/extract.py index 7250ca1..7026a53 100644 --- a/src/readurlist/ingest/extract.py +++ b/src/readurlist/ingest/extract.py @@ -4,7 +4,8 @@ import random import re import time -from urllib.parse import urlparse +from html.parser import HTMLParser +from urllib.parse import urljoin, urlparse import httpx import trafilatura @@ -50,6 +51,27 @@ def find_urls(text: str) -> list[str]: return URL_RE.findall(text or "") +class _PdfTag(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.href: str | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + a = dict(attrs) + if tag == "meta" and self.href is None and (a.get("name") or "").lower() == "citation_pdf_url": + self.href = (a.get("content") or "").strip() or None + + +def pdf_link(html: str, page_url: str) -> str | None: + """The paper's PDF named in the page's citation_pdf_url tag (build map stage 12).""" + parser = _PdfTag() + parser.feed(html or "") + if not parser.href: + return None + href = urljoin(page_url, parser.href) + return href if urlparse(href).scheme in ("http", "https") else None + + def _retry_after_seconds(response: httpx.Response, attempt: int) -> float: header = response.headers.get("Retry-After") if header: @@ -127,8 +149,8 @@ def _fetch_html(url: str, timeout: float) -> str: raise FetchError(f"Could not fetch page after {_MAX_ATTEMPTS} attempts: {last_exc}") -def extract_article(url: str, timeout: float = 30.0) -> tuple[str, str]: - """Fetch URL and extract main text. Returns (title, text). Raises on failure.""" +def extract_article(url: str, timeout: float = 30.0) -> tuple[str, str, str | None]: + """Fetch URL and extract main text. Returns (title, text, pdf link or None). Raises on failure.""" parsed = urlparse(url) if parsed.scheme not in ("http", "https"): raise ValueError(f"Unsupported URL scheme: {parsed.scheme}") @@ -152,4 +174,4 @@ def extract_article(url: str, timeout: float = 30.0) -> tuple[str, str]: title=title.strip() or url, ) - return title.strip(), extracted.strip() + return title.strip(), extracted.strip(), pdf_link(html, url) diff --git a/tests/test_arxiv.py b/tests/test_arxiv.py index 8343d9a..f0748f0 100644 --- a/tests/test_arxiv.py +++ b/tests/test_arxiv.py @@ -102,7 +102,7 @@ def test_arxiv_link_takes_its_length_from_the_paper(tmp_path, monkeypatch): def fake_extract(url, timeout=30.0): fetched.append(url) - return "Compost paper", ABSTRACT + return "Compost paper", ABSTRACT, None monkeypatch.setattr("readurlist.ingest.extract_article", fake_extract) monkeypatch.setattr("readurlist.ingest.paper_word_count", lambda paper_id: 9000) @@ -121,7 +121,7 @@ def fake_extract(url, timeout=30.0): def test_arxiv_link_keeps_the_abstract_count_when_the_paper_fails(tmp_path, monkeypatch): db = f"sqlite:///{tmp_path / 't.db'}" init_db(db) - monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT)) + monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT, None)) monkeypatch.setattr("readurlist.ingest.paper_word_count", lambda paper_id: None) item, _ = ingest_url("https://arxiv.org/abs/2410.12345", OneAnswerLLM(), _settings(db)) # type: ignore[arg-type] @@ -132,7 +132,7 @@ def test_arxiv_link_keeps_the_abstract_count_when_the_paper_fails(tmp_path, monk def test_other_links_do_not_fetch_a_paper(tmp_path, monkeypatch): db = f"sqlite:///{tmp_path / 't.db'}" init_db(db) - monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT)) + monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT, None)) def boom(paper_id): raise AssertionError("no paper fetch for ordinary links") @@ -205,3 +205,18 @@ def test_pdf_reader_notes_are_hidden(): import logging assert logging.getLogger("pypdf").getEffectiveLevel() > logging.ERROR + + +def test_an_arxiv_paper_is_not_fetched_again_from_its_page_tag(tmp_path, monkeypatch): + db = f"sqlite:///{tmp_path / 't.db'}" + init_db(db) + pdf = "https://arxiv.org/pdf/2608.00858" + monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT, pdf)) + monkeypatch.setattr("readurlist.ingest.paper_word_count", lambda paper_id: None) # withdrawn + + def boom(pdf_url, *, page_url): + raise AssertionError("the arXiv PDF was already tried") + + monkeypatch.setattr("readurlist.ingest.pdf_word_count", boom) + item, _ = ingest_url("https://arxiv.org/abs/2608.00858", OneAnswerLLM(), _settings(db)) # type: ignore[arg-type] + assert item.word_count == 120 diff --git a/tests/test_effort.py b/tests/test_effort.py index 2ac84a5..8abd1f0 100644 --- a/tests/test_effort.py +++ b/tests/test_effort.py @@ -46,7 +46,7 @@ def test_a_read_page_stores_its_word_count(tmp_path, monkeypatch): init_db(db) monkeypatch.setattr( "readurlist.ingest.extract_article", - lambda url, timeout=30.0: ("Title", "compost " * 500), + lambda url, timeout=30.0: ("Title", "compost " * 500, None), ) item, _ = ingest_url("https://example.com/a", FakeLLM(), _settings(db)) # type: ignore[arg-type] assert item.word_count == 500 diff --git a/tests/test_ingest_dedup.py b/tests/test_ingest_dedup.py index 3339545..d98358a 100644 --- a/tests/test_ingest_dedup.py +++ b/tests/test_ingest_dedup.py @@ -20,7 +20,7 @@ def test_same_url_does_not_insert_second_item(tmp_path, monkeypatch): init_db(f"sqlite:///{db}") def fake_extract(url: str, timeout: float = 30.0): - return "Example", "x" * 200 + return "Example", "x" * 200, None monkeypatch.setattr("readurlist.ingest.extract_article", fake_extract) monkeypatch.setattr("readurlist.ingest.summarize_article", lambda *a, **k: __import__( diff --git a/tests/test_ingest_vocab.py b/tests/test_ingest_vocab.py index 0c1d886..6e61ad3 100644 --- a/tests/test_ingest_vocab.py +++ b/tests/test_ingest_vocab.py @@ -39,7 +39,7 @@ def test_ingest_url_sees_existing_subjects(tmp_path, monkeypatch): monkeypatch.setattr( "readurlist.ingest.extract_article", - lambda url: ("Hint", "Body text " * 40), + lambda url: ("Hint", "Body text " * 40, None), ) settings = Settings( telegram_bot_token="x", diff --git a/tests/test_paper_pdf.py b/tests/test_paper_pdf.py new file mode 100644 index 0000000..40ad7d9 --- /dev/null +++ b/tests/test_paper_pdf.py @@ -0,0 +1,151 @@ +import pytest +from test_arxiv import _pdf, _transport + +from readurlist.ingest import arxiv +from readurlist.ingest.arxiv import pdf_word_count +from readurlist.ingest.extract import pdf_link + + +@pytest.mark.parametrize( + ("html", "expected"), + [ + ('', "https://journal.org/a.pdf"), + ("", "https://site.org/content/a.full.pdf"), + ('', "https://journal.org/a.pdf"), + ('', None), + ('', None), + ('', None), + ("

no tags

", None), + ], +) +def test_pdf_link(html, expected): + assert pdf_link(f"{html}", "https://site.org/doi/10.1/x") == expected + + +PAGE = "https://journal.org/doi/10.1/x" +PDF = "https://journal.org/doi/pdf/10.1/x" + + +def test_pdf_word_count_reads_any_pdf(monkeypatch): + seen: list[str] = [] + monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(_pdf("one two three"), seen)) + assert pdf_word_count(PDF, page_url=PAGE) == 3 + assert seen == [PDF] + + +def test_a_blocked_pdf_is_one_plain_line(monkeypatch, caplog): + monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(None, [])) + with caplog.at_level("WARNING"): + assert pdf_word_count(PDF, page_url=PAGE) is None + assert [r.getMessage() for r in caplog.records if r.name.startswith("readurlist")] == [ + f"Paper {PAGE} (PDF {PDF}): PDF not found (HTTP 404); kept the page count" + ] + + +def test_extract_article_returns_the_pdf_link(monkeypatch): + from readurlist.ingest.extract import extract_article + + body = "

" + "We study compost in gardens. " * 20 + "

" + html = f'A paper
{body}
' + monkeypatch.setattr("readurlist.ingest.extract._fetch_html", lambda url, timeout: html) + title, text, pdf = extract_article(PAGE) + assert "compost" in text + assert pdf == "https://journal.org/pdf/x" + + +def _ingest(tmp_path, monkeypatch, pdf, count): + from test_arxiv import ABSTRACT, OneAnswerLLM, _settings + + from readurlist.db import init_db + from readurlist.ingest import ingest_url + + db = f"sqlite:///{tmp_path / 't.db'}" + init_db(db) + monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("T", ABSTRACT, pdf)) + asked: list[tuple[str, str]] = [] + + def fake_count(pdf_url, *, page_url): + asked.append((pdf_url, page_url)) + return count + + monkeypatch.setattr("readurlist.ingest.pdf_word_count", fake_count) + item, _ = ingest_url(PAGE, OneAnswerLLM(), _settings(db)) # type: ignore[arg-type] + return item, asked + + +def test_a_paper_page_takes_its_length_from_its_pdf(tmp_path, monkeypatch): + item, asked = _ingest(tmp_path, monkeypatch, PDF, 9000) + assert asked == [(PDF, PAGE)] + assert item.word_count == 9000 + assert "We study compost." in item.extracted_text # the page's text is kept, not the PDF's + + +def test_a_paper_page_keeps_its_count_when_the_pdf_fails(tmp_path, monkeypatch): + item, _ = _ingest(tmp_path, monkeypatch, PDF, None) + assert item.word_count == 120 + + +def test_a_page_without_the_tag_fetches_no_pdf(tmp_path, monkeypatch): + item, asked = _ingest(tmp_path, monkeypatch, None, 9000) + assert asked == [] + assert item.word_count == 120 + + +def test_one_off_fix_rechecks_short_paper_pages(tmp_path, monkeypatch, caplog): + from test_arxiv import ABSTRACT + + from readurlist.db import Item, get_session, init_db + from readurlist.ingest.arxiv import fix_saved_paper_pages + from readurlist.ingest.extract import FetchError + + init_db(f"sqlite:///{tmp_path / 't.db'}") + session = get_session() + for url, status, words in [ + ("https://journal.org/paper", "ready", 120), # Short, has a tag: fixed + ("https://blog.org/post", "ready", 900), # Short, no tag: left alone + ("https://journal.org/blocked", "ready", 150), # Short, PDF blocked: kept + ("https://gone.org/page", "ready", 200), # Short, page gone: kept + ("https://journal.org/long", "ready", 5000), # Long: not rechecked + ("https://arxiv.org/abs/2410.00001", "ready", 120), # arXiv: stage 11's fix + ("https://journal.org/pending", "pending_body", 120), # not ready + ]: + session.add(Item(url=url, title="T", snapshot="s", subject="a", topics=[], keywords=[], + extracted_text=ABSTRACT, priority=3, ingest_status=status, word_count=words)) + session.commit() + session.close() + tag = '' + pages = { + "https://journal.org/paper": tag.format("/paper.pdf"), + "https://blog.org/post": "

no tag

", + "https://journal.org/blocked": tag.format("/blocked.pdf"), + } + fetched: list[str] = [] + + def fake_fetch(url, timeout): + fetched.append(url) + if url not in pages: + raise FetchError("Could not fetch page (HTTP 404).", status_code=404) + return pages[url] + + monkeypatch.setattr("readurlist.ingest.extract._fetch_html", fake_fetch) + counts = {"https://journal.org/paper.pdf": 8000, "https://journal.org/blocked.pdf": None} + monkeypatch.setattr(arxiv, "pdf_word_count", lambda pdf_url, *, page_url: counts[pdf_url]) + waits: list[float] = [] + + with caplog.at_level("WARNING"): + result = fix_saved_paper_pages(sleep=waits.append) + + assert "Paper https://gone.org/page: page could not be fetched (HTTP 404); kept the page count" in [ + r.getMessage() for r in caplog.records + ] + assert result == (1, 1, 2) # counted from their PDF, no PDF tag, kept the page count + assert fetched == ["https://journal.org/paper", "https://blog.org/post", + "https://journal.org/blocked", "https://gone.org/page"] + assert waits == [2.0, 2.0, 2.0] + session = get_session() + words = {i.url: i.word_count for i in session.query(Item)} + session.close() + assert words["https://journal.org/paper"] == 8000 + assert words["https://blog.org/post"] == 900 + assert words["https://journal.org/blocked"] == 150 + assert words["https://gone.org/page"] == 200 diff --git a/tests/test_tripwire.py b/tests/test_tripwire.py index a9fd703..e14c0c6 100644 --- a/tests/test_tripwire.py +++ b/tests/test_tripwire.py @@ -28,7 +28,7 @@ def _settings(db: str) -> Settings: def _setup(tmp_path, monkeypatch, text: str): db = f"sqlite:///{tmp_path / 't.db'}" init_db(db) - monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("Title", text)) + monkeypatch.setattr("readurlist.ingest.extract_article", lambda url, timeout=30.0: ("Title", text, None)) return db From 5ee1a27707bb84d92d2580c507838a57e335ef0e Mon Sep 17 00:00:00 2001 From: Alex Lee <13192723+leeaky@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:04:00 +1100 Subject: [PATCH 3/3] Map: stage 12 check-log row from the owner's explain-back Co-Authored-By: Claude Opus 5.5 --- docs/build-map.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/build-map.md b/docs/build-map.md index 1f15c61..f2608c4 100644 --- a/docs/build-map.md +++ b/docs/build-map.md @@ -431,6 +431,7 @@ Before any feature work, dev and live are separated: the live worker, bot, datab | 9 Library tidy-up | Skip P on All, then restore a month later. First answer: "to be honest, i do not know"; explained from the sources. Explain-back: "once skipped, the url goes to a skipped list on the website. It can be restored but the scoring is lower due to recency." Matches. Owner then asked for a change: "I would suggest if restored, the date restarts to restoration date" (added to stage 5 (c)). | Context §Ranking (skip leaves the queue without marking read); README §Website (Skipped filter, restore) | 2026-10-31 | | 10 Is it working? | Week: 12 picks shown, 5 read, 3 skipped, 10 saved, pile 140 last Monday. First: "not sure i get the context of whether it is working"; explained (pile 140 + 10 − 5 − 3 = 142). Explain-back: "the pile did not shrink so no it did not work". Matches Context's test as written. Raised Q9: the pile measure mixes list quality with how much is saved. | Context §Success metric | 2026-10-31 | | 11 arXiv papers get their real length | During the live one-off fix, a paper's PDF came back "404 Not Found" (2608.00858). Owner looked it up: "Data-Driven Measures of High-Frequency - it is withdrawn by the author", and its card on All: "short 1min". Matches: no PDF, so the abstract's count is kept (stage 11 (a) Detail). | Stage 11 (a) Detail; live run output | 2026-11-01 | +| 12 Papers from other sites get their real length | Owner: "stage 12 extends what we did for arxiv to other site where there is a tagged pdf paper". Matches: same rules as stage 11 (length from the PDF, summary from the page, page count kept if the PDF fails), for any page with a `citation_pdf_url` tag. Added: papers already saved are rechecked once, Short ones only. | Stage 12 (a), (b) | 2026-11-01 | | 2–10 Deploy order (changed 2026-10-31) | Stage 4 merged an hour before its effort column is added to the live database: "the website will display errors when looking for the non-existent effort column in that hour". Matches; added: the live worker keeps running fine that hour because it only gets the new code at `git pull` and restart; done the right way round, the old site and worker simply ignore the new column until the merge, so nothing breaks. Covers the Deploy tasks and `Changed:` lines of stages 2–10; stage 1 changed only its dependency on stage 0, covered by the stage 0 row. | R-0009 §What "apart" means, §On the map; owner's deploy-order rule of 2026-10-31 | 2026-10-31 | ## Code overlay